input CHANGE optimize newlines counting in input handler
Moves counting newlines from input handler functions to the parsers
which have much better idea about the content of the data beeing
processed and can better optimize counting newlines. Counting newlines
in the input handler functions mostly caused reading data twice.
diff --git a/src/in.c b/src/in.c
index 46673e9..a9c99a9 100644
--- a/src/in.c
+++ b/src/in.c
@@ -349,12 +349,6 @@
return LY_EDENIED;
}
- for (size_t i = 0; i < count; i++) {
- if (in->current[i] == '\n') {
- LY_IN_NEW_LINE(in);
- }
- }
-
memcpy(buf, in->current, count);
in->current += count;
return LY_SUCCESS;
@@ -374,12 +368,6 @@
return LY_EDENIED;
}
- for (size_t i = 0; i < count; i++) {
- if (in->current[i] == '\n') {
- LY_IN_NEW_LINE(in);
- }
- }
-
in->current += count;
return LY_SUCCESS;
}
diff --git a/src/in_internal.h b/src/in_internal.h
index 11f33c3..35e90fc 100644
--- a/src/in_internal.h
+++ b/src/in_internal.h
@@ -48,6 +48,9 @@
/**
* @brief Read bytes from an input.
*
+ * Does not count new lines, which is expected from the caller who has better idea about
+ * the content of the read data and can better optimize counting.
+ *
* @param[in] in Input structure.
* @param[in] buf Destination buffer.
* @param[in] count Number of bytes to read.
@@ -59,6 +62,9 @@
/**
* @brief Just skip bytes in an input.
*
+ * Does not count new lines, which is expected from the caller who has better idea about
+ * the content of the skipped data and can better optimize counting.
+ *
* @param[in] in Input structure.
* @param[in] count Number of bytes to skip.
* @return LY_SUCCESS on success,
diff --git a/src/json.c b/src/json.c
index eca191e..5bcb852 100644
--- a/src/json.c
+++ b/src/json.c
@@ -71,6 +71,9 @@
{
/* skip leading whitespaces */
while (*jsonctx->in->current != '\0' && is_jsonws(*jsonctx->in->current)) {
+ if (*jsonctx->in->current == '\n') {
+ LY_IN_NEW_LINE(jsonctx->in);
+ }
ly_in_skip(jsonctx->in, 1);
}
if (*jsonctx->in->current == '\0') {
diff --git a/src/parser_yang.c b/src/parser_yang.c
index f3d98fe..a35e904 100644
--- a/src/parser_yang.c
+++ b/src/parser_yang.c
@@ -153,6 +153,7 @@
ctx->in->current -= len;
if (c == '\n') {
ctx->indent = 0;
+ LY_IN_NEW_LINE(ctx->in);
} else {
/* note - even the multibyte character is count as 1 */
++ctx->indent;
@@ -427,6 +428,8 @@
switch (ctx->in->current[0]) {
case 'n':
ctx->in->current = "\n";
+ /* fix false newline count in buf_store_char() */
+ ctx->in->line--;
break;
case 't':
ctx->in->current = "\t";
@@ -455,6 +458,8 @@
need_buf = 1;
break;
case '\n':
+ LY_IN_NEW_LINE(ctx->in);
+ /* fall through */
case ' ':
case '\t':
/* just skip */
@@ -468,6 +473,8 @@
case STRING_PAUSED_CONTINUE:
switch (ctx->in->current[0]) {
case '\n':
+ LY_IN_NEW_LINE(ctx->in);
+ /* fall through */
case ' ':
case '\t':
/* skip */
@@ -584,6 +591,7 @@
/* word is finished */
goto str_end;
}
+ LY_IN_NEW_LINE(ctx->in);
MOVE_INPUT(ctx, 1);
/* reset indent */
@@ -667,6 +675,7 @@
continue;
case '\n':
/* skip whitespaces (optsep) */
+ LY_IN_NEW_LINE(ctx->in);
ctx->indent = 0;
break;
case ' ':
@@ -4453,6 +4462,9 @@
LY_CHECK_RET(skip_comment(ctx, 2));
} else if (isspace(ctx->in->current[0])) {
/* whitespace */
+ if (ctx->in->current[0] == '\n') {
+ LY_IN_NEW_LINE(ctx->in);
+ }
ly_in_skip(ctx->in, 1);
} else {
break;
diff --git a/src/parser_yin.c b/src/parser_yin.c
index c13ce60..171fb36 100644
--- a/src/parser_yin.c
+++ b/src/parser_yin.c
@@ -3818,6 +3818,9 @@
/* skip possible trailing whitespaces at end of the input */
while (isspace(in->current[0])) {
+ if (in->current[0] == '\n') {
+ LY_IN_NEW_LINE(in);
+ }
ly_in_skip(in, 1);
}
if (in->current[0]) {
@@ -3878,6 +3881,9 @@
/* skip possible trailing whitespaces at end of the input */
while (isspace(in->current[0])) {
+ if (in->current[0] == '\n') {
+ LY_IN_NEW_LINE(in);
+ }
ly_in_skip(in, 1);
}
if (in->current[0]) {
diff --git a/src/xml.c b/src/xml.c
index d72d07f..cf6810a 100644
--- a/src/xml.c
+++ b/src/xml.c
@@ -37,26 +37,38 @@
LY_CHECK_ERR_RET(!c->in->current[0], LOGVAL(c->ctx, LY_VCODE_EOF), LY_EVALID)
/* Ignore whitespaces in the input string p */
-#define ign_xmlws(c) while (is_xmlws(*(c)->in->current)) {ly_in_skip(c->in, 1);}
+#define ign_xmlws(c) \
+ while (is_xmlws(*(c)->in->current)) { \
+ if (*(c)->in->current == '\n') { \
+ LY_IN_NEW_LINE((c)->in); \
+ } \
+ ly_in_skip(c->in, 1); \
+ }
static LY_ERR lyxml_next_attr_content(struct lyxml_ctx *xmlctx, const char **value, size_t *value_len, ly_bool *ws_only,
ly_bool *dynamic);
/**
- * @brief Ignore any characters until the delim of the size delim_len is read
+ * @brief Ignore and skip any characters until the delim of the size delim_len is read, including the delim
*
- * Detects number of read new lines.
- * Returns Boolean value whether delim was found or not.
+ * @param[in] xmlctx XML parser context to provide input handler and libyang context
+ * @param[in] in input handler to read the data, it is updated only in case the section is correctly terminated.
+ * @param[in] delim Delimiter to detect end of the section.
+ * @param[in] delim_len Length of the delimiter string to use.
+ * @param[in] sectname Section name to refer in error message.
*/
-static ly_bool
-ign_todelim(register const char *input, const char *delim, size_t delim_len, size_t *parsed)
+LY_ERR
+skip_section(struct lyxml_ctx *xmlctx, const char *delim, size_t delim_len, const char *sectname)
{
size_t i;
- register const char *a, *b;
+ register const char *input, *a, *b;
+ uint64_t parsed = 0, newlines = 0;
- (*parsed) = 0;
- for ( ; *input; ++input, ++(*parsed)) {
+ for (input = xmlctx->in->current; *input; ++input, ++parsed) {
if (*input != *delim) {
+ if (*input == '\n') {
+ ++newlines;
+ }
continue;
}
a = input;
@@ -68,12 +80,16 @@
}
if (i == delim_len) {
/* delim found */
- return 0;
+ xmlctx->in->line += newlines;
+ ly_in_skip(xmlctx->in, parsed + delim_len);
+ return LY_SUCCESS;
}
}
- /* delim not found */
- return 1;
+ /* delim not found,
+ * do not update input handler to refer to the beginning of the section in error message */
+ LOGVAL(xmlctx->ctx, LY_VCODE_NTERM, sectname);
+ return LY_EVALID;
}
/**
@@ -224,8 +240,7 @@
{
const struct ly_ctx *ctx = xmlctx->ctx; /* shortcut */
const char *endtag, *sectname;
- size_t endtag_len, parsed;
- ly_bool rc;
+ size_t endtag_len;
while (1) {
ign_xmlws(xmlctx);
@@ -267,13 +282,9 @@
LOGVAL(ctx, LYVE_SYNTAX, "Unknown XML section \"%.20s\".", &xmlctx->in->current[-2]);
return LY_EVALID;
}
- rc = ign_todelim(xmlctx->in->current, endtag, endtag_len, &parsed);
- LY_CHECK_ERR_RET(rc, LOGVAL(ctx, LY_VCODE_NTERM, sectname), LY_EVALID);
- ly_in_skip(xmlctx->in, parsed + endtag_len);
+ LY_CHECK_RET(skip_section(xmlctx, endtag, endtag_len, sectname));
} else if (xmlctx->in->current[0] == '?') {
- rc = ign_todelim(xmlctx->in->current, "?>", 2, &parsed);
- LY_CHECK_ERR_RET(rc, LOGVAL(ctx, LY_VCODE_NTERM, "Declaration"), LY_EVALID);
- ly_in_skip(xmlctx->in, parsed + 2);
+ LY_CHECK_RET(skip_section(xmlctx, "?>", 2, "Declaration"));
} else {
/* other non-WS character */
break;
diff --git a/tests/utests/schema/test_parser_yang.c b/tests/utests/schema/test_parser_yang.c
index c1fcab7..f1ab8da 100644
--- a/tests/utests/schema/test_parser_yang.c
+++ b/tests/utests/schema/test_parser_yang.c
@@ -180,14 +180,16 @@
CHECK_LOG_CTX("Invalid identifier character ':' (0x003a).", "Line number 1.");
}
-#define TEST_GET_ARGUMENT_SUCCESS(INPUT_TEXT, CTX, ARG_TYPE, EXPECT_WORD, EXPECT_LEN, EXPECT_CURRENT)\
+#define TEST_GET_ARGUMENT_SUCCESS(INPUT_TEXT, CTX, ARG_TYPE, EXPECT_WORD, EXPECT_LEN, EXPECT_CURRENT, EXPECT_LINE)\
{\
- const char * text = INPUT_TEXT;\
+ const char * text = INPUT_TEXT;\
+ in.line = 1;\
in.current = text;\
assert_int_equal(LY_SUCCESS, get_argument(CTX, Y_MAYBE_STR_ARG, NULL, &word, &buf, &len));\
assert_string_equal(word, EXPECT_WORD);\
assert_int_equal(len, EXPECT_LEN);\
assert_string_equal(EXPECT_CURRENT, in.current);\
+ assert_int_equal(EXPECT_LINE, in.line);\
}
static void
@@ -195,27 +197,27 @@
{
char *word, *buf;
size_t len;
- const char *in_text;
YCTX_INIT;
- // in.current = " // this is a text of / one * line */ comment\nargument;";
- in_text = " // this is a text of / one * line */ comment\nargument;";
- TEST_GET_ARGUMENT_SUCCESS(in_text, YCTX, Y_STR_ARG, "argument;", 8, ";");
+ TEST_GET_ARGUMENT_SUCCESS(" // this is a text of / one * line */ comment\nargument;",
+ YCTX, Y_STR_ARG, "argument;", 8, ";", 2);
assert_null(buf);
- in_text = "/* this is a \n * text // of / block * comment */\"arg\" + \"ume\" \n + \n \"nt\";";
- TEST_GET_ARGUMENT_SUCCESS(in_text, YCTX, Y_STR_ARG, "argument", 8, ";");
+ TEST_GET_ARGUMENT_SUCCESS("/* this is a \n * text // of / block * comment */\"arg\" + \"ume\" \n + \n \"nt\";",
+ YCTX, Y_STR_ARG, "argument", 8, ";", 4);
assert_ptr_equal(buf, word);
free(word);
+ in.line = 1;
in.current = " this is one line comment on last line";
assert_int_equal(LY_SUCCESS, skip_comment(YCTX, 1));
assert_true(in.current[0] == '\0');
+ in.line = 1;
in.current = " this is a not terminated comment x";
assert_int_equal(LY_EVALID, skip_comment(YCTX, 2));
- CHECK_LOG_CTX("Unexpected end-of-input, non-terminated comment.", "Line number 5.");
+ CHECK_LOG_CTX("Unexpected end-of-input, non-terminated comment.", "Line number 1.");
assert_true(in.current[0] == '\0');
}
@@ -241,7 +243,7 @@
assert_int_equal(LY_EVALID, get_argument(YCTX, Y_STR_ARG, NULL, &word, &buf, &len));
CHECK_LOG_CTX("Double-quoted string unknown special character \'\\s\'.", "Line number 1.");
- TEST_GET_ARGUMENT_SUCCESS("\'\\s\'", YCTX, Y_STR_ARG, "\\s\'", 2, "");
+ TEST_GET_ARGUMENT_SUCCESS("\'\\s\'", YCTX, Y_STR_ARG, "\\s\'", 2, "", 1);
/* invalid character after the argument */
in.current = "hello\"";
@@ -265,63 +267,65 @@
CHECK_LOG_CTX("Statement argument is required.", "Line number 1.");
/* slash is not an invalid character */
- TEST_GET_ARGUMENT_SUCCESS("hello/x\t", YCTX, Y_STR_ARG, "hello/x\t", 7, "\t");
+ TEST_GET_ARGUMENT_SUCCESS("hello/x\t", YCTX, Y_STR_ARG, "hello/x\t", 7, "\t", 1);
assert_null(buf);
/* different quoting */
- TEST_GET_ARGUMENT_SUCCESS("hello/x\t", YCTX, Y_STR_ARG, "hello/x\t", 7, "\t");
+ TEST_GET_ARGUMENT_SUCCESS("hello/x\t", YCTX, Y_STR_ARG, "hello/x\t", 7, "\t", 1);
- TEST_GET_ARGUMENT_SUCCESS("hello ", YCTX, Y_STR_ARG, "hello ", 5, " ");
+ TEST_GET_ARGUMENT_SUCCESS("hello ", YCTX, Y_STR_ARG, "hello ", 5, " ", 1);
- TEST_GET_ARGUMENT_SUCCESS("hello/*comment*/\n", YCTX, Y_STR_ARG, "hello/*comment*/\n", 5, "\n");
+ TEST_GET_ARGUMENT_SUCCESS("hello/*comment*/\n", YCTX, Y_STR_ARG, "hello/*comment*/\n", 5, "\n", 1);
- TEST_GET_ARGUMENT_SUCCESS("\"hello\\n\\t\\\"\\\\\";", YCTX, Y_STR_ARG, "hello\n\t\"\\", 9, ";");
+ TEST_GET_ARGUMENT_SUCCESS("\"hello\\n\\t\\\"\\\\\";", YCTX, Y_STR_ARG, "hello\n\t\"\\", 9, ";", 1);
free(buf);
YCTX->indent = 14;
/* - space and tabs before newline are stripped out
* - space and tabs after newline (indentation) are stripped out
*/
- TEST_GET_ARGUMENT_SUCCESS("\"hello \t\n\t\t world!\"", YCTX, Y_STR_ARG, "hello\n world!", 14, "");
+ TEST_GET_ARGUMENT_SUCCESS("\"hello \t\n\t\t world!\"", YCTX, Y_STR_ARG, "hello\n world!", 14, "", 2);
free(buf);
/* In contrast to previous, the backslash-escaped tabs are expanded after trimming, so they are preserved */
YCTX->indent = 14;
- TEST_GET_ARGUMENT_SUCCESS("\"hello \\t\n\t\\t world!\"", YCTX, Y_STR_ARG, "hello \t\n\t world!", 16, "");
+ TEST_GET_ARGUMENT_SUCCESS("\"hello \\t\n\t\\t world!\"", YCTX, Y_STR_ARG, "hello \t\n\t world!", 16, "", 2);
assert_ptr_equal(word, buf);
free(buf);
/* Do not handle whitespaces after backslash-escaped newline as indentation */
YCTX->indent = 14;
- TEST_GET_ARGUMENT_SUCCESS("\"hello\\n\t\t world!\"", YCTX, Y_STR_ARG, "hello\n\t\t world!", 15, "");
+ TEST_GET_ARGUMENT_SUCCESS("\"hello\\n\t\t world!\"", YCTX, Y_STR_ARG, "hello\n\t\t world!", 15, "", 1);
assert_ptr_equal(word, buf);
free(buf);
YCTX->indent = 14;
- TEST_GET_ARGUMENT_SUCCESS("\"hello\n \tworld!\"", YCTX, Y_STR_ARG, "hello\nworld!", 12, "");
+ TEST_GET_ARGUMENT_SUCCESS("\"hello\n \tworld!\"", YCTX, Y_STR_ARG, "hello\nworld!", 12, "", 2);
assert_ptr_equal(word, buf);
free(buf);
- TEST_GET_ARGUMENT_SUCCESS("\'hello\'", YCTX, Y_STR_ARG, "hello'", 5, "");
+ TEST_GET_ARGUMENT_SUCCESS("\'hello\'", YCTX, Y_STR_ARG, "hello'", 5, "", 1);
- TEST_GET_ARGUMENT_SUCCESS("\"hel\" +\t\n\"lo\"", YCTX, Y_STR_ARG, "hello", 5, "");
+ TEST_GET_ARGUMENT_SUCCESS("\"hel\" +\t\n\"lo\"", YCTX, Y_STR_ARG, "hello", 5, "", 2);
assert_ptr_equal(word, buf);
free(buf);
+ in.line = 1;
in.current = "\"hel\" +\t\nlo"; /* unquoted the second part */
assert_int_equal(LY_EVALID, get_argument(YCTX, Y_STR_ARG, NULL, &word, &buf, &len));
- CHECK_LOG_CTX("Both string parts divided by '+' must be quoted.", "Line number 8.");
+ CHECK_LOG_CTX("Both string parts divided by '+' must be quoted.", "Line number 2.");
- TEST_GET_ARGUMENT_SUCCESS("\'he\'\t\n+ \"llo\"", YCTX, Y_STR_ARG, "hello", 5, "");
+ TEST_GET_ARGUMENT_SUCCESS("\'he\'\t\n+ \"llo\"", YCTX, Y_STR_ARG, "hello", 5, "", 2);
free(buf);
- TEST_GET_ARGUMENT_SUCCESS(" \t\n\"he\"+\'llo\'", YCTX, Y_STR_ARG, "hello", 5, "");
+ TEST_GET_ARGUMENT_SUCCESS(" \t\n\"he\"+\'llo\'", YCTX, Y_STR_ARG, "hello", 5, "", 2);
free(buf);
/* missing argument */
+ in.line = 1;
in.current = ";";
assert_int_equal(LY_EVALID, get_argument(YCTX, Y_STR_ARG, NULL, &word, &buf, &len));
- CHECK_LOG_CTX("Invalid character sequence \";\", expected an argument.", "Line number 10.");
+ CHECK_LOG_CTX("Invalid character sequence \";\", expected an argument.", "Line number 1.");
}
#define TEST_STMS_SUCCESS(INPUT_TEXT, CTX, ACTION, EXPECT_WORD)\