instanceid BUGFIX nested types for LYB

While LYB and JSON are equal for instance-identifier,
they are not for many other types, which can be nested
in a LYB instance-identifier value.

Fixes sysrepo/sysrepo#2686
diff --git a/src/plugins_types/node_instanceid.c b/src/plugins_types/node_instanceid.c
index 9edbf9f..31781d6 100644
--- a/src/plugins_types/node_instanceid.c
+++ b/src/plugins_types/node_instanceid.c
@@ -189,9 +189,11 @@
         LY_CHECK_GOTO(ret = lys_compile_expr_implement(ctx, exp, format, prefix_data, 1, unres, NULL), cleanup);
     }
 
-    /* resolve it on schema tree */
+    /* resolve it on schema tree, use JSON format instead of LYB because for this type they are equal but for some
+     * nested types (such as numbers in predicates in the path) LYB would be invalid */
     ret = ly_path_compile(ctx, NULL, ctx_node, NULL, exp, (ctx_node->flags & LYS_IS_OUTPUT) ?
-            LY_PATH_OPER_OUTPUT : LY_PATH_OPER_INPUT, LY_PATH_TARGET_MANY, 1, format, prefix_data, &path);
+            LY_PATH_OPER_OUTPUT : LY_PATH_OPER_INPUT, LY_PATH_TARGET_MANY, 1, (format == LY_VALUE_LYB) ?
+            LY_VALUE_JSON : format, prefix_data, &path);
     if (ret) {
         ret = ly_err_new(err, ret, LYVE_DATA, NULL, NULL,
                 "Invalid instance-identifier \"%.*s\" value - semantic error.", (int)value_len, (char *)value);