Fix tools to show available datamodel correctly

.gitlab-ci.yml

@@ -47,7 +47,7 @@ run_tools_test:
       password $CI_JOB_TOKEN
       " > ~/.netrc
   - "./gitlab-ci/pipeline_setup.sh"
-  - "./gitlab-ci/setup.sh ms"
+  - "./gitlab-ci/setup.sh"
   - "./gitlab-ci/tools-test.sh"
   - "./gitlab-ci/generate_supported_dm.sh"
 

gitlab-ci/generate_supported_dm.sh

@@ -20,11 +20,9 @@ install_libeasy
 install_libethernet
 install_libqos
 
-[ ! -d "${BBFDM_MS_DIR}" ] && {
+[ ! -d "${BBFDM_MS_DIR}" ] && mkdir -p "${BBFDM_MS_DIR}"
-	mkdir -p "${BBFDM_MS_DIR}"
+rm -rf ${BBFDM_MS_DIR}/*
-	rm -rf ${BBFDM_MS_DIR}/*
+mkdir -p ${BBFDM_MS_DIR}/core
-	mkdir -p ${BBFDM_MS_DIR}/core
-}
 
 if [ -z "${1}" ]; then
 	./tools/generate_dm.py tools/tools_input.json
@@ -47,4 +45,13 @@ echo "Validate datamodel_default generated XML file"
 xmllint --schema test/tools/cwmp-datamodel-*.xsd out/datamodel_default.xml --noout
 check_ret $?
 
+# Check if the specified log file exists, which indicates errors during plugin loading
+if [ -f ${BBFDM_LOG_FILE} ]; then
+	echo "Some plugins failed to load! Please check the errors below"
+	echo "*****************************************************"
+	cat "${BBFDM_LOG_FILE}"
+	echo "*****************************************************"
+	exit 1
+fi
+
 echo "Generation of xml and xls artifacts :: PASS"

libbbfdm-api/legacy/plugin/dotso_plugin.c

@@ -69,6 +69,9 @@ int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path)
 #endif
 	if (!handle) {
 		char *err_msg = dlerror();
+#ifdef BBF_SCHEMA_FULL_TREE
+		TRACE_FILE("Failed to add DotSo plugin '%s', [%s]\n", plugin_path, err_msg);
+#endif
 		BBF_ERR("Failed to add DotSo plugin '%s', [%s]\n", plugin_path, err_msg);
 		return 0;
 	}
@@ -79,6 +82,9 @@ int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path)
 
 	if (dynamic_obj == NULL) {
 		dlclose(handle);
+#ifdef BBF_SCHEMA_FULL_TREE
+		TRACE_FILE("Plugin %s missing init symbol ...\n", plugin_path);
+#endif
 		BBF_ERR("Plugin %s missing init symbol ...", plugin_path);
 		return 0;
 	}
@@ -87,6 +93,9 @@ int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path)
 
 		DMOBJ *dm_entryobj = find_entry_obj(entryobj, dynamic_obj[i].path);
 		if (!dm_entryobj) {
+#ifdef BBF_SCHEMA_FULL_TREE
+			TRACE_FILE("Failed to add DotSo plugin '%s' to main tree with parent DM index '%d' => '%s'", plugin_path, i, dynamic_obj[i].path);
+#endif
 			BBF_ERR("Failed to add DotSo plugin '%s' to main tree with parent DM index '%d' => '%s'", plugin_path, i, dynamic_obj[i].path);
 			continue;
 		}

libbbfdm-ubus/plugin.c

@@ -129,7 +129,12 @@ int bbfdm_free_dotso_plugin(struct bbfdm_context *bbfdm_ctx, void **lib_handle)
 	return 0;
 }
 
-static int bbfdm_load_json_plugin(struct bbfdm_context *bbfdm_ctx, struct list_head *json_plugin, struct list_head *json_list,
+static int browse_obj(struct dmctx *dmctx, DMNODE *parent_node, void *prev_data, char *prev_instance)
+{
+	return 0;
+}
+
+int bbfdm_load_json_plugin(struct bbfdm_context *bbfdm_ctx, struct list_head *json_plugin, struct list_head *json_list,
 		struct list_head *json_memhead, const char *file_path, DMOBJ **main_entry)
 {
 	DMOBJ *dm_entryobj = NULL;
@@ -187,6 +192,16 @@ static int bbfdm_load_json_plugin(struct bbfdm_context *bbfdm_ctx, struct list_h
 			return -1;
 		}
 
+		// Parent Path is multi-instance object
+		bool multi_instances = false;
+		if (node_obj[obj_prefix_len - 1] == '.' &&
+				node_obj[obj_prefix_len] == '{' &&
+				node_obj[obj_prefix_len + 1] == 'i' &&
+				node_obj[obj_prefix_len + 2] == '}' &&
+				node_obj[obj_prefix_len + 3] == '.') {
+			multi_instances = true;
+		}
+			
 		// Remove '.' from object prefix
 		if (obj_prefix[obj_prefix_len - 1] == '.')
 			obj_prefix[obj_prefix_len - 1] = 0;
@@ -206,6 +221,7 @@ static int bbfdm_load_json_plugin(struct bbfdm_context *bbfdm_ctx, struct list_h
 		dm_entryobj[idx].permission = &DMREAD;
 		dm_entryobj[idx].nextobj = (DMOBJ *)dm_dynamic_calloc(json_memhead, 2, sizeof(DMOBJ));
 		dm_entryobj[idx].leaf = NULL;
+		dm_entryobj[idx].browseinstobj = multi_instances ? browse_obj : NULL;
 		dm_entryobj[idx].bbfdm_type = BBFDM_BOTH;
 
 		parse_obj(node_obj, jobj, dm_entryobj[idx].nextobj, 0, json_plugin_version, json_list);

libbbfdm-ubus/plugin.h

@@ -16,6 +16,8 @@ int bbfdm_load_external_plugin(struct bbfdm_context *bbfdm_ctx, void **lib_handl
 int bbfdm_load_dotso_plugin(struct bbfdm_context *bbfdm_ctx, void **lib_handle, const char *file_path, DMOBJ **main_entry);
 int bbfdm_free_dotso_plugin(struct bbfdm_context *bbfdm_ctx, void *lib_handle);
 
+int bbfdm_load_json_plugin(struct bbfdm_context *bbfdm_ctx, struct list_head *json_plugin, struct list_head *json_list,
+		struct list_head *json_memhead, const char *file_path, DMOBJ **main_entry);
 int bbfdm_free_json_plugin(void);
 
 #endif /* PLUGIN_H */

test/tools/dm-cli.c

@@ -10,6 +10,10 @@
 #include "../../libbbfdm-ubus/plugin.h"
 #include "../../libbbfdm/device.h"
 
+extern struct list_head loaded_json_files;
+extern struct list_head json_list;
+extern struct list_head json_memhead;
+
 static int cli_exec_schema(struct dmctx *bbfdm_ctx, char *in_path)
 {
 	int err = 0;
@@ -84,6 +88,9 @@ int main(int argc, char **argv)
 	if (plugin_path == NULL) {
 		err = bbfdm_load_internal_plugin(NULL, tDynamicObj, &CLI_DM_ROOT_OBJ);
 	} else {
+		if (strstr(plugin_path, ".json") != NULL)
+			err = bbfdm_load_json_plugin(NULL, &loaded_json_files, &json_list, &json_memhead, plugin_path, &CLI_DM_ROOT_OBJ);
+		else
 			err = bbfdm_load_dotso_plugin(NULL, &cli_lib_handle, plugin_path, &CLI_DM_ROOT_OBJ);
 	}
 
@@ -112,5 +119,8 @@ int main(int argc, char **argv)
 	// Free plugin handle
 	bbfdm_free_dotso_plugin(NULL, &cli_lib_handle);
 
+	// Free JSON plugin handle
+	bbfdm_free_json_plugin();
+
 	return err;
 }

tools/bbf_common.py

@@ -416,6 +416,9 @@ def download_and_build_plugins(plugins, vendor_prefix):
                     if filename.endswith('.c'):
                         LIST_FILES.append(filename)
                     elif filename.endswith('.json'):
+                        if is_microservice is True:
+                            install_json_plugin(filename, "/usr/share/bbfdm/micro_services/"+f"{plugin_index}_{name}.json", prefix)
+                        else:
                             install_json_plugin(filename, "/usr/share/bbfdm/micro_services/core/"+f"{plugin_index}_{name}.json", prefix)
                     else:
                         BBF_ERROR_CODE += 1

tools/generate_dm_xml.py

@@ -114,8 +114,8 @@ def get_info_from_json(data, dm_json_files=None):
                     if i != (len(list_data) - 1) and list_data[i + 1] == list_data[i] + "{i}.":
                         continue
                     try:
-                        if str(list_data[i]).find("X_IOWRT_EU_") != -1:
+                        if str(list_data[i]).find("X_IOWRT_EU_") != -1 or str(list_data[i]).find("X_GENEXIS_EU_") != -1:
-                            param = str(list_data[i]).replace("X_IOWRT_EU_", "{BBF_VENDOR_PREFIX}")
+                            param = str(list_data[i]).replace("X_IOWRT_EU_", "{BBF_VENDOR_PREFIX}").replace("X_GENEXIS_EU_", "{BBF_VENDOR_PREFIX}")
                         else:
                             param = str(list_data[i])
 
@@ -173,6 +173,8 @@ def generate_bbf_xml_file(output_file, dm_json_files=None):
 
             ob_description = ET.SubElement(objec, "description")
             ob_description.text = desc.replace("<", "{").replace(">", "}") if desc is not None else ""
+            if desc is None:
+                print(f'#### Description should be added for {name} object ####')
 
             DM_OBJ_COUNT += 1
         else:
@@ -185,6 +187,8 @@ def generate_bbf_xml_file(output_file, dm_json_files=None):
 
             p_description = ET.SubElement(parameter, "description")
             p_description.text = desc.replace("<", "{").replace(">", "}") if desc is not None else ""
+            if desc is None:
+                print(f'#### Description should be added for {name} parameter ####')
 
             syntax = ET.SubElement(parameter, "syntax")
 

tools/tools_input.json

@@ -63,6 +63,7 @@
     {
       "repo": "https://dev.iopsys.eu/bbf/xmppc.git",
       "proto": "git",
+      "is_microservice": true,
       "dm_files": [
         "src/datamodel.c"
       ]
@@ -71,6 +72,7 @@
       "repo": "https://dev.iopsys.eu/bbf/stunc.git",
       "proto": "git",
       "version": "devel",
+      "is_microservice": true,
       "dm_files": [
         "src/datamodel.c"
       ]
@@ -145,6 +147,7 @@
         "-lcrypt"
       ],
       "dm_files": [
+        "src/main.c",
         "src/users.c"
       ]
     },
@@ -152,7 +155,11 @@
       "repo": "https://dev.iopsys.eu/network/parental-control.git",
       "proto": "git",
       "version": "devel",
+      "dm_info_file": "src/datamodel_description.json",
       "dm_files": [
+        "src/config.c",
+        "src/utils.c",
+        "src/dns_cache.c",
         "src/parentalcontrol.c"
       ]
     },
@@ -168,6 +175,8 @@
       "repo": "https://dev.iopsys.eu/iopsys/hostmngr.git",
       "proto": "git",
       "version": "devel",
+      "dm_info_file": "src/bbf_plugin/datamodel_description.json",
+      "is_microservice": true,
       "extra_dependencies": [
         "-DHAS_VENDOR_EXT",
         "-DCUSTOM_PREFIX=\\\"X_IOWRT_EU_\\\""
@@ -244,7 +253,9 @@
       "repo": "https://dev.iopsys.eu/hal/qosmngr.git",
       "proto": "git",
       "version": "devel",
+      "dm_info_file": "src/datamodel_description.json",
       "dm_files": [
+        "src/main.c",
         "src/qos_bbf.c"
       ]
     },
@@ -314,6 +325,7 @@
       "repo": "https://dev.iopsys.eu/network/bridgemngr.git",
       "proto": "git",
       "version": "devel",
+      "is_microservice": true,
       "dm_files": [
         "src/driver_vlan_backend/vendor.c",
         "src/driver_vlan_backend/common.c"
@@ -359,11 +371,25 @@
       "version": "devel",
       "dm_files": [
         "src/wifidmd.c",
-        "src/wifi.c",
+        "src/wifi.c"
+      ],
+      "extra_dependencies": [
+        "-DSERVICE_NAME=\\\"wifidmd\\\"",
+        "-lm"
+      ]
+    },
+    {
+      "repo": "https://dev.iopsys.eu/bbf/wifidmd.git",
+      "proto": "git",
+      "version": "devel",
+      "is_microservice": true,
+      "dm_files": [
+        "src/wifidmd.c",
         "src/dataelements.c"
       ],
       "extra_dependencies": [
         "-DSERVICE_NAME=\\\"wifidmd\\\"",
+        "-DWIFIDMD_DISABLE_LEGACY_WIFI",
         "-DWIFIDMD_ENABLE_WIFI_DATAELEMENTS",
         "-lm"
       ]
@@ -434,6 +460,7 @@
       "repo": "https://dev.iopsys.eu/voice/tr104.git",
       "proto": "git",
       "version": "devel",
+      "is_microservice": true,
       "dm_info_file": "libdm/extensions/iowrt/datamodel_description.json",
       "dm_files": [
         "libdm/tr104/*.c"
@@ -447,6 +474,7 @@
       "repo": "https://dev.iopsys.eu/voice/tr104.git",
       "proto": "git",
       "version": "devel",
+      "is_microservice": true,
       "dm_files": [
         "libdm/extensions/iowrt/*.c"
       ],
@@ -459,6 +487,7 @@
       "repo": "https://dev.iopsys.eu/system/logmngr.git",
       "proto": "git",
       "version": "devel",
+      "is_microservice": true,
       "dm_files": [
         "bbf_plugin/deviceinfologrotate.c",
         "bbf_plugin/common.c"
@@ -468,6 +497,7 @@
       "repo": "https://dev.iopsys.eu/system/logmngr.git",
       "proto": "git",
       "version": "devel",
+      "is_microservice": true,
       "dm_files": [
         "bbf_plugin/deviceinfovendorlog.c",
         "bbf_plugin/common.c"
@@ -486,6 +516,8 @@
       "repo": "https://dev.iopsys.eu/feed/gnx.git",
       "proto": "git",
       "version": "devel",
+      "is_microservice": true,
+      "dm_info_file": "package/gnx-files-ftth/bbf_plugin/datamodel_description.json",
       "vendor_prefix": "X_GENEXIS_EU_",
       "dm_files": [
         "package/gnx-files-ftth/bbf_plugin/link_forwarding.json"
@@ -504,6 +536,7 @@
       "repo": "https://dev.iopsys.eu/feed/gnx.git",
       "proto": "git",
       "version": "devel",
+      "dm_info_file": "genexis-packages/gnx-catv/datamodel/datamodel_description.json",
       "vendor_prefix": "X_GENEXIS_EU_",
       "dm_files": [
         "genexis-packages/gnx-catv/datamodel/datamodel.c"
@@ -513,6 +546,7 @@
       "repo": "https://dev.iopsys.eu/feed/gnx.git",
       "proto": "git",
       "version": "devel",
+      "is_microservice": true,
       "vendor_prefix": "X_GENEXIS_EU_",
       "dm_files": [
         "genexis-packages/easywifi/files/wifi_extn.json"
@@ -522,6 +556,7 @@
       "repo": "https://dev.iopsys.eu/feed/gnx.git",
       "proto": "git",
       "version": "devel",
+      "dm_info_file": "genexis-packages/dhcp-on-boarding/src/datamodel_description.json",
       "vendor_prefix": "X_GENEXIS_EU_",
       "dm_files": [
         "genexis-packages/dhcp-on-boarding/src/datamodel.c"
@@ -531,6 +566,7 @@
       "repo": "https://dev.iopsys.eu/feed/gnx.git",
       "proto": "git",
       "version": "devel",
+      "dm_info_file": "genexis-packages/gnx-loop-detector/datamodel/datamodel_description.json",
       "vendor_prefix": "X_GENEXIS_EU_",
       "dm_files": [
         "genexis-packages/gnx-loop-detector/datamodel/datamodel.c"
@@ -540,6 +576,7 @@
       "repo": "https://dev.iopsys.eu/feed/gnx.git",
       "proto": "git",
       "version": "devel",
+      "dm_info_file": "genexis-packages/gnx-sfp/src/bbf_plugin/datamodel_description.json",
       "vendor_prefix": "X_GENEXIS_EU_",
       "dm_files": [
         "genexis-packages/gnx-sfp/src/bbf_plugin/datamodel.c"