diff --git a/bbfdmd/ubus/bbfdmd.c b/bbfdmd/ubus/bbfdmd.c index 898f62a7..de75fc99 100644 --- a/bbfdmd/ubus/bbfdmd.c +++ b/bbfdmd/ubus/bbfdmd.c @@ -1465,6 +1465,11 @@ static int bbfdm_load_deamon_config(bbfdm_config_t *config, const char *json_pat snprintf(config->cli_in_name, sizeof(config->cli_in_name), "%s", opt_val); } + opt_val = dmjson_get_value(json_obj, 3, "cli", "input", "plugin_dir"); + if (DM_STRLEN(opt_val)) { + snprintf(config->cli_in_plugin_dir, sizeof(config->cli_in_plugin_dir), "%s", opt_val); + } + opt_val = dmjson_get_value(json_obj, 3, "cli", "output", "type"); if (DM_STRLEN(opt_val)) { snprintf(config->cli_out_type, sizeof(config->cli_out_type), "%s", opt_val); @@ -1691,8 +1696,9 @@ int main(int argc, char **argv) } openlog(bbfdm_ctx.config.out_name, LOG_CONS | LOG_PID | LOG_NDELAY, LOG_LOCAL1); + if (cli_argc) { - err = bbfdm_cli_exec_command(&bbfdm_ctx.config , cli_argc, cli_argv); + err = bbfdm_cli_exec_command(&bbfdm_ctx.config, cli_argc, cli_argv); goto exit; } diff --git a/bbfdmd/ubus/bbfdmd.h b/bbfdmd/ubus/bbfdmd.h index 4cfc8b80..96afd30d 100644 --- a/bbfdmd/ubus/bbfdmd.h +++ b/bbfdmd/ubus/bbfdmd.h @@ -31,6 +31,7 @@ typedef struct bbfdm_config { char out_root_obj[32]; char cli_in_type[32]; char cli_in_name[128]; + char cli_in_plugin_dir[128]; char cli_out_type[32]; } bbfdm_config_t; diff --git a/bbfdmd/ubus/cli.c b/bbfdmd/ubus/cli.c index 86d31419..501632e0 100644 --- a/bbfdmd/ubus/cli.c +++ b/bbfdmd/ubus/cli.c @@ -31,6 +31,7 @@ typedef struct { unsigned int instance_mode; unsigned int proto; char in_name[128]; + char in_plugin_dir[128]; char in_type[32]; char out_type[32]; char *cmd; @@ -107,7 +108,6 @@ static struct blob_attr *get_results_array(struct blob_attr *msg) static void __ubus_callback(struct ubus_request *req, int type __attribute__((unused)), struct blob_attr *msg) { struct blob_attr *cur = NULL; - bool print_msg = false; int rem = 0; const struct blobmsg_policy p[7] = { { "path", BLOBMSG_TYPE_STRING }, @@ -163,56 +163,7 @@ static void __ubus_callback(struct ubus_request *req, int type __attribute__((un char *type = tb[2] ? blobmsg_get_string(tb[2]) : ""; int cmd = get_dm_type(type); - if (print_msg == false) { - printf("\nDumping %s Schema...\n\n", name); - print_msg = true; - } - - printf("%s\n", name); - - if (cmd == DMT_COMMAND) { - struct blob_attr *input = tb[4]; - struct blob_attr *output = tb[5]; - struct blob_attr *in_cur = NULL, *out_cur = NULL; - int in_rem = 0, out_rem = 0; - - if (input) { - blobmsg_for_each_attr(in_cur, input, in_rem) { - struct blob_attr *in_tb[7] = {0}; - - blobmsg_parse(p, 7, in_tb, blobmsg_data(in_cur), blobmsg_len(in_cur)); - - char *arg = in_tb[0] ? blobmsg_get_string(in_tb[0]) : ""; - printf("%s input:%s\n", name, arg); - } - } - - if (output) { - blobmsg_for_each_attr(out_cur, output, out_rem) { - struct blob_attr *out_tb[7] = {0}; - - blobmsg_parse(p, 7, out_tb, blobmsg_data(out_cur), blobmsg_len(out_cur)); - - char *arg = out_tb[0] ? blobmsg_get_string(out_tb[0]) : ""; - printf("%s output:%s\n", name, arg); - } - } - } else if (cmd == DMT_EVENT) { - struct blob_attr *input = tb[4]; - struct blob_attr *in_cur = NULL; - int in_rem = 0; - - if (input) { - blobmsg_for_each_attr(in_cur, input, in_rem) { - struct blob_attr *in_tb[7] = {0}; - - blobmsg_parse(p, 7, in_tb, blobmsg_data(in_cur), blobmsg_len(in_cur)); - - char *arg = in_tb[0] ? blobmsg_get_string(in_tb[0]) : ""; - printf("%s event_arg:%s\n", name, arg); - } - } - } + printf("%s %s %s\n", name, type, (cmd != DMT_EVENT && cmd != DMT_COMMAND) ? data : "0"); } cli_data->ubus_status = true; @@ -278,6 +229,11 @@ static int bbfdm_load_cli_config(bbfdm_config_t *bbf_config, cli_data_t *cli_dat return -1; } + opt_val = bbf_config->cli_in_plugin_dir; + if (opt_val && strlen(opt_val)) { + snprintf(cli_data->in_plugin_dir, sizeof(cli_data->in_plugin_dir), "%s", opt_val); + } + opt_val = bbf_config->cli_out_type; if (opt_val && strlen(opt_val)) { snprintf(cli_data->out_type, sizeof(cli_data->out_type), "%s", opt_val); @@ -514,46 +470,9 @@ static int in_dotso_out_cli_exec_schema(cli_data_t *cli_data, char *argv[]) if (!err) { struct dm_parameter *n; - printf("\nDumping %s Schema...\n\n", cli_data->bbf_ctx.in_param); - list_for_each_entry(n, &cli_data->bbf_ctx.list_parameter, list) { int cmd = get_dm_type(n->type); - - printf("%s\n", n->name); - - if (cmd == DMT_COMMAND) { - if (n->data) { - const char **in, **out; - operation_args *args; - int i; - - args = (operation_args *) n->data; - in = args->in; - if (in) { - for (i = 0; in[i] != NULL; i++) - printf("%s input:%s\n", n->name, in[i]); - } - - out = args->out; - if (out) { - for (i = 0; out[i] != NULL; i++) - printf("%s output:%s\n", n->name, out[i]); - } - } - } else if (cmd == DMT_EVENT) { - if (n->data) { - event_args *ev; - - ev = (event_args *)n->data; - - if (ev->param) { - const char **in = ev->param; - - for (int i = 0; in[i] != NULL; i++) - printf("%s event_arg:%s\n", n->name, in[i]); - } - } - } + printf("%s %s %s\n", n->name, n->type, (cmd != DMT_EVENT && cmd != DMT_COMMAND) ? n->data : "0"); } } else { printf("ERROR: %d retrieving %s\n", err, cli_data->bbf_ctx.in_param); @@ -615,7 +534,7 @@ static int cli_exec_command(cli_data_t *cli_data, int argc, char *argv[]) goto end; } - bbf_global_init(CLI_DM_ROOT_OBJ, CLI_DM_VENDOR_EXTENSION, CLI_DM_VENDOR_EXTENSION_EXCLUDE, NULL); + bbf_global_init(CLI_DM_ROOT_OBJ, CLI_DM_VENDOR_EXTENSION, CLI_DM_VENDOR_EXTENSION_EXCLUDE, cli_data->in_plugin_dir); bbf_ctx_init(&cli_data->bbf_ctx, CLI_DM_ROOT_OBJ, CLI_DM_VENDOR_EXTENSION, CLI_DM_VENDOR_EXTENSION_EXCLUDE); @@ -679,6 +598,7 @@ int bbfdm_cli_exec_command(bbfdm_config_t *bbf_config, int argc, char *argv[]) } memset(&cli_data, 0, sizeof(cli_data_t)); + err = bbfdm_load_cli_config(bbf_config, &cli_data); if (err) { printf("ERROR: required cli config missing\n"); diff --git a/gitlab-ci/generate_supported_dm.sh b/gitlab-ci/generate_supported_dm.sh index d70d718c..81cada09 100755 --- a/gitlab-ci/generate_supported_dm.sh +++ b/gitlab-ci/generate_supported_dm.sh @@ -4,32 +4,10 @@ echo "Generate xml and xls artifacts" source ./gitlab-ci/shared.sh -mkdir -p /etc/supervisor/conf.d/ -cp ./gitlab-ci/core_service.conf /etc/supervisor/conf.d/ - -is_supervisor_running=0 -pp="$(pidof python3)" -[ -n "${pp}" ] && { - if ps -p ${pp}|grep -wq supervisord; then - is_supervisor_running=1 - fi -} - -if [ "${is_supervisor_running}" -eq "1" ] ; then - # starting base services - supervisorctl reread - supervisorctl update -else - /usr/bin/supervisord -c /etc/supervisor/supervisord.conf -fi - -sleep 5 -supervisorctl status all - # install required packages exec_cmd apt update exec_cmd apt install -y python3-pip libxml2-utils -exec_cmd pip3 install jsonschema xlwt ubus pylint +exec_cmd pip3 install xlwt if [ -n "${CI_SERVER_HOST}" ]; then echo "machine ${CI_SERVER_HOST}" >>~/.netrc @@ -42,13 +20,6 @@ repo_dir="/etc/bbfdm/plugins" [ ! -d "${repo_dir}" ] && mkdir -p "${repo_dir}" rm -f ${repo_dir}/* -# Make sure that all micro-services are removed -rm -rf /etc/app* - -if pidof bbfdmd >/dev/null; then - kill -9 $(pidof bbfdmd) -fi - if [ -z "${1}" ]; then ./tools/generate_dm.py tools/tools_input.json else diff --git a/gitlab-ci/install-dependencies.sh b/gitlab-ci/install-dependencies.sh index a5ebd127..3d4376f1 100755 --- a/gitlab-ci/install-dependencies.sh +++ b/gitlab-ci/install-dependencies.sh @@ -6,7 +6,7 @@ source ./gitlab-ci/shared.sh # install required packages exec_cmd apt update -exec_cmd apt install -y python3-pip iproute2 libmxml-dev uuid-dev zip +exec_cmd apt install -y python3-pip iproute2 exec_cmd pip3 install pexpect ubus # compile and install libbbf diff --git a/gitlab-ci/shared.sh b/gitlab-ci/shared.sh index 2da515a9..87514cf1 100755 --- a/gitlab-ci/shared.sh +++ b/gitlab-ci/shared.sh @@ -152,12 +152,12 @@ function install_libcwmpdm() fi echo "Compiling libcwmpdm" - cd /opt/dev/icwmp - cmake -DWITH_OPENSSL=ON -DCMAKE_INSTALL_PREFIX=/ + cd /opt/dev/icwmp/bbf_plugin + cmake -DCMAKE_INSTALL_PREFIX=/ exec_cmd_verbose make echo "installing libcwmpdm" - install_plugin /opt/dev/icwmp/libcwmpdm.so + install_plugin /opt/dev/icwmp/bbf_plugin/libcwmpdm.so cd /builds/bbf/bbfdm } diff --git a/gitlab-ci/tools-test.sh b/gitlab-ci/tools-test.sh index e068898e..eb55954c 100755 --- a/gitlab-ci/tools-test.sh +++ b/gitlab-ci/tools-test.sh @@ -4,17 +4,10 @@ echo "Verification of BBF Tools" pwd source ./gitlab-ci/shared.sh -cp ./gitlab-ci/core_service.conf /etc/supervisor/conf.d/ - -# starting base services -supervisorctl reread -supervisorctl update -sleep 5 - # install required packages exec_cmd apt update -exec_cmd apt install -y python3-pip libxml2-utils -exec_cmd pip3 install jsonschema xlwt ubus pylint +exec_cmd apt install -y python3-pip +exec_cmd pip3 install jsonschema xlwt pylint echo "Validating PEP8 syntax on tools" exec_cmd_verbose pylint -d R,C,W0603 tools/*.py diff --git a/libbbfdm-api/dmplugin.c b/libbbfdm-api/dmplugin.c index cf5dbad6..5ac00bfb 100644 --- a/libbbfdm-api/dmplugin.c +++ b/libbbfdm-api/dmplugin.c @@ -28,31 +28,14 @@ struct service char *object; }; -static void disable_srv_obj(DMOBJ *entryobj, char *srv_parent_dm, char *srv_obj) -{ - DMOBJ *dm_entryobj = NULL; - char obj_path[1024]; - - if (!entryobj || !srv_parent_dm || !srv_obj) - return; - - snprintf(obj_path, sizeof(obj_path), "%s%s.", srv_parent_dm, srv_obj); - - bool obj_exists = find_entry_obj(entryobj, obj_path, &dm_entryobj); - if (obj_exists == true && dm_entryobj) - dm_entryobj->bbfdm_type = BBFDM_NONE; -} - static bool add_service_to_main_tree(DMOBJ *main_dm, char *srv_name, char *srv_parent_dm, char *srv_obj) { - DMOBJ *dm_entryobj = NULL; - - bool obj_exists = find_entry_obj(main_dm, srv_parent_dm, &dm_entryobj); - if (obj_exists == false || !dm_entryobj) + DMOBJ *dm_entryobj = find_entry_obj(main_dm, srv_parent_dm); + if (!dm_entryobj) return false; // Disable service object if it already exists in the main tree - disable_srv_obj(main_dm, srv_parent_dm, srv_obj); + disable_entry_obj(dm_entryobj, srv_obj); if (dm_entryobj->nextdynamicobj == NULL) { dm_entryobj->nextdynamicobj = calloc(__INDX_DYNAMIC_MAX, sizeof(struct dm_dynamic_obj)); @@ -249,6 +232,15 @@ static void free_all_dynamic_nodes(DMOBJ *entryobj) if (entryobj->nextdynamicobj) { for (int i = 0; i < __INDX_DYNAMIC_MAX; i++) { struct dm_dynamic_obj *next_dyn_array = entryobj->nextdynamicobj + i; + + if (next_dyn_array->nextobj) { + for (int j = 0; next_dyn_array->nextobj[j]; j++) { + DMOBJ *jentryobj = next_dyn_array->nextobj[j]; + if (jentryobj) + free_all_dynamic_nodes(jentryobj); + } + } + FREE(next_dyn_array->nextobj); } FREE(entryobj->nextdynamicobj); @@ -283,8 +275,8 @@ static int plugin_obj_match(char *in_param, struct dmnode *node) return FAULT_9005; } -static void dm_check_dynamic_obj(DMNODE *parent_node, DMOBJ *entryobj, char *full_obj, DMOBJ **root_entry, bool *obj_found); -static void dm_check_dynamic_obj_entry(DMNODE *parent_node, DMOBJ *entryobj, char *parent_obj, char *full_obj, DMOBJ **root_entry, bool *obj_found) +static void dm_check_dynamic_obj(DMNODE *parent_node, DMOBJ *entryobj, char *full_obj, DMOBJ **root_entry); +static void dm_check_dynamic_obj_entry(DMNODE *parent_node, DMOBJ *entryobj, char *parent_obj, char *full_obj, DMOBJ **root_entry) { DMNODE node = {0}; node.obj = entryobj; @@ -295,7 +287,6 @@ static void dm_check_dynamic_obj_entry(DMNODE *parent_node, DMOBJ *entryobj, cha dmasprintf(&(node.current_object), "%s%s.", parent_obj, entryobj->obj); if (DM_STRCMP(node.current_object, full_obj) == 0) { *root_entry = entryobj; - *obj_found = true; return; } @@ -304,16 +295,16 @@ static void dm_check_dynamic_obj_entry(DMNODE *parent_node, DMOBJ *entryobj, cha return; if (entryobj->nextobj || entryobj->nextdynamicobj) - dm_check_dynamic_obj(&node, entryobj->nextobj, full_obj, root_entry, obj_found); + dm_check_dynamic_obj(&node, entryobj->nextobj, full_obj, root_entry); } -static void dm_check_dynamic_obj(DMNODE *parent_node, DMOBJ *entryobj, char *full_obj, DMOBJ **root_entry, bool *obj_found) +static void dm_check_dynamic_obj(DMNODE *parent_node, DMOBJ *entryobj, char *full_obj, DMOBJ **root_entry) { char *parent_obj = parent_node->current_object; for (; (entryobj && entryobj->obj); entryobj++) { - dm_check_dynamic_obj_entry(parent_node, entryobj, parent_obj, full_obj, root_entry, obj_found); - if (*obj_found == true) + dm_check_dynamic_obj_entry(parent_node, entryobj, parent_obj, full_obj, root_entry); + if (*root_entry != NULL) return; } @@ -325,8 +316,8 @@ static void dm_check_dynamic_obj(DMNODE *parent_node, DMOBJ *entryobj, char *ful for (int j = 0; next_dyn_array->nextobj[j]; j++) { DMOBJ *jentryobj = next_dyn_array->nextobj[j]; for (; (jentryobj && jentryobj->obj); jentryobj++) { - dm_check_dynamic_obj_entry(parent_node, jentryobj, parent_obj, full_obj, root_entry, obj_found); - if (*obj_found == true) + dm_check_dynamic_obj_entry(parent_node, jentryobj, parent_obj, full_obj, root_entry); + if (*root_entry != NULL) return; } } @@ -336,19 +327,35 @@ static void dm_check_dynamic_obj(DMNODE *parent_node, DMOBJ *entryobj, char *ful } } -bool find_entry_obj(DMOBJ *root_entry, char *in_param, DMOBJ **entryobj) +DMOBJ *find_entry_obj(DMOBJ *entryobj, char *obj_path) { - if (!root_entry || !in_param || !entryobj) + if (!entryobj || !obj_path) return false; DMNODE node = {.current_object = ""}; - bool obj_found = false; + DMOBJ *obj = NULL; - char *obj_path = replace_str(in_param, ".{i}.", "."); - dm_check_dynamic_obj(&node, root_entry, obj_path, entryobj, &obj_found); - FREE(obj_path); + char *in_obj = replace_str(obj_path, ".{i}.", "."); + dm_check_dynamic_obj(&node, entryobj, in_obj, &obj); + FREE(in_obj); - return (obj_found && *entryobj) ? true : false; + return obj; +} + +void disable_entry_obj(DMOBJ *entryobj, char *obj_path) +{ + if (!entryobj || DM_STRLEN(obj_path) == 0) + return; + + DMOBJ *nextobj = entryobj->nextobj; + + for (; (nextobj && nextobj->obj); nextobj++) { + + if (DM_STRCMP(nextobj->obj, obj_path) == 0) { + nextobj->bbfdm_type = BBFDM_NONE; + return; + } + } } void dm_exclude_obj(DMOBJ *entryobj, DMNODE *parent_node, char *obj_path) diff --git a/libbbfdm-api/dmplugin.h b/libbbfdm-api/dmplugin.h index a8e1943b..c7262a3c 100644 --- a/libbbfdm-api/dmplugin.h +++ b/libbbfdm-api/dmplugin.h @@ -11,7 +11,8 @@ #ifndef __DMPLUGIN_H__ #define __DMPLUGIN_H__ -bool find_entry_obj(DMOBJ *root_entry, char *in_param, DMOBJ **entryobj); +DMOBJ *find_entry_obj(DMOBJ *entryobj, char *obj_path); +void disable_entry_obj(DMOBJ *entryobj, char *obj_path); void dm_exclude_obj(DMOBJ *entryobj, DMNODE *parent_node, char *obj_path); diff --git a/libbbfdm-api/plugin/dotso_plugin.c b/libbbfdm-api/plugin/dotso_plugin.c index 6d658eb5..293de124 100644 --- a/libbbfdm-api/plugin/dotso_plugin.c +++ b/libbbfdm-api/plugin/dotso_plugin.c @@ -44,7 +44,11 @@ static void free_all_list_open_library(struct list_head *library_list) int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path) { +#ifndef BBF_SCHEMA_FULL_TREE void *handle = dlopen(plugin_path, RTLD_NOW|RTLD_LOCAL); +#else + void *handle = dlopen(plugin_path, RTLD_LAZY); +#endif if (!handle) { TRACE("Plugin failed [%s]\n", dlerror()); return 0; @@ -62,13 +66,15 @@ int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path) for (int i = 0; dynamic_obj[i].path; i++) { - DMOBJ *dm_entryobj = NULL; - bool obj_exists = find_entry_obj(entryobj, dynamic_obj[i].path, &dm_entryobj); - if (obj_exists == false || !dm_entryobj) + DMOBJ *dm_entryobj = find_entry_obj(entryobj, dynamic_obj[i].path); + if (!dm_entryobj) continue; if (dynamic_obj[i].root_obj) { + // Disable object if it already exists in the main tree + disable_entry_obj(dm_entryobj, dynamic_obj[i].root_obj->obj); + if (dm_entryobj->nextdynamicobj == NULL) { dm_entryobj->nextdynamicobj = calloc(__INDX_DYNAMIC_MAX, sizeof(struct dm_dynamic_obj)); dm_entryobj->nextdynamicobj[INDX_JSON_MOUNT].idx_type = INDX_JSON_MOUNT; diff --git a/libbbfdm-api/plugin/json_plugin.c b/libbbfdm-api/plugin/json_plugin.c index e59b00b0..1d53c99c 100644 --- a/libbbfdm-api/plugin/json_plugin.c +++ b/libbbfdm-api/plugin/json_plugin.c @@ -1879,18 +1879,22 @@ int load_json_plugins(DMOBJ *entryobj, const char *plugin_path) continue; } - DMOBJ *dm_entryobj = NULL; char obj_prefix[MAX_DM_LENGTH] = {0}; + char obj_name[64] = {0}; char *obj_path = replace_str(key, "{BBF_VENDOR_PREFIX}", BBF_VENDOR_PREFIX); find_prefix_obj(obj_path, obj_prefix, MAX_DM_LENGTH); + find_current_obj(obj_path, obj_name, sizeof(obj_name)); - bool obj_exists = find_entry_obj(entryobj, obj_prefix, &dm_entryobj); - if (obj_exists == 0 || !dm_entryobj) { + DMOBJ *dm_entryobj = find_entry_obj(entryobj, obj_prefix); + if (!dm_entryobj) { FREE(obj_path); continue; } + // Disable object if it already exists in the main tree + disable_entry_obj(dm_entryobj, obj_name); + if (dm_entryobj->nextdynamicobj == NULL) { dm_entryobj->nextdynamicobj = calloc(__INDX_DYNAMIC_MAX, sizeof(struct dm_dynamic_obj)); dm_entryobj->nextdynamicobj[INDX_JSON_MOUNT].idx_type = INDX_JSON_MOUNT; diff --git a/libbbfdm-api/plugin/vendor_plugin.c b/libbbfdm-api/plugin/vendor_plugin.c index 9e102526..8a7c9d26 100644 --- a/libbbfdm-api/plugin/vendor_plugin.c +++ b/libbbfdm-api/plugin/vendor_plugin.c @@ -89,9 +89,8 @@ static void load_vendor_extension_arrays(DMOBJ *entryobj, DM_MAP_VENDOR *vendor_ for (int i = 0; vendor_obj[i].path; i++) { - DMOBJ *dm_entryobj = NULL; - bool obj_exists = find_entry_obj(entryobj, vendor_obj[i].path, &dm_entryobj); - if (obj_exists == false || !dm_entryobj) + DMOBJ *dm_entryobj = find_entry_obj(entryobj, vendor_obj[i].path); + if (!dm_entryobj) continue; if (vendor_obj[i].root_obj) { @@ -157,12 +156,11 @@ static void load_vendor_extension_overwrite_arrays(DMOBJ *entryobj, DM_MAP_VENDO continue; DM_MAP_OBJ *dynamic_overwrite_obj = vendor_map_obj[j].vendor_obj; - DMOBJ *dm_entryobj = NULL; for (int i = 0; dynamic_overwrite_obj[i].path; i++) { - bool obj_exists = find_entry_obj(entryobj, dynamic_overwrite_obj[i].path, &dm_entryobj); - if (obj_exists == false || !dm_entryobj) + DMOBJ *dm_entryobj = find_entry_obj(entryobj, dynamic_overwrite_obj[i].path); + if (!dm_entryobj) continue; if (dynamic_overwrite_obj[i].root_obj) { @@ -198,7 +196,6 @@ static void exclude_obj(DMOBJ *dm_entryobj, char *in_obj) static void exclude_param(DMOBJ *dm_entryobj, char *in_param) { - DMOBJ *entryobj = NULL; char obj_prefix[256] = {'\0'}; if (in_param == NULL) @@ -208,9 +205,9 @@ static void exclude_param(DMOBJ *dm_entryobj, char *in_param) if (ret) DM_STRNCPY(obj_prefix, in_param, ret - in_param + 2); - bool obj_exists = find_entry_obj(dm_entryobj, obj_prefix, &entryobj); + DMOBJ *entryobj = find_entry_obj(dm_entryobj, obj_prefix); - if (entryobj && obj_exists == true) { + if (entryobj) { DMLEAF *leaf = entryobj->leaf; for (; (leaf && leaf->parameter); leaf++) { diff --git a/tools/README.md b/tools/README.md index 4085ea71..65b37003 100644 --- a/tools/README.md +++ b/tools/README.md @@ -147,11 +147,11 @@ The parameters/keys used in tools_input.json file are mostly self-explanatory bu | dm_json_files | This should contain the list of json file path, where each file contains the definition of DM objects/parameters | | vendor_prefix | The prefix used by vendor for vendor extension in DM objects/parameters, e.g., "X_IOPSYS_EU_" | | plugins | A list of plugins with associated repositories and data model files | -| | repo: The URL of the plugin repository | +| | repo: The path of the plugin repository. Could be 'URL' or 'folder_path' | +| | proto: The protocol of the plugin repository. Could be 'git' or 'local' | | | version: (optional): The version of the git plugin | | | dm_files: A list of data model files associated with the plugin | | | extra_dependencies: (optional): Extra dependencies for the plugin, if any | -| | micro-service: (optional): Information about the micro-service, including its name, parent data model, object, and root object | | output.acs | Currently the tool support two variants of xml definitions of DM objects/parameters | | | hdm: This variant of xml is compatible with Nokia HDM ACS | | | default: This contains the generic definition which has the capability to define more descriptive DM objects/parameters | @@ -186,6 +186,7 @@ The input json file should be defined as follow: "plugins": [ { "repo": "https://dev.iopsys.eu/bbf/mydatamodel.git", + "proto": "git", "version": "tag/hash/branch", "dm_files": [ "src/datamodel.c", @@ -194,6 +195,7 @@ The input json file should be defined as follow: }, { "repo": "https://dev.iopsys.eu/bbf/mybbfplugin.git", + "proto": "git", "version": "tag/hash/branch", "dm_files": [ "dm.c" @@ -201,19 +203,15 @@ The input json file should be defined as follow: }, { "repo": "https://dev.iopsys.eu/bbf/mydatamodeljson.git", + "proto": "git", "version": "tag/hash/branch", "dm_files": [ "src/plugin/datamodel.json" - ], - "micro-service": { - "name": "bbfdm.wifi", - "parent_dm": "Device.WiFi.", - "object": "DataElements", - "root_obj": "bbfdm" - } + ] }, { "repo": "/home/iopsys/sdk/mypackage/", + "proto": "local", "dm_files": [ "src/datamodel.c", "additional_datamodel.c" @@ -221,6 +219,7 @@ The input json file should be defined as follow: }, { "repo": "/src/feeds/mypackage/", + "proto": "local", "dm_files": [ "datamodel.c", "src/datamodel.json" @@ -245,8 +244,6 @@ The input json file should be defined as follow: --- **NOTE** - 1. All defined plugins will be treated as plugins except the ones that have the micro-service option defined will be treated as micro-services. - 2. The `micro-service` option should be defined in the plugin only if you want to overwrite a specific Object introduced in the main tree. - 3. All the tools need to be executed from the top directory. +> All the tools need to be executed from the top directory. --- diff --git a/tools/bbf_common.py b/tools/bbf_common.py index 6bfc6688..87206d9a 100755 --- a/tools/bbf_common.py +++ b/tools/bbf_common.py @@ -8,8 +8,6 @@ import os import subprocess import shutil import json -import ubus -import time import glob # Constants @@ -226,60 +224,67 @@ def build_and_install_bbfdm(vendor_prefix, vendor_list): print('Compiling and installing bbfdmd done') -def run_command(command): - try: - # Use subprocess.Popen to start the daemon process - subprocess.Popen(command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - # The daemon process will continue running in the background - time.sleep(1) - - except subprocess.CalledProcessError as e: - # Handle subprocess errors here - print(f"Error running the daemon process: {e}") - sys.exit(1) - - -def create_input_json_file(file_path, input_type, input_name, micro_service_config): +def create_bbfdm_input_json_file(proto): data = { "daemon": { + }, + "cli": { + "config": { + "proto": proto, + "instance_mode": 0 + }, "input": { - "type": input_type, - "name": input_name + "type": "DotSo", + "name": "/lib/libbbfdm.so", + "plugin_dir": "/etc/bbfdm/plugins" }, "output": { - "type": "UBUS", - "name": micro_service_config.get("name"), - "parent_dm": micro_service_config.get("parent_dm"), - "object": micro_service_config.get("object"), - "root_obj": micro_service_config.get("root_obj") + "type": "CLI" } } } + + file_path = '/tmp/bbfdm/input.json' + + # Ensure the directory exists + os.makedirs(os.path.dirname(file_path), exist_ok=True) + # Open the file in 'w' mode (create or truncate) with open(file_path, 'w', encoding='utf-8') as json_file: json.dump(data, json_file, indent=4) -def transform_schema_to_dm(schema_out, dm_list): - if not schema_out or not isinstance(schema_out, list) or len(schema_out) == 0: - return +def fill_list_dm(proto, dm_list): + create_bbfdm_input_json_file(proto) + + command = "bbfdmd -c schema Device." + try: + # Run the command + result = subprocess.run(command, shell=True, text=True, capture_output=True, check=True) + + # Get the output from the result + output = result.stdout - result_list = schema_out[0].get("results", []) + # Split the output into lines + lines = output.strip().split('\n') + + # Iterate through each line and parse the information + for line in lines: + parts = line.split() + path, n_type, data = parts[0], parts[1], parts[2] + permission = "readWrite" if data == "1" else "readOnly" + p_type = n_type[4:] + entry = { + "param": path, + "permission": permission, + "type": p_type, + } + dm_list.append(entry) - for result in result_list: - path = result.get("path", "") - data = result.get("data", "0") - permission = "readOnly" if data == "0" else "readWrite" - p_type = result.get("type", "xsd:string")[4:] - - entry = { - "param": path, - "permission": permission, - "type": p_type, - } - - dm_list.append(entry) + except subprocess.CalledProcessError as e: + # Handle subprocess errors here + print(f"Error running command: {e}") + sys.exit(1) def remove_duplicate_elements(input_list): @@ -296,42 +301,15 @@ def remove_duplicate_elements(input_list): def fill_list_supported_dm(): - # Wait for 5 seconds to be sure that all micro-services started successfully - time.sleep(5) - # pylint: disable=E1101 - ubus.connect() - - usp_schema_out = ubus.call('bbfdm', 'schema', {"path": "Device.", "optional": {"proto": "usp"}}) - transform_schema_to_dm(usp_schema_out, LIST_SUPPORTED_USP_DM) + fill_list_dm("usp", LIST_SUPPORTED_USP_DM) LIST_SUPPORTED_USP_DM.sort(key=lambda x: x['param'], reverse=False) LIST_SUPPORTED_USP_DM[:] = remove_duplicate_elements(LIST_SUPPORTED_USP_DM) - cwmp_schema_out = ubus.call('bbfdm', 'schema', {"path": "Device.", "optional": {"proto": "cwmp"}}) - transform_schema_to_dm(cwmp_schema_out, LIST_SUPPORTED_CWMP_DM) + fill_list_dm("cwmp", LIST_SUPPORTED_CWMP_DM) LIST_SUPPORTED_CWMP_DM.sort(key=lambda x: x['param'], reverse=False) LIST_SUPPORTED_CWMP_DM[:] = remove_duplicate_elements(LIST_SUPPORTED_CWMP_DM) - ubus.disconnect() - # pylint: enable=E1101 - - -def get_micro_service_config(micro_service): - parent_dm = get_option_value(micro_service, "parent_dm") - root_obj = get_option_value(micro_service, "root_obj") - obj = get_option_value(micro_service, "object") - name = get_option_value(micro_service, "name") - - if not isinstance(micro_service, dict) or None in (parent_dm, root_obj, obj, name): - return None - - return { - "parent_dm": parent_dm, - "root_obj": root_obj, - "object": obj, - "name": name - } - def clone_git_repository(repo, version=None): try: @@ -351,32 +329,6 @@ def get_repo_version_info(repo, version=None): return f'{repo}^{version}' -def process_json_file(filename, idx, micro_service, micro_service_config): - if micro_service is None: - move_file(filename, "/etc/bbfdm/plugins") - else: - micro_srv_path = f"/etc/app{idx}" - micro_srv_lib_path = f"{micro_srv_path}/file{idx}.json" - micro_srv_input_path = f"{micro_srv_path}/input.json" - create_folder(micro_srv_path) - move_file(filename, micro_srv_lib_path) - create_input_json_file(micro_srv_input_path, "JSON", micro_srv_lib_path, micro_service_config) - run_command(f"/usr/sbin/bbfdmd -m {micro_srv_input_path}") - - -def process_c_files(LIST_FILES, vendor_prefix, extra_dependencies, idx, micro_service, micro_service_config): - if micro_service is None: - generate_shared_library(f"/etc/bbfdm/plugins/lib{idx}.so", LIST_FILES, vendor_prefix, extra_dependencies) - else: - micro_srv_path = f"/etc/app{idx}" - micro_srv_lib_path = f"{micro_srv_path}/lib{idx}.so" - micro_srv_input_path = f"{micro_srv_path}/input.json" - create_folder(micro_srv_path) - generate_shared_library(micro_srv_lib_path, LIST_FILES, vendor_prefix, extra_dependencies) - create_input_json_file(micro_srv_input_path, "DotSo", micro_srv_lib_path, micro_service_config) - run_command(f"/usr/sbin/bbfdmd -m {micro_srv_input_path}") - - def download_and_build_plugins(plugins, vendor_prefix): global BBF_ERROR_CODE @@ -387,40 +339,37 @@ def download_and_build_plugins(plugins, vendor_prefix): print("Generating data models from defined plugins...") for plugin_index, plugin in enumerate(plugins): - micro_service_config = None repo = get_option_value(plugin, "repo") + proto = get_option_value(plugin, "proto") dm_files = get_option_value(plugin, "dm_files") extra_dependencies = get_option_value(plugin, "extra_dependencies", []) - if repo is None or dm_files is None or not isinstance(dm_files, list): + if repo is None or proto is None or dm_files is None or not isinstance(dm_files, list): print("Necessary input missing") BBF_ERROR_CODE += 1 continue - micro_service = get_option_value(plugin, "micro-service") - if micro_service is not None: - micro_service_config = get_micro_service_config(micro_service) - if micro_service_config is None: - print("Micro service config not defined") - BBF_ERROR_CODE += 1 - continue - print(f' - Processing plugin: {plugin}') - version = get_option_value(plugin, "version") + if proto == "git": + repo_path = ".repo/" + version = get_option_value(plugin, "version") - remove_folder(".repo") + remove_folder(".repo") - if not clone_git_repository(repo, version): - print(f"Failed to clone {repo}") - BBF_ERROR_CODE += 1 - continue - - print(f' Processing {get_repo_version_info(repo, version)}') + if not clone_git_repository(repo, version): + print(f"Failed to clone {repo}") + BBF_ERROR_CODE += 1 + continue + + print(f' Processing {get_repo_version_info(repo, version)}') + elif proto == "local": + repo_path = repo + print(f' Processing {get_repo_version_info(repo, proto)}') LIST_FILES = [] - os.chdir(".repo/") + os.chdir(repo_path) for dm_file in dm_files: filename = dm_file if filename.endswith('*.c'): @@ -430,19 +379,19 @@ def download_and_build_plugins(plugins, vendor_prefix): if filename.endswith('.c'): LIST_FILES.append(filename) elif filename.endswith('.json'): - process_json_file(filename, plugin_index, micro_service, micro_service_config) + move_file(filename, "/etc/bbfdm/plugins") else: print(f"Unknown file format {filename}") BBF_ERROR_CODE += 1 else: - print(f"File not accessible {filename}") + print(f"Error: File not accessible {filename} !!!!!!") BBF_ERROR_CODE += 1 if len(LIST_FILES) > 0: - process_c_files(LIST_FILES, vendor_prefix, extra_dependencies, plugin_index, micro_service, micro_service_config) + generate_shared_library(f"/etc/bbfdm/plugins/lib{plugin_index}.so", LIST_FILES, vendor_prefix, extra_dependencies) clear_list(LIST_FILES) - os.chdir("..") + cd_dir(CURRENT_PATH) remove_folder(".repo") @@ -465,12 +414,6 @@ def generate_supported_dm(vendor_prefix=None, vendor_list=None, plugins=None): # Download && Build Plugins Data Models download_and_build_plugins(plugins, vendor_prefix) - # Run bbfdm daemon - run_command("/usr/sbin/bbfdmd") - # Fill the list supported data model fill_list_supported_dm() - # kill all related bbfdm daemon - run_command("kill -9 $(pidof bbfdmd)") - diff --git a/tools/generate_dm.py b/tools/generate_dm.py index 89e2cfdb..0725b6b3 100755 --- a/tools/generate_dm.py +++ b/tools/generate_dm.py @@ -86,33 +86,36 @@ for option, value in json_data.items(): print_dm_usage() exit(1) -bbf.generate_supported_dm(VENDOR_PREFIX, VENDOR_LIST, PLUGINS) +if OUTPUT is None: + bbf.download_and_build_plugins(PLUGINS, VENDOR_PREFIX) +else: + bbf.generate_supported_dm(VENDOR_PREFIX, VENDOR_LIST, PLUGINS) + file_format = bbf.get_option_value(OUTPUT, "file_format", ['xml']) + output_file_prefix = bbf.get_option_value(OUTPUT, "output_file_prefix", "datamodel") + output_dir = bbf.get_option_value(OUTPUT, "output_dir", "./out") + + bbf.create_folder(output_dir) + + if isinstance(file_format, list): + for _format in file_format: + + if _format == "xml": + acs = bbf.get_option_value(OUTPUT, "acs", ['default']) + if isinstance(acs, list): + for acs_format in acs: + + output_file_name = output_dir + '/' + output_file_prefix + '_' + acs_format + '.xml' + if acs_format == "hdm": + bbf_xml.generate_xml('HDM', DM_JSON_FILES, output_file_name) + + if acs_format == "default": + bbf_xml.generate_xml('default', DM_JSON_FILES, output_file_name) + + if _format == "xls": + output_file_name = output_dir + '/' + output_file_prefix + '.xls' + bbf_excel.generate_excel(['tr181', 'tr104'], output_file_name) + + print("Datamodel generation completed, aritifacts shall be available in out directory or as per input json configuration") -file_format = bbf.get_option_value(OUTPUT, "file_format", ['xml']) -output_file_prefix = bbf.get_option_value(OUTPUT, "output_file_prefix", "datamodel") -output_dir = bbf.get_option_value(OUTPUT, "output_dir", "./out") - -bbf.create_folder(output_dir) - -if isinstance(file_format, list): - for _format in file_format: - - if _format == "xml": - acs = bbf.get_option_value(OUTPUT, "acs", ['default']) - if isinstance(acs, list): - for acs_format in acs: - - output_file_name = output_dir + '/' + output_file_prefix + '_' + acs_format + '.xml' - if acs_format == "hdm": - bbf_xml.generate_xml('HDM', DM_JSON_FILES, output_file_name) - - if acs_format == "default": - bbf_xml.generate_xml('default', DM_JSON_FILES, output_file_name) - - if _format == "xls": - output_file_name = output_dir + '/' + output_file_prefix + '.xls' - bbf_excel.generate_excel(['tr181', 'tr104'], output_file_name) - -print("Datamodel generation completed, aritifacts shall be available in out directory or as per input json configuration") sys.exit(bbf.BBF_ERROR_CODE) diff --git a/tools/generate_dm.sh b/tools/generate_dm.sh index 071a83e3..29ac4bd0 100755 --- a/tools/generate_dm.sh +++ b/tools/generate_dm.sh @@ -2,7 +2,7 @@ # Set variables CONTAINER_NAME="generate_dm_tools" -IMAGE_NAME="dev.iopsys.eu:5050/iopsys/gitlab-ci-pipeline/code-analysis:latest" +IMAGE_NAME="dev.iopsys.eu:5050/bbf/bbfdm/tools:latest" INPUT="" root="${PWD/tools}" diff --git a/tools/tools_input.json b/tools/tools_input.json index 2365100f..149f2d77 100644 --- a/tools/tools_input.json +++ b/tools/tools_input.json @@ -17,25 +17,29 @@ "plugins": [ { "repo": "https://dev.iopsys.eu/bbf/icwmp.git", + "proto": "git", "version": "devel", "dm_files": [ - "src/cwmp_dm/datamodel.c" + "bbf_plugin/datamodel.c" ] }, { "repo": "https://dev.iopsys.eu/bbf/bulkdata.git", + "proto": "git", "dm_files": [ "bbf_plugin/bulkdata.json" ] }, { "repo": "https://dev.iopsys.eu/bbf/xmppc.git", + "proto": "git", "dm_files": [ "src/datamodel.c" ] }, { "repo": "https://dev.iopsys.eu/bbf/stunc.git", + "proto": "git", "version": "devel", "dm_files": [ "src/datamodel.c" @@ -43,6 +47,7 @@ }, { "repo": "https://dev.iopsys.eu/bbf/udpecho.git", + "proto": "git", "version": "devel", "dm_files": [ "src/datamodel.c" @@ -50,6 +55,7 @@ }, { "repo": "https://dev.iopsys.eu/bbf/twamp-light.git", + "proto": "git", "version": "devel", "dm_files": [ "src/datamodel.c" @@ -57,6 +63,7 @@ }, { "repo": "https://dev.iopsys.eu/bbf/periodicstats.git", + "proto": "git", "version": "devel", "dm_files": [ "bbf_plugin/bbf_plugin.c" @@ -64,6 +71,7 @@ }, { "repo": "https://dev.iopsys.eu/feed/iopsys.git", + "proto": "git", "version": "devel", "dm_files": [ "urlfilter/files/etc/bbfdm/plugins/urlfilter.json", @@ -74,6 +82,7 @@ }, { "repo": "https://dev.iopsys.eu/lcm/swmodd.git", + "proto": "git", "version": "devel", "dm_files": [ "src/datamodel.c" @@ -81,6 +90,7 @@ }, { "repo": "https://dev.iopsys.eu/bbf/usermngr.git", + "proto": "git", "version": "devel", "extra_dependencies": [ "-lcrypt" @@ -91,6 +101,7 @@ }, { "repo": "https://dev.iopsys.eu/iopsys/hostmngr.git", + "proto": "git", "version": "devel", "dm_files": [ "src/bbf_plugin/hosts.c" @@ -98,19 +109,15 @@ }, { "repo": "https://dev.iopsys.eu/bbf/timemngr.git", + "proto": "git", "version": "devel", "dm_files": [ "src/times.c" - ], - "micro-service": { - "name": "bbfdm.time", - "parent_dm": "Device.", - "object": "Time", - "root_obj": "bbfdm" - } + ] }, { "repo": "https://dev.iopsys.eu/voice/tr104.git", + "proto": "git", "version": "devel", "dm_files": [ "libdm/tr104/*.c" @@ -122,6 +129,7 @@ }, { "repo": "https://dev.iopsys.eu/voice/tr104.git", + "proto": "git", "version": "devel", "dm_files": [ "libdm/extensions/iowrt/*.c"