T#8408: JSON Plugin: Fixed issues when multi-instance object mapping for multiple sections

This commit is contained in:
Amin Ben Ramdhane 2022-07-19 00:04:46 +01:00
parent 37ddd5f707
commit 95c7459bce
10 changed files with 1369 additions and 96 deletions

View file

@ -85,6 +85,7 @@ run_memory_test:
when: always
paths:
- timestamp.log
- output-report-device-get.txt
- build/libbbf-1.0.0-Linux.sh
doxygen:

View file

@ -417,9 +417,16 @@ static int browse_obj(struct dmctx *dmctx, DMNODE *parent_node, void *prev_data,
for (int i = 0; buf_alias[i]; i++)
buf_alias[i] = tolower(buf_alias[i]);
if(file && section_type && dmmap_file) {
if (file && section_type && dmmap_file) {
synchronize_specific_config_sections_with_dmmap(json_object_get_string(file), json_object_get_string(section_type), json_object_get_string(dmmap_file), &dup_list);
list_for_each_entry(p, &dup_list, list) {
char *dm_parent = NULL;
dmuci_get_value_by_section_string(p->config_section, "dm_parent", &dm_parent);
if (prev_data && DM_STRLEN(dm_parent)) {
if (strcmp(section_name((struct uci_section *)prev_data), dm_parent) != 0)
continue;
}
inst = handle_instance(dmctx, parent_node, p->dmmap_section, buf_instance, buf_alias);
@ -465,7 +472,7 @@ static int browse_obj(struct dmctx *dmctx, DMNODE *parent_node, void *prev_data,
free_ubus_arguments(u_args, u_args_size);
if (res && key) {
char arr_name[32] = {0};
char arr_name[64] = {0};
int id = 0, i = 0;
json_object *arr_obj = get_requested_json_obj(res, prev_instance, json_object_get_string(key), arr_name, sizeof(arr_name));
@ -512,7 +519,8 @@ static int add_obj(char *refparam, struct dmctx *ctx, void *data, char **instanc
struct json_object *section_type = NULL;
struct json_object *dmmap_file = NULL;
char *object = NULL;
char buf_instance[64];
char buf_instance[128];
char sec_name[128];
json_object_object_get_ex((mapping_0 && json_version == JSON_VERSION_1) ? mapping_0 : mapping_obj, "uci", &uci_obj);
json_object_object_get_ex(uci_obj, "file", &file);
@ -522,17 +530,24 @@ static int add_obj(char *refparam, struct dmctx *ctx, void *data, char **instanc
object = find_current_obj(refparam);
snprintf(buf_instance, sizeof(buf_instance), "%s_instance", object);
for (int i = 0; buf_instance[i]; i++) {
snprintf(sec_name, sizeof(sec_name), "%s%s_%s", data ? section_name((struct uci_section *)data) : "", object, *instance);
for (int i = 0; buf_instance[i]; i++)
buf_instance[i] = tolower(buf_instance[i]);
}
for (int i = 0; sec_name[i]; i++)
sec_name[i] = tolower(sec_name[i]);
if (file && section_type && dmmap_file) {
struct uci_section *s = NULL, *dmmap_s = NULL;
dmuci_add_section(json_object_get_string(file), json_object_get_string(section_type), &s);
dmuci_rename_section_by_section(s, sec_name);
if (data) dmuci_set_value_by_section(s, "dm_parent", section_name((struct uci_section *)data));
dmuci_add_section_bbfdm(json_object_get_string(dmmap_file), json_object_get_string(section_type), &dmmap_s);
dmuci_set_value_by_section(dmmap_s, "section_name", section_name(s));
if (data) dmuci_set_value_by_section(dmmap_s, "dm_parent", section_name((struct uci_section *)data));
dmuci_set_value_by_section(dmmap_s, buf_instance, *instance);
}
}
@ -589,6 +604,17 @@ static int delete_obj(char *refparam, struct dmctx *ctx, void *data, char *insta
break;
case DEL_ALL:
uci_foreach_sections_safe(json_object_get_string(file), json_object_get_string(section_type), stmp, s) {
if (data) {
char *dm_parent = NULL;
dmuci_get_value_by_section_string(s, "dm_parent", &dm_parent);
if (data && DM_STRLEN(dm_parent)) {
if (strcmp(section_name((struct uci_section *)data), dm_parent) != 0)
continue;
}
}
get_dmmap_section_of_config_section(json_object_get_string(dmmap_file), json_object_get_string(section_type), section_name(s), &dmmap_section);
dmuci_delete_by_section(dmmap_section, NULL, NULL);
@ -613,7 +639,6 @@ static char *uci_get_value(json_object *mapping_obj, int json_version, char *ref
struct json_object *option_name = NULL;
struct json_object *list = NULL;
struct json_object *list_name = NULL;
struct json_object *path = NULL;
struct json_object *linker_jobj = NULL;
char *linker = NULL;
char *value = "";
@ -627,7 +652,6 @@ static char *uci_get_value(json_object *mapping_obj, int json_version, char *ref
json_object_object_get_ex(option, "name", &option_name);
json_object_object_get_ex(obj, "list", &list);
json_object_object_get_ex(list, "name", &list_name);
json_object_object_get_ex(obj, "path", &path);
json_object_object_get_ex(mapping_obj, "linker_obj", &linker_jobj);
char *opt_temp = NULL;
@ -651,6 +675,14 @@ static char *uci_get_value(json_object *mapping_obj, int json_version, char *ref
int cnt = 0;
uci_foreach_sections(json_object_get_string(file), json_object_get_string(type), s) {
char *dm_parent = NULL;
dmuci_get_value_by_section_string(s, "dm_parent", &dm_parent);
if (data && DM_STRLEN(dm_parent)) {
if (strcmp(section_name((struct uci_section *)data), dm_parent) != 0)
continue;
}
cnt++;
}
dmasprintf(&value, "%d", cnt);
@ -662,22 +694,35 @@ static char *uci_get_value(json_object *mapping_obj, int json_version, char *ref
dmasprintf(&value, "%s", section_name((struct uci_section *)data));
} else {
char uci_type[32] = {0};
snprintf(uci_type, sizeof(uci_type), "@%s[%ld]", json_object_get_string(type), instance ? DM_STRTOL(instance)-1 : 0);
if (option) {
char *res = dmuci_get_value_by_path(json_object_get_string(path), json_object_get_string(file), uci_type, opt_temp);
char *res = NULL;
dmuci_get_value_by_section_string((struct uci_section *)data, opt_temp, &res);
if (DM_STRLEN(res) == 0)
dmuci_get_option_value_string(json_object_get_string(file), uci_type, opt_temp, &res);
if (linker_jobj)
adm_entry_get_linker_param(ctx, linker, res, &value);
else
value = res;
} else {
struct uci_list *list_val;
dmuci_get_option_value_list(json_object_get_string(file), uci_type, opt_temp, &list_val);
dmuci_get_value_by_section_list((struct uci_section *)data, opt_temp, &list_val);
if (list_val == NULL)
dmuci_get_option_value_list(json_object_get_string(file), uci_type, opt_temp, &list_val);
value = dmuci_list_to_string(list_val, ",");
}
}
} else if (file && section_name && opt_temp) {
if (option) {
char *res = dmuci_get_value_by_path(json_object_get_string(path), json_object_get_string(file), json_object_get_string(section_name), opt_temp);
char *res = NULL;
dmuci_get_option_value_string(json_object_get_string(file), json_object_get_string(section_name), opt_temp, &res);
if (linker_jobj)
adm_entry_get_linker_param(ctx, linker, res, &value);
else
@ -735,7 +780,7 @@ static char *ubus_get_value(json_object *mapping_obj, int json_version, char *re
json_object *json_obj = NULL;
json_object *arr_obj = NULL;
char key_buf[128] = {0};
char key_name[32] = {0};
char key_name[128] = {0};
DM_STRNCPY(key_buf, json_object_get_string(key), sizeof(key_buf));
@ -1249,7 +1294,6 @@ static void uci_set_value(json_object *mapping_obj, int json_version, char *refp
struct json_object *option_name = NULL;
struct json_object *list = NULL;
struct json_object *list_name = NULL;
struct json_object *path = NULL;
struct json_object *linker_jobj = NULL;
char *linker = NULL;
@ -1262,7 +1306,6 @@ static void uci_set_value(json_object *mapping_obj, int json_version, char *refp
json_object_object_get_ex(option, "name", &option_name);
json_object_object_get_ex(uci_obj, "list", &list);
json_object_object_get_ex(list, "name", &list_name);
json_object_object_get_ex(uci_obj, "path", &path);
json_object_object_get_ex(mapping_obj, "linker_obj", &linker_jobj);
char *opt_temp = NULL;
@ -1273,6 +1316,7 @@ static void uci_set_value(json_object *mapping_obj, int json_version, char *refp
}
if (data && file && type && opt_temp) {
char uci_type[32] = {0};
snprintf(uci_type, sizeof(uci_type), "@%s[%ld]", json_object_get_string(type), instance ? DM_STRTOL(instance)-1 : 0);
@ -1285,7 +1329,8 @@ static void uci_set_value(json_object *mapping_obj, int json_version, char *refp
get_dmmap_section_of_config_section(buf, json_object_get_string(type), section_name((struct uci_section *)data), &dmmap_section);
dmuci_set_value_by_section(dmmap_section, "section_name", value);
dmuci_rename_section(json_object_get_string(file), uci_type, value);
if (dmuci_rename_section_by_section((struct uci_section *)data, value) == -1)
dmuci_rename_section(json_object_get_string(file), uci_type, value);
return;
}
@ -1295,14 +1340,18 @@ static void uci_set_value(json_object *mapping_obj, int json_version, char *refp
else
linker = value;
dmuci_set_value_by_path(json_object_get_string(path), json_object_get_string(file), uci_type, opt_temp, linker);
if (dmuci_set_value_by_section((struct uci_section *)data, opt_temp, linker) == -1)
dmuci_set_value(json_object_get_string(file), uci_type, opt_temp, linker);
} else {
if (value != NULL) {
dmuci_delete(json_object_get_string(file), uci_type, opt_temp, NULL);
if (dmuci_delete_by_section((struct uci_section *)data, opt_temp, NULL) == -1)
dmuci_delete(json_object_get_string(file), uci_type, opt_temp, NULL);
char *p = strtok(value, ",");
while (p) {
strip_lead_trail_whitespace(p);
dmuci_add_list_value(json_object_get_string(file), uci_type, opt_temp, p);
if (dmuci_add_list_value_by_section((struct uci_section *)data, opt_temp, p) == -1)
dmuci_add_list_value(json_object_get_string(file), uci_type, opt_temp, p);
p = strtok(NULL, ",");
}
}
@ -1314,7 +1363,7 @@ static void uci_set_value(json_object *mapping_obj, int json_version, char *refp
else
linker = value;
dmuci_set_value_by_path(json_object_get_string(path), json_object_get_string(file), json_object_get_string(section_name), opt_temp, linker);
dmuci_set_value(json_object_get_string(file), json_object_get_string(section_name), opt_temp, linker);
} else {
if (value != NULL) {
dmuci_delete(json_object_get_string(file), json_object_get_string(section_name), opt_temp, NULL);

View file

@ -409,6 +409,34 @@ Ubus example for the same
}
```
## How to map Multi Instance Object for Multiple Sections:
- This is only applicable when mapping multi-instance Objects to **uci** config sections
- Multi instance object must be mapped to uci 'config' sections
- Uci options can only be mapped to leaf dm nodes
- Uci list options can only be mapped to leaf dm nodes which show the values as csv list
- It better to use a named config sections in place of unnamed config sections for multi-instance objects
- Children on multi-instance objects needs to have reference to their parent using 'dm_parent' option in uci section like below:
```bash
config agent 'agent'
option timezone 'GMT0BST,M3.5.0/1,M10.5.0'
config task 'http_get_mt'
option dm_parent 'agent'
option name 'http_get_mt'
config task_option 'http_get_mt_target'
option dm_parent 'http_get_mt'
option id 'target'
```
This object 'Device.LMAP.MeasurementAgent.{i}.Task.{i}.Option.{i}.' maps to the config above. It contains 3 Multi instance objects:
1. MeasurementAgent.{i}: parent object which maps to 'agent' section
2. Task.{i}: child object of MeasurementAgent object which maps to 'task' section, it must have a 'dm_parent' option with the value 'agent'
3. Option.{i}: child object of Task object which maps to 'task_option' section, it must have a 'dm_parent' option with the value 'http_get_mt'
## Feature supported with this mapping
- Use of `json_plugin_version` for mapping extensions
- Use different mappings for get/set

View file

@ -19,7 +19,6 @@ date +%s > timestamp.log
install_libbbf
install_libbbf_test
install_libbulkdata
install_libperiodicstats
supervisorctl update
@ -40,6 +39,12 @@ function run_valgrind_verbose()
exec_cmd_verbose valgrind -q --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes ./test/bbf_test/bbf_dm $@
}
function run_valgrind_redirect()
{
echo "Running # bbf_dm $@ #" > output-report-device-get.txt
exec_cmd_verbose valgrind -q --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes ./test/bbf_test/bbf_dm $@ | tee -a output-report-device-get.txt
}
echo "Running memory check on datamodel"
run_valgrind_verbose -u get Device.RootDataModelVersion
@ -61,6 +66,8 @@ run_valgrind -u get_info Device. 3
run_valgrind -u get Device.
run_valgrind -c get Device.
run_valgrind_redirect -u get Device.
supervisorctl stop all
supervisorctl status

View file

@ -106,19 +106,6 @@ function install_libbbf_test()
#make -C test/dynamicdm_ubus_test/
}
function install_libbulkdata()
{
# clone and compile libbulkdata
rm -rf /opt/dev/bulkdata
exec_cmd git clone -b devel https://dev.iopsys.eu/iopsys/bulkdata.git /opt/dev/bulkdata
echo "Compiling libbulkdata"
make clean -C /opt/dev/bulkdata/
make CFLAGS="-D_GNU_SOURCE -DWC_NO_HARDEN" -C /opt/dev/bulkdata/
echo "installing libbulkdata"
cp -f /opt/dev/bulkdata/bbf_plugin/libbulkdata.so /usr/lib/bbfdm
}
function install_libperiodicstats()
{
# clone and compile libperiodicstats

View file

@ -263,7 +263,7 @@ struct uci_section *get_origin_section_from_config(char *package, char *section_
struct uci_section *s = NULL;
uci_foreach_sections(package, section_type, s) {
char sec_name[32] = {0};
char sec_name[128] = {0};
dmuci_replace_invalid_characters_from_section_name(orig_section_name, sec_name, sizeof(sec_name));
@ -280,7 +280,7 @@ struct uci_section *get_dup_section_in_dmmap(char *dmmap_package, char *section_
uci_path_foreach_sections(bbfdm, dmmap_package, section_type, s) {
char *dmmap_sec_name = NULL;
char sec_name[32] = {0};
char sec_name[128] = {0};
dmuci_get_value_by_section_string(s, "section_name", &dmmap_sec_name);
dmuci_replace_invalid_characters_from_section_name(dmmap_sec_name, sec_name, sizeof(sec_name));

View file

@ -887,58 +887,6 @@ void commit_and_free_uci_ctx_bbfdm(char *dmmap_config)
uci_ctx_bbfdm = NULL;
}
char *dmuci_get_value_by_path(char *path, char *package, char *section, char *option)
{
struct uci_option *o;
char *val = "";
if (!package || !section || !option)
return val;
o = dmuci_get_option_ptr((path) ? path : UCI_CONFIG_DIR, package, section, option);
if (!o)
return val;
if(o->type == UCI_TYPE_LIST)
return dmuci_list_to_string(&o->v.list, " ");
if (o->v.string)
return dmstrdup(o->v.string);
return val;
}
char *dmuci_set_value_by_path(char *path, char *package, char *section, char *option, char *value)
{
struct uci_context *save_uci_ctx = NULL;
struct uci_ptr ptr = {0};
char *val = "";
if (!package || !section || !option || !value)
return val;
if (path && strcmp(path, BBFDM_CONFIG) == 0) {
save_uci_ctx = uci_ctx;
uci_ctx = uci_ctx_bbfdm;
}
if (dmuci_lookup_ptr(uci_ctx, &ptr, package, section, option, value))
goto end;
if (uci_set(uci_ctx, &ptr) != UCI_OK)
goto end;
if (ptr.o && ptr.o->v.string)
val = dmstrdup(ptr.o->v.string);
end:
if (path && strcmp(path, BBFDM_CONFIG) == 0)
uci_ctx = save_uci_ctx;
return val;
}
bool dmuci_string_to_boolean(char *value)
{
if (!value)

View file

@ -375,8 +375,6 @@ int db_get_value_string(char *package, char *section, char *option, char **value
int dmuci_get_option_value_string_varstate(char *package, char *section, char *option, char **value);
int dmuci_set_value_varstate(char *package, char *section, char *option, char *value);
char *dmuci_get_value_by_path(char *path, char *package, char *section, char *option);
char *dmuci_set_value_by_path(char *path, char *package, char *section, char *option, char *value);
bool dmuci_string_to_boolean(char *value);
void dmuci_replace_invalid_characters_from_section_name(char *old_sec_name, char *new_sec_name, size_t len);

File diff suppressed because it is too large Load diff

View file

@ -4,8 +4,7 @@ config bulkdata 'bulkdata'
#Log levels: Critical=0, Warning=1, Notice=2, Info=3, Debug=4
option log_level '3'
config profile
option profile_id '1'
config profile 'profile_1'
option enable '0'
option name ''
option nbre_of_retained_failed_reports '0'
@ -31,13 +30,79 @@ config profile
option http_retry_interval_multiplier '2000'
option http_persist_across_reboot '0'
config profile_parameter
option profile_id '1'
config profile_parameter 'profile_param_11'
option dm_parent 'profile_1'
option name ''
option reference ''
option reference 'Device.WiFi.AccessPoint.*.AssociatedDevice.*.Stats.BytesSent'
config profile_http_request_uri_parameter
option profile_id '1'
config profile_parameter 'profile_param_12'
option dm_parent 'profile_1'
option name 'BroadPktSent'
option reference 'Device.MoCA.Interface.1.Stats.BroadPktSent'
config profile_parameter 'profile_param_13'
option dm_parent 'profile_1'
option name 'WiFi_AP1_Assoc10'
option reference 'Device.WiFi.AccessPoint.1.AssociatedDevice.10.Stats.'
config profile_http_request_uri_parameter 'profile_uri_param_11'
option dm_parent 'profile_1'
option name 'ct'
option reference 'Device.Time.CurrentLocalTime'
config profile_http_request_uri_parameter 'profile_uri_param_12'
option dm_parent 'profile_1'
option name 'lan_name'
option reference 'Device.IP.Interface.1.Name'
config profile_http_request_uri_parameter 'profile_uri_param_13'
option dm_parent 'profile_1'
option name 'tm_enable'
option reference 'Device.Time.Enable'
config profile 'profile_2'
option enable '0'
option name ''
option reference ''
option nbre_of_retained_failed_reports '0'
option protocol 'http'
option encoding_type ''
option reporting_interval '86400'
option time_reference '0'
option csv_encoding_field_separator ','
option csv_encoding_row_separator '
'
option csv_encoding_escape_character '"'
option csv_encoding_report_format 'column'
option csv_encoding_row_time_stamp 'unix'
option json_encoding_report_format 'objecthierarchy'
option json_encoding_report_time_stamp 'unix'
option http_url ''
option http_username ''
option http_password ''
option http_compression 'none'
option http_method 'post'
option http_use_date_header '1'
option http_retry_enable '0'
option http_retry_minimum_wait_interval '5'
option http_retry_interval_multiplier '2000'
option http_persist_across_reboot '0'
config profile_parameter 'profile_param_21'
option dm_parent 'profile_2'
option name ''
option reference 'Device.WiFi.AccessPoint.*.AssociatedDevice.*.Stats.BytesReceived'
config profile_parameter 'profile_param_22'
option dm_parent 'profile_2'
option name 'UnicastPktSent'
option reference 'Device.MoCA.Interface.1.Stats.UnicastPacketsSent'
config profile_http_request_uri_parameter 'profile_uri_param_21'
option dm_parent 'profile_2'
option name 'ltz'
option reference 'Device.Time.LocalTimeZone'
config profile_http_request_uri_parameter 'profile_uri_param_22'
option dm_parent 'profile_2'
option name 'wan_name'
option reference 'Device.IP.Interface.2.Name'