From 94fd8ccbb7fc03cde72bccf16cd67da8b95d16ce Mon Sep 17 00:00:00 2001 From: Amin Ben Romdhane Date: Thu, 7 Nov 2024 10:26:54 +0000 Subject: [PATCH] Improve pipeline to run external modules as micro-service --- gitlab-ci/bbfdm_services.conf | 2 +- gitlab-ci/install-dependencies.sh | 16 +++++ gitlab-ci/memory-test.sh | 1 + gitlab-ci/micro_service.conf | 11 ++++ gitlab-ci/reload_service.conf | 4 +- gitlab-ci/setup.sh | 4 ++ gitlab-ci/shared.sh | 101 ++++++++++++++++++++++++++++- libbbfdm-api/plugin/dotso_plugin.c | 4 -- 8 files changed, 134 insertions(+), 9 deletions(-) create mode 100644 gitlab-ci/micro_service.conf diff --git a/gitlab-ci/bbfdm_services.conf b/gitlab-ci/bbfdm_services.conf index d3267ac5..71c84e8a 100644 --- a/gitlab-ci/bbfdm_services.conf +++ b/gitlab-ci/bbfdm_services.conf @@ -1,3 +1,3 @@ [program:bbfdmd] -priority=10 +priority=15 command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/bbfdmd" diff --git a/gitlab-ci/install-dependencies.sh b/gitlab-ci/install-dependencies.sh index 1cb9730e..d03b8548 100755 --- a/gitlab-ci/install-dependencies.sh +++ b/gitlab-ci/install-dependencies.sh @@ -30,4 +30,20 @@ if [ -z "${1}" ]; then check_ret $? ls -l /usr/share/bbfdm/plugins/ +else + # Create directories for micro-service configuration and shared files + mkdir -p /etc/bbfdm/micro_services + mkdir -p /usr/share/bbfdm/micro_services + + #install WiFi Data Model as a micro-service + echo "Installing WiFi Data Model (wifidmd) as a micro-service" + install_wifidmd_as_micro_service + + #install Network Data Model as a micro-service + echo "Installing Network Data Model (netmngr) as a micro-service" + install_netmngr_as_micro_service + + #install SYSMNGR Data Model as a micro-service + echo "Installing System Manager (SYSMNGR) Data Model as a micro-service" + install_sysmngr_as_micro_service fi diff --git a/gitlab-ci/memory-test.sh b/gitlab-ci/memory-test.sh index f82b0031..15c631d9 100755 --- a/gitlab-ci/memory-test.sh +++ b/gitlab-ci/memory-test.sh @@ -62,6 +62,7 @@ supervisorctl status cp /tmp/memory-*.xml . check_valgrind_xml "memory-report.xml" "bbfdmd" check_valgrind_xml "memory-config-report.xml" "bbf.config" + #report part #GitLab-CI output gcovr -r . 2> /dev/null #throw away stderr diff --git a/gitlab-ci/micro_service.conf b/gitlab-ci/micro_service.conf new file mode 100644 index 00000000..5b410136 --- /dev/null +++ b/gitlab-ci/micro_service.conf @@ -0,0 +1,11 @@ +[program:sysmng] +priority=5 +command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-sysmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/sysmngr" + +[program:netmngr] +priority=6 +command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-netmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m netmngr" + +[program:wifidmd] +priority=7 +command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-wifidmd-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m wifidmd" diff --git a/gitlab-ci/reload_service.conf b/gitlab-ci/reload_service.conf index 97fb47c1..373dba92 100644 --- a/gitlab-ci/reload_service.conf +++ b/gitlab-ci/reload_service.conf @@ -1,5 +1,3 @@ [program:bbf.config] -priority=5 +priority=4 command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-config-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/bbf_configd" - - diff --git a/gitlab-ci/setup.sh b/gitlab-ci/setup.sh index 2bc618de..2a664bb8 100755 --- a/gitlab-ci/setup.sh +++ b/gitlab-ci/setup.sh @@ -20,6 +20,10 @@ mkdir -p /tmp/bbfdm/.bbfdm /tmp/bbfdm/.cwmp /tmp/bbfdm/.usp cp ./gitlab-ci/core_service.conf /etc/supervisor/conf.d/ cp ./gitlab-ci/reload_service.conf /etc/supervisor/conf.d/ +if [ -n "$1" ]; then + cp ./gitlab-ci/micro_service.conf /etc/supervisor/conf.d/ +fi + rm -f /etc/bbfdm/dmmap/* echo "Starting base services..." diff --git a/gitlab-ci/shared.sh b/gitlab-ci/shared.sh index 99e00f8a..d038c807 100755 --- a/gitlab-ci/shared.sh +++ b/gitlab-ci/shared.sh @@ -43,6 +43,40 @@ function exec_cmd_verbose() fi } +generate_input_schema() +{ + service_name="$1" + schema='{ + "daemon": { + "enable": "1", + "service_name": "'"$service_name"'", + "config": { + "loglevel": "4" + } + } +}' + echo "$schema" +} + +generate_input_schema_with_output_name() +{ + service_name="$1" + output_name="$2" + schema='{ + "daemon": { + "enable": "1", + "service_name": "'"$service_name"'", + "config": { + "loglevel": "4" + }, + "output": { + "name": "'"$output_name"'" + } + } +}' + echo "$schema" +} + function install_plugin() { exec_cmd cp -f "${1}" ${BBFDM_PLUGIN_DIR}/ @@ -65,7 +99,7 @@ function install_libbbf() mkdir -p build cd build - cmake ../ -DCMAKE_C_FLAGS="$COV_CFLAGS " -DCMAKE_EXE_LINKER_FLAGS="$COV_LDFLAGS -lm" -DBBF_VENDOR_PREFIX="$VENDOR_PREFIX" -DBBF_MAX_OBJECT_INSTANCES=255 -DBBF_SCHEMA_FULL_TREE=ON -DBBFDMD_MAX_MSG_LEN=1048576 -DCMAKE_INSTALL_PREFIX=/ + cmake ../ -DCMAKE_C_FLAGS="$COV_CFLAGS " -DCMAKE_EXE_LINKER_FLAGS="$COV_LDFLAGS -lm" -DBBF_VENDOR_PREFIX="$VENDOR_PREFIX" -DBBF_MAX_OBJECT_INSTANCES=255 -DBBFDMD_MAX_MSG_LEN=1048576 -DCMAKE_INSTALL_PREFIX=/ exec_cmd_verbose make echo "installing libbbf" @@ -89,6 +123,71 @@ function install_libbbf_test() install_plugin ./test/bbf_test/libbbf_test.so } +function install_wifidmd_as_micro_service() +{ + [ -d "/opt/dev/wifidmd" ] && return 0 + + exec_cmd git clone https://dev.iopsys.eu/bbf/wifidmd.git /opt/dev/wifidmd + + exec_cmd make -C /opt/dev/wifidmd/src/ clean && make -C /opt/dev/wifidmd/src/ CFLAGS="-D'BBF_VENDOR_PREFIX=\"X_IOPSYS_EU_\"'" + exec_cmd cp -f /opt/dev/wifidmd/src/libwifi.so /usr/share/bbfdm/micro_services/wifidmd.so + exec_cmd mkdir -p /usr/share/bbfdm/micro_services/wifidmd + exec_cmd cp -f /opt/dev/wifidmd/src/libdataelements.so /usr/share/bbfdm/micro_services/wifidmd + + generate_input_schema_with_output_name "wifidmd" "WiFi" > /etc/bbfdm/micro_services/wifidmd.json +} + +function install_netmngr_as_micro_service() +{ + [ -d "/opt/dev/netmngr" ] && return 0 + + exec_cmd git clone https://dev.iopsys.eu/network/netmngr.git /opt/dev/netmngr + + exec_cmd make -C /opt/dev/netmngr/src/ clean && make -C /opt/dev/netmngr/src/ CFLAGS="-D'BBF_VENDOR_PREFIX=\"X_IOPSYS_EU_\"'" + exec_cmd cp -f /opt/dev/netmngr/src/libnetmngr.so /usr/share/bbfdm/micro_services/netmngr.so + exec_cmd cp -f /opt/dev/netmngr/src/libinterface_stack.so /usr/share/bbfdm/plugins + exec_cmd mkdir -p /usr/share/bbfdm/micro_services/netmngr + + generate_input_schema_with_output_name "netmngr" "Network" > /etc/bbfdm/micro_services/netmngr.json + + exec_cmd git clone https://dev.iopsys.eu/bbf/tr143d.git /opt/dev/tr143d + exec_cmd make -C /opt/dev/tr143d/src/ clean && make -C /opt/dev/tr143d/src/ + exec_cmd cp -f /opt/dev/tr143d/src/libtr143d.so /usr/share/bbfdm/micro_services/netmngr + + exec_cmd git clone https://dev.iopsys.eu/bbf/tr471d.git /opt/dev/tr471d + exec_cmd make -C /opt/dev/tr471d/src/ clean && make -C /opt/dev/tr471d/src/ + exec_cmd cp -f /opt/dev/tr471d/src/libtr471d.so /usr/share/bbfdm/micro_services/netmngr + + exec_cmd git clone https://dev.iopsys.eu/bbf/twamp-light.git /opt/dev/twamp + exec_cmd make -C /opt/dev/twamp clean && make -C /opt/dev/twamp + exec_cmd cp -f /opt/dev/twamp/libtwamp.so /usr/share/bbfdm/micro_services/netmngr + + exec_cmd git clone https://dev.iopsys.eu/bbf/udpecho.git /opt/dev/udpecho + exec_cmd make -C /opt/dev/udpecho/src/ clean && make -C /opt/dev/udpecho/src/ + exec_cmd cp -f /opt/dev/udpecho/src/libudpechoserver.so /usr/share/bbfdm/micro_services/netmngr +} + +function install_sysmngr_as_micro_service() +{ + [ -d "/opt/dev/sysmngr" ] && return 0 + + exec_cmd git clone https://dev.iopsys.eu/system/sysmngr.git /opt/dev/sysmngr + + exec_cmd make -C /opt/dev/sysmngr/src/ clean && \ + exec_cmd make -C /opt/dev/sysmngr/src/ \ + CFLAGS+="-DBBF_VENDOR_PREFIX=\\\"X_IOPSYS_EU_\\\"" \ + SYSMNGR_VENDOR_CONFIG_FILE='y' \ + SYSMNGR_MEMORY_STATUS='y' \ + SYSMNGR_PROCESS_STATUS='y' \ + SYSMNGR_SUPPORTED_DATA_MODEL='y' \ + SYSMNGR_FIRMWARE_IMAGE='y' \ + SYSMNGR_REBOOTS='y' \ + SYSMNGR_NETWORK_PROPERTIES='y' \ + SYSMNGR_VENDOR_EXTENSIONS='y' + + exec_cmd cp /opt/dev/sysmngr/src/sysmngr /usr/sbin/ +} + function error_on_zero() { ret=$1 diff --git a/libbbfdm-api/plugin/dotso_plugin.c b/libbbfdm-api/plugin/dotso_plugin.c index 9620975b..776ee9d8 100644 --- a/libbbfdm-api/plugin/dotso_plugin.c +++ b/libbbfdm-api/plugin/dotso_plugin.c @@ -69,9 +69,7 @@ int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path) #endif if (!handle) { char *err_msg = dlerror(); -#ifdef BBF_SCHEMA_FULL_TREE TRACE_FILE("Failed to add DotSo plugin '%s', [%s]\n", plugin_path, err_msg); -#endif BBF_ERR("Failed to add DotSo plugin '%s', [%s]\n", plugin_path, err_msg); return 0; } @@ -90,9 +88,7 @@ int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path) DMOBJ *dm_entryobj = find_entry_obj(entryobj, dynamic_obj[i].path); if (!dm_entryobj) { -#ifdef BBF_SCHEMA_FULL_TREE TRACE_FILE("Failed to add DotSo plugin '%s' to main tree with parent DM index '%d' => '%s'", plugin_path, i, dynamic_obj[i].path); -#endif BBF_ERR("Failed to add DotSo plugin '%s' to main tree with parent DM index '%d' => '%s'", plugin_path, i, dynamic_obj[i].path); continue; }