From df6e91db4adaa0cf41547cf6fce4e53786b234a5 Mon Sep 17 00:00:00 2001 From: Amin Ben Romdhane Date: Wed, 13 Dec 2023 14:54:27 +0000 Subject: [PATCH] Optimize ci/cd pipeline --- gitlab-ci/bbfdm_services.conf | 12 --- gitlab-ci/bbfdmd-functional-test.sh | 3 - gitlab-ci/install-dependencies.sh | 44 +++------ gitlab-ci/shared.sh | 124 +------------------------- test/cmocka/functional_test_bbfd.c | 6 +- test/python/validate_schema_notify.py | 2 +- 6 files changed, 17 insertions(+), 174 deletions(-) diff --git a/gitlab-ci/bbfdm_services.conf b/gitlab-ci/bbfdm_services.conf index c99ebb67..d3267ac5 100644 --- a/gitlab-ci/bbfdm_services.conf +++ b/gitlab-ci/bbfdm_services.conf @@ -1,15 +1,3 @@ [program:bbfdmd] priority=10 command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/bbfdmd" - -[program:bbfdm_dataelementsd] -priority=11 -command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-report-dataelements.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/bbfdmd -m /tmp/wifi_dataelements.json" - -[program:bbfdm_bulkdatad] -priority=12 -command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-report-bulkdata.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/bbfdmd -m /etc/bulkdata/input.json" - -[program:bbfdm_periodicstatsd] -priority=13 -command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-report-periodicstats.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/bbfdmd -m /etc/periodicstats/input.json" diff --git a/gitlab-ci/bbfdmd-functional-test.sh b/gitlab-ci/bbfdmd-functional-test.sh index 096407a9..a35357e4 100755 --- a/gitlab-ci/bbfdmd-functional-test.sh +++ b/gitlab-ci/bbfdmd-functional-test.sh @@ -57,9 +57,6 @@ gcovr -r . echo > memory-report.xml check_valgrind_xml "Main Service bbfdmd" "/tmp/memory-report.xml" -check_valgrind_xml "Micro Service bbfdm_dataelementsd" "/tmp/memory-report-dataelements.xml" -check_valgrind_xml "Micro Service bbfdm_bulkdatad" "/tmp/memory-report-bulkdata.xml" -check_valgrind_xml "Micro Service bbfdm_periodicstatsd" "/tmp/memory-report-periodicstats.xml" if [ "${fault}" -ne 0 ]; then echo "Failed running ubus-api-validator fault[$fault]" diff --git a/gitlab-ci/install-dependencies.sh b/gitlab-ci/install-dependencies.sh index 3d4376f1..20cea6cb 100755 --- a/gitlab-ci/install-dependencies.sh +++ b/gitlab-ci/install-dependencies.sh @@ -6,48 +6,28 @@ source ./gitlab-ci/shared.sh # install required packages exec_cmd apt update -exec_cmd apt install -y python3-pip iproute2 -exec_cmd pip3 install pexpect ubus +exec_cmd apt install -y python3-pip iproute2 jq +exec_cmd pip3 install pexpect ubus xlwt + +# Make sure that all plugins are removed +repo_dir="/etc/bbfdm/plugins" +[ ! -d "${repo_dir}" ] && mkdir -p "${repo_dir}" +rm -f ${repo_dir}/* # compile and install libbbf install_libbbf ${1} -mkdir -p /etc/bbfdm/plugins #compile and install libbbf_test dynamic extension library install_libbbf_test ${1} -#compile and install libwifi_dataelements dynamic extension library -install_libwifi_dataelements ${1} - # Install datamodel plugins/micro-service only when pipeline trigger for bbfdm if [ -z "${1}" ]; then - git clone --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/feed/iopsys.git /opt/dev/iopsys - git clone --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/bbf/bulkdata.git /opt/dev/bulkdata - - install_plugin /opt/dev/iopsys/urlfilter/files/etc/bbfdm/plugins/urlfilter.json - install_plugin /opt/dev/iopsys/obuspa/files/etc/bbfdm/json/USPAgent.json - install_plugin /opt/dev/iopsys/icwmp/files/etc/bbfdm/json/CWMPManagementServer.json - install_plugin /opt/dev/iopsys/ponmngr/files/etc/bbfdm/json/xpon.json + # Generate plugin_input.json + jq 'del(.output)' tools/tools_input.json > plugin_input.json - # install bulkdata micro-service - mkdir -p /etc/bulkdata - cp -f /opt/dev/bulkdata/bbf_plugin/bulkdata.json /etc/bulkdata - cp -f /opt/dev/iopsys/bulkdata/files/etc/bulkdata/input.json /etc/bulkdata - - # install usermngr plugin - install_libusermngr - - # install periodicstats micro-service - install_periodicstats - - # install cwmpdm plugin - install_libcwmpdm + # Install datamodel plugins + ./tools/generate_dm.py plugin_input.json + check_ret $? - # install hosts micro-service - install_hosts_micro_service - - # install time micro-service - install_time_micro_service - ls -l /etc/bbfdm/plugins/ fi diff --git a/gitlab-ci/shared.sh b/gitlab-ci/shared.sh index 87514cf1..96d3abce 100755 --- a/gitlab-ci/shared.sh +++ b/gitlab-ci/shared.sh @@ -44,25 +44,6 @@ function install_plugin() exec_cmd cp -f "${1}" /etc/bbfdm/plugins/ } -function install_libusermngr() -{ - # clone and compile libusermngr - [ -d "/opt/dev/usermngr" ] && rm -rf /opt/dev/usermngr - - if [ -n "${USERMNGR_BRANCH}" ]; then - exec_cmd git clone -b ${USERMNGR_BRANCH} https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/bbf/usermngr.git /opt/dev/usermngr - else - exec_cmd git clone -b devel https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/bbf/usermngr.git /opt/dev/usermngr - fi - - echo "Compiling libusermngr" - exec_cmd_verbose make clean -C /opt/dev/usermngr/src/ - exec_cmd_verbose make -C /opt/dev/usermngr/src/ - - echo "installing libusermngr" - install_plugin /opt/dev/usermngr/src/libusermngr.so -} - function install_libbbf() { # Enable coverage flags only for test @@ -81,7 +62,7 @@ function install_libbbf() mkdir -p build cd build - cmake ../ -DCMAKE_C_FLAGS="$COV_CFLAGS " -DCMAKE_EXE_LINKER_FLAGS="$COV_LDFLAGS -lm" -DWITH_OPENSSL=ON -DBBF_VENDOR_EXTENSION=ON -DBBF_WIFI_DATAELEMENTS=OFF -DBBF_VENDOR_LIST="$VENDOR_LIST" -DBBF_VENDOR_PREFIX="$VENDOR_PREFIX" -DBBF_MAX_OBJECT_INSTANCES=255 -DBBFDMD_MAX_MSG_LEN=1048576 -DCMAKE_INSTALL_PREFIX=/ + cmake ../ -DCMAKE_C_FLAGS="$COV_CFLAGS " -DCMAKE_EXE_LINKER_FLAGS="$COV_LDFLAGS -lm" -DWITH_OPENSSL=ON -DBBF_VENDOR_EXTENSION=ON -DBBF_VENDOR_LIST="$VENDOR_LIST" -DBBF_VENDOR_PREFIX="$VENDOR_PREFIX" -DBBF_MAX_OBJECT_INSTANCES=255 -DBBFDMD_MAX_MSG_LEN=1048576 -DCMAKE_INSTALL_PREFIX=/ exec_cmd_verbose make echo "installing libbbf" @@ -104,107 +85,6 @@ function install_libbbf_test() install_plugin ./test/bbf_test/libbbf_test.so } -function install_libwifi_dataelements() -{ - # Enable coverage flags only for test - [ -n "${1}" ] && return 0; - - # compile and install libwifi_dataelements - echo "Compiling libwifi_dataelements" - exec_cmd_verbose make clean -C test/wifi_dataelements/ - exec_cmd_verbose make -C test/wifi_dataelements/ - - echo "installing libwifi_dataelements" - cp -f test/wifi_dataelements/wifi_dataelements.json /tmp/wifi_dataelements.json - cp -f test/wifi_dataelements/libwifi_dataelements.so /tmp/libwifi_dataelements.so -} - -function install_libperiodicstats() -{ - # clone and compile libperiodicstats - [ -d "/opt/dev/periodicstats" ] && rm -rf /opt/dev/periodicstats - - if [ -n "${PERIODICSTATS_BRANCH}" ]; then - exec_cmd git clone -b ${PERIODICSTATS_BRANCH} https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/bbf/periodicstats.git /opt/dev/periodicstats - else - exec_cmd git clone -b devel https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/bbf/periodicstats.git /opt/dev/periodicstats - fi - - echo "Compiling libperiodicstats" - exec_cmd_verbose make clean -C /opt/dev/periodicstats/ - exec_cmd_verbose make -C /opt/dev/periodicstats/ - - echo "installing libperiodicstats" - mkdir -p /etc/periodicstats - cp -f /opt/dev/periodicstats/bbf_plugin/libperiodicstats.so /etc/periodicstats/ - cp -f /opt/dev/iopsys/periodicstats/files/etc/periodicstats/input.json /etc/periodicstats -} - -function install_libcwmpdm() -{ - # clone and compile libcwmpdm - [ -d "/opt/dev/icwmp" ] && rm -rf /opt/dev/icwmp - - if [ -n "${ICWMP_BRANCH}" ]; then - exec_cmd git clone -b ${ICWMP_BRANCH} --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/bbf/icwmp.git /opt/dev/icwmp - else - exec_cmd git clone -b devel --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/bbf/icwmp.git /opt/dev/icwmp - fi - - echo "Compiling libcwmpdm" - cd /opt/dev/icwmp/bbf_plugin - cmake -DCMAKE_INSTALL_PREFIX=/ - exec_cmd_verbose make - - echo "installing libcwmpdm" - install_plugin /opt/dev/icwmp/bbf_plugin/libcwmpdm.so - - cd /builds/bbf/bbfdm -} - - -function install_hosts_micro_service() -{ - # clone and compile hostdm - [ -d "/opt/dev/hostmngr" ] && rm -rf /opt/dev/hostmngr - - if [ -n "${HOSTS_BRANCH}" ]; then - exec_cmd git clone -b ${HOSTS_BRANCH} --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/iopsys/hostmngr.git /opt/dev/hostmngr - else - exec_cmd git clone -b devel --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/iopsys/hostmngr.git /opt/dev/hostmngr - fi - - echo "Compiling hosts micro-service" - exec_cmd_verbose make clean -C /opt/dev/hostmngr/src/bbf_plugin - exec_cmd_verbose make -C /opt/dev/hostmngr/src/bbf_plugin - - echo "installing hosts micro-service" - mkdir -p /etc/hostmngr - cp -f /opt/dev/hostmngr/src/bbf_plugin/libhostmngr.so /etc/hostmngr/ - cp -f /opt/dev/iopsys/hostmngr/files/etc/hostmngr/input.json /etc/hostmngr -} - -function install_time_micro_service() -{ - # clone and compile timedm - [ -d "/opt/dev/timemngr" ] && rm -rf /opt/dev/timemngr - - if [ -n "${TIME_BRANCH}" ]; then - exec_cmd git clone -b ${TIME_BRANCH} --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/bbf/timemngr.git /opt/dev/timemngr - else - exec_cmd git clone -b devel --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@dev.iopsys.eu/bbf/timemngr.git /opt/dev/timemngr - fi - - echo "Compiling time micro-service" - exec_cmd_verbose make clean -C /opt/dev/timemngr/src - exec_cmd_verbose make -C /opt/dev/timemngr/src - - echo "installing time micro-service" - mkdir -p /etc/timemngr - cp -f /opt/dev/timemngr/src/libtimemngr.so /etc/timemngr/ - cp -f /opt/dev/iopsys/timemngr/files/etc/timemngr/input.json /etc/timemngr -} - function error_on_zero() { ret=$1 @@ -217,8 +97,6 @@ function error_on_zero() } function check_valgrind_xml() { - cat ${1} >> memory-report.xml - echo "${1}: Checking memory leaks..." echo "checking UninitCondition" grep -q "UninitCondition" ${2} diff --git a/test/cmocka/functional_test_bbfd.c b/test/cmocka/functional_test_bbfd.c index 97c7db12..929ffc1e 100644 --- a/test/cmocka/functional_test_bbfd.c +++ b/test/cmocka/functional_test_bbfd.c @@ -1509,7 +1509,7 @@ static void test_api_bbfdm_valid_library_event(void **state) idx++; } - assert_int_equal(idx, 6); + assert_int_equal(idx, 9); } static void test_api_bbfdm_valid_json_event(void **state) @@ -1556,7 +1556,7 @@ static void test_api_bbfdm_valid_json_event(void **state) idx++; } - assert_int_equal(idx, 6); + assert_int_equal(idx, 9); } static void test_api_bbfdm_valid_json_v1_event(void **state) @@ -1605,7 +1605,7 @@ static void test_api_bbfdm_valid_json_v1_event(void **state) idx++; } - assert_int_equal(idx, 6); + assert_int_equal(idx, 9); } int main(void) diff --git a/test/python/validate_schema_notify.py b/test/python/validate_schema_notify.py index 4b26aa08..b9acb8b4 100755 --- a/test/python/validate_schema_notify.py +++ b/test/python/validate_schema_notify.py @@ -18,7 +18,7 @@ except: if ret == 0: try: - ret = child.expect('schema_update_available') + ret = child.expect('bbfdm.DelObj') except: print("FAIL: Schema updater notification")