Update pipeline to cover all micro-services

.gitlab-ci.yml

@@ -5,6 +5,14 @@ variables:
   CPPCHECK_OPTIONS: "--suppress=cert-MSC24-C -DBBF_VENDOR_PREFIX=X_IOPSYS_EU_"
   CPD_OPTIONS: "--exclude ./build/ --minimum-tokens 200"
 
+before_script:
+  - |
+    echo "
+    machine dev.iopsys.eu
+    login gitlab-ci-token
+    password $CI_JOB_TOKEN
+    " > ~/.netrc
+
 include:
   - project: 'iopsys/gitlab-ci-pipeline'
     file: '/static-code-analysis.yml'
@@ -26,7 +34,7 @@ run_unit_test:
   allow_failure: false
   script:
   - "./gitlab-ci/pipeline_setup.sh"
-  - "./gitlab-ci/install-dependencies.sh ms"
+  - "./gitlab-ci/install-dependencies-ms.sh bbfdm"
   - "./gitlab-ci/setup.sh ms"
   - "./gitlab-ci/unit-test.sh"
 
@@ -40,17 +48,10 @@ run_tools_test:
   image: ${COMMON_IMAGE}
   allow_failure: false
   script:
-  - |
-      echo "
-      machine dev.iopsys.eu
-      login gitlab-ci-token
-      password $CI_JOB_TOKEN
-      " > ~/.netrc
   - "./gitlab-ci/pipeline_setup.sh"
-  - "./gitlab-ci/setup.sh"
+  - "./gitlab-ci/install-dependencies-ms.sh tools"
   - "./gitlab-ci/tools-test.sh"
   - "./gitlab-ci/generate_supported_dm.sh"
-
   artifacts:
       when: always
       paths:
@@ -64,7 +65,7 @@ run_libbbfdm_api_functional_test:
   allow_failure: false
   script:
   - "./gitlab-ci/pipeline_setup.sh"
-  - "./gitlab-ci/install-dependencies.sh ms"
+  - "./gitlab-ci/install-dependencies-ms.sh bbfdm"
   - "./gitlab-ci/setup.sh ms"
   - "./gitlab-ci/functional-api-test.sh"
 
@@ -79,7 +80,7 @@ run_libbbfdm_functional_test:
   allow_failure: false
   script:
   - "./gitlab-ci/pipeline_setup.sh"
-  - "./gitlab-ci/install-dependencies.sh ms"
+  - "./gitlab-ci/install-dependencies-ms.sh bbfdm"
   - "./gitlab-ci/setup.sh ms"
   - "./gitlab-ci/functional-test.sh"
 
@@ -94,7 +95,7 @@ run_libbbfdm_memory_test:
   allow_failure: false
   script:
   - "./gitlab-ci/pipeline_setup.sh"
-  - "./gitlab-ci/install-dependencies.sh ms"
+  - "./gitlab-ci/install-dependencies-ms.sh bbfdm"
   - "./gitlab-ci/setup.sh ms"
   - "./gitlab-ci/memory-test.sh"
   artifacts:
@@ -109,7 +110,7 @@ run_bbfd_functional_test:
     allow_failure: false
     script:
     - "./gitlab-ci/pipeline_setup.sh"
-    - "./gitlab-ci/install-dependencies.sh ms"
+    - "./gitlab-ci/install-dependencies-ms.sh bbfdm"
     - "./gitlab-ci/setup.sh ms"
     - "./gitlab-ci/bbfdmd-functional-test.sh"
     artifacts:

gitlab-ci/bbfdmd-functional-test.sh

@@ -24,15 +24,6 @@ echo "Checking system resources"
 free -h
 df -h
 
-# Check if the specified log file exists, which indicates errors during plugin loading
-if [ -f ${BBFDM_LOG_FILE} ]; then
-	 echo "Some plugins failed to load! Please check the errors below"
-	echo "*****************************************************"
-	cat "${BBFDM_LOG_FILE}"
-	echo "*****************************************************"
-	exit 1
-fi
-
 echo "## Running python based verification of functionalities ##"
 echo > ./funl-result.log
 num=0

gitlab-ci/full_micro_service.conf

@@ -0,0 +1,131 @@
+[program:sysmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-sysmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/sysmngr -l 3"
+
+[program:netmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-netmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m netmngr -l 3"
+
+[program:wifidmd]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-wifidmd-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/wifidmd -l 3"
+
+[program:core]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-core-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m core -l 3"
+
+[program:ethmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-ethmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/ethmngr -l 3"
+
+[program:icwmp]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-icwmp-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m icwmp -l 3"
+
+[program:bulkdata]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-bulkdata-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/bulkdatad -l 3"
+
+[program:periodicstats]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-periodicstats-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/periodicstatsd -l 3"
+
+[program:ponmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-ponmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m ponmngr -l 3"
+
+[program:ssdpd]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-ssdpd-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m ssdpd -l 3"
+
+[program:swmodd]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-swmodd-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m swmodd -l 3"
+
+[program:usermngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-usermngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/usermngr -l 3"
+
+[program:parentalcontrol]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-parentalcontrol-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/urlfilter -l 3"
+
+[program:hostmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-hostmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m hostmngr -l 3"
+
+[program:timemngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-timemngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/timemngr -l 3"
+
+[program:dnsmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-dnsmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m dnsmngr -l 3"
+
+[program:dhcpmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-dhcpmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m dhcpmngr -l 3"
+
+[program:qosmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-qosmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/qosmngr -l 3"
+
+[program:tr104]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-tr104-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m tr104 -l 3"
+
+[program:mcastmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-mcastmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m mcastmngr -l 3"
+
+[program:ieee1905]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-ieee1905-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m ieee1905 -l 3"
+
+[program:bridgemngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-bridgemngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m bridgemngr -l 3"
+
+[program:ddnsmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-ddnsmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m ddnsmngr -l 3"
+
+[program:sshmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-sshmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m sshmngr -l 3"
+
+[program:firewallmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-firewallmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m firewallmngr -l 3"
+
+[program:dslmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-dslmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m dslmngr -l 3"
+
+[program:usbmngr]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-usbmngr-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m usbmngr -l 3"
+
+[program:obuspa]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-obuspa-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m obuspa -l 3"
+
+[program:gnx-ux-manager]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-gnx-ux-manager-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m gnx-ux-manager -l 3"
+
+[program:gnx-catv]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-gnx-catv-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m gnx-catv -l 3"
+
+[program:dhcp-on-boarding]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-dhcp-on-boarding-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m dhcp-on-boarding -l 3"
+
+[program:gnx-loop-detector]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-gnx-loop-detector-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m gnx-loop-detector -l 3"
+
+[program:gnx-sfp]
+priority=10
+command=/bin/bash -c "/usr/bin/valgrind --xml=yes --xml-file=/tmp/memory-gnx-sfp-report.xml --leak-check=full --show-reachable=yes --show-leak-kinds=all --errors-for-leak-kinds=all --error-exitcode=1 --track-origins=yes --leak-resolution=high --show-error-list=yes --child-silent-after-fork=yes /usr/sbin/dm-service -m gnx-sfp -l 3"

gitlab-ci/generate_supported_dm.sh

@@ -15,15 +15,6 @@ if [ -n "${CI_SERVER_HOST}" ]; then
 	echo "password ${CI_JOB_TOKEN}" >>~/.netrc
 fi
 
-install_cmph
-install_libeasy
-install_libethernet
-install_libqos
-
-[ ! -d "${BBFDM_MS_DIR}" ] && mkdir -p "${BBFDM_MS_DIR}"
-rm -rf ${BBFDM_MS_DIR}/*
-mkdir -p ${BBFDM_MS_DIR}/core
-
 if [ -z "${1}" ]; then
 	./tools/generate_dm.py tools/tools_input.json
 else
@@ -45,13 +36,4 @@ echo "Validate datamodel_default generated XML file"
 xmllint --schema test/tools/cwmp-datamodel-*.xsd out/datamodel_default.xml --noout
 check_ret $?
 
-# Check if the specified log file exists, which indicates errors during plugin loading
-if [ -f ${BBFDM_LOG_FILE} ]; then
-	echo "Some plugins failed to load! Please check the errors below"
-	echo "*****************************************************"
-	cat "${BBFDM_LOG_FILE}"
-	echo "*****************************************************"
-	exit 1
-fi
-
 echo "Generation of xml and xls artifacts :: PASS"

gitlab-ci/install-dependencies-ms.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+echo "install dependencies of bbfdm"
+
+source ./gitlab-ci/shared.sh
+
+# install required packages
+exec_cmd apt update
+exec_cmd pip3 install xlwt
+
+# Create directories for micro-service configuration and shared files
+[ ! -d "${BBFDM_MS_CONF}" ] && mkdir -p "${BBFDM_MS_CONF}"
+[ ! -d "${BBFDM_MS_DIR}" ] && mkdir -p "${BBFDM_MS_DIR}"
+
+# Clean up generated files
+rm -rf ${BBFDM_MS_DIR}/*
+rm -f ${BBFDM_MS_CONF}/*
+rm -f ${BBFDM_DMMAP_DIR}/*
+
+# compile and install Core Data Model as a micro-service
+install_libbbf ${1}
+
+if [ "$1" == "bbfdm" ]; then
+	#compile and install libbbf_test dynamic extension library
+	install_libbbf_test
+fi
+
+# Install datamodel plugins/micro-service only when pipeline trigger for bbfdm
+if [ -z "${1}" ]; then
+	echo "Skip installation of micro-services ...."
+else
+	JSON_FILE=$BBFDM_TOOLS_INPUT_FILE
+	JSON_DESC_PATH="/tmp/desc_files"
+	plugin_count=$(jq '.plugins | length' "$JSON_FILE")
+
+	# Create desc_files directory
+	[ -d "${JSON_DESC_PATH}" ] && rm -f ${JSON_DESC_PATH}
+	mkdir -p ${JSON_DESC_PATH}
+
+	for i in $(seq 0 $((plugin_count - 1))); do
+		echo "==== Processing plugin [$i] ===="
+
+		repo=$(jq -r ".plugins[$i].repo" "$JSON_FILE")
+		version=$(jq -r ".plugins[$i].version" "$JSON_FILE")
+		plugin_name=$(basename "$repo" .git)
+		dest="$BBFDM_PLUGIN_DEST/$plugin_name"
+
+		# Adjust repo URL based on ~/.netrc existence
+		NETRC_PATH="$HOME/.netrc"
+		if [ ! -f "$NETRC_PATH" ]; then
+			repo=${repo/https:\/\/dev.iopsys.eu\//git@dev.iopsys.eu:}
+		fi
+
+		echo "Repo path: $repo"
+		echo "Plugin name: $plugin_name"
+		echo "Destination: $dest"
+
+		# Install dependencies
+		if [ "$plugin_name" == "ethmngr" ]; then
+			install_libeasy
+			install_libethernet
+			install_libqos
+		fi
+
+		if [ "$plugin_name" == "parental-control" ]; then
+			install_cmph
+		fi
+
+		if [ -d "$dest" ]; then
+			echo "Directory $dest already exists, skipping clone."
+		else
+			echo "Cloning $repo into $dest..."
+			git clone -b "$version" "$repo" "$dest" || { echo "❌ Git clone failed"; exit -1; }
+		fi
+
+		cd $dest
+
+		# Compilation
+		echo "Starting compilation..."
+		jq -r ".plugins[$i].compile[]" "$JSON_FILE" | while read -r cmd; do
+			echo "Executing: $cmd"
+			eval "$cmd" || { echo "❌ Compilation command failed"; exit -1; }
+		done
+
+		# Post-install
+		echo "Running post-install steps..."
+		jq -r ".plugins[$i].post_install[]" "$JSON_FILE" 2>/dev/null | while read -r post_cmd; do
+			echo "Executing: $post_cmd"
+			eval "$post_cmd" || { echo "❌ Post-install command failed"; exit -1; }
+		done
+
+		# Save dm_info_file if defined
+		dm_info_file=$(jq -r ".plugins[$i].dm_info_file // empty" "$JSON_FILE")
+		if [ -n "$dm_info_file" ]; then
+			src_file="$dest/$dm_info_file"
+			out_file="/tmp/desc_files/${i}_${plugin_name}.json"
+			if [ -f "$src_file" ]; then
+				echo "Saving dm_info_file to $out_file"
+				cp -f "$src_file" "$out_file"
+			else
+				echo "❌ dm_info_file not found: $src_file"
+			fi
+		fi
+
+		cd $(pwd)
+
+		echo "✅ Plugin [$plugin_name] build complete."
+	done
+fi

gitlab-ci/install-dependencies.sh

@@ -16,7 +16,6 @@ exec_cmd pip3 install xlwt
 rm -rf ${BBFDM_MS_DIR}/*
 rm -f ${BBFDM_MS_CONF}/*
 rm -f ${BBFDM_DMMAP_DIR}/*
-rm -f ${BBFDM_LOG_FILE}
 
 # compile and install Core Data Model as a micro-service
 install_libbbf ${1}

gitlab-ci/memory-test.sh

@@ -58,8 +58,9 @@ supervisorctl stop all
 supervisorctl status
 
 cp /tmp/memory-*.xml .
-check_valgrind_xml "memory-report.xml" "bbfdmd"
+for file in memory-*.xml; do
-check_valgrind_xml "memory-config-report.xml" "bbf.config"
+    check_valgrind_xml "$file"
+done
 
 #report part
 #GitLab-CI output

gitlab-ci/setup.sh

@@ -26,7 +26,11 @@ cp ./gitlab-ci/core_service.conf /etc/supervisor/conf.d/
 cp ./gitlab-ci/reload_service.conf /etc/supervisor/conf.d/
 
 if [ -n "$1" ]; then
-	cp ./gitlab-ci/micro_service.conf /etc/supervisor/conf.d/
+	if [ "$1" == "bbfdm" ]; then
+		cp ./gitlab-ci/full_micro_service.conf /etc/supervisor/conf.d/
+	else
+		cp ./gitlab-ci/micro_service.conf /etc/supervisor/conf.d/
+	fi
 fi
 
 rm -f /etc/bbfdm/dmmap/*

gitlab-ci/shared.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 
+BBFDM_TOOLS_INPUT_FILE="$(pwd)/tools/tools_input.json"
+BBFDM_PLUGIN_DEST="/opt/dev"
 BBFDM_MS_DIR="/usr/share/bbfdm/micro_services"
 BBFDM_MS_CONF="/etc/bbfdm/services"
 BBFDM_DMMAP_DIR="etc/bbfdm/dmmap/"
-BBFDM_LOG_FILE="/tmp/bbfdm.log"
 
 if [ -z "${CI_PROJECT_PATH}" ]; then
 	CI_PROJECT_PATH=${PWD}
@@ -55,10 +56,15 @@ function install_ms_plugin()
 	exec_cmd cp -f "${1}" ${BBFDM_MS_DIR}/${2}/
 }
 
+function install_ms_config()
+{
+	exec_cmd cp -f "${1}" ${BBFDM_MS_CONF}/${2}.json
+}
+
 function install_libbbf()
 {
-	# Enable coverage flags only for test
+	# Enable coverage flags only for bbfdm test
-	if [ -z "${1}" ]; then
+	if [ "$1" == "bbfdm" ]; then
 		COV_CFLAGS='-fprofile-arcs -ftest-coverage'
 		COV_LDFLAGS='--coverage'
 	fi
@@ -72,7 +78,27 @@ function install_libbbf()
 
 	mkdir -p build
 	cd build
-	cmake ../ -DCMAKE_C_FLAGS="$COV_CFLAGS " -DCMAKE_EXE_LINKER_FLAGS="$COV_LDFLAGS -lm" -DBBF_VENDOR_PREFIX="$VENDOR_PREFIX" -DBBF_MAX_OBJECT_INSTANCES=255 -DBBFDMD_MAX_MSG_LEN=1048576 -DCMAKE_INSTALL_PREFIX=/
+
+	# Construct the CMake command as an array for safety
+	cmake_args=(
+		../
+		-DCMAKE_C_FLAGS="$COV_CFLAGS"
+		-DCMAKE_EXE_LINKER_FLAGS="$COV_LDFLAGS -lm"
+		-DBBF_VENDOR_PREFIX="$VENDOR_PREFIX"
+		-DBBF_MAX_OBJECT_INSTANCES=255
+		-DBBFDMD_MAX_MSG_LEN=1048576
+		-DCMAKE_INSTALL_PREFIX=/
+	)
+
+	# Add this flag only if $1 is "tools"
+	if [ "$1" == "tools" ]; then
+		cmake_args+=(-DBBF_SCHEMA_FULL_TREE=ON)
+	fi
+
+	# Run cmake with arguments
+	cmake "${cmake_args[@]}"
+
+	# Compile and install
 	exec_cmd_verbose make
 
 	echo "installing libbbf"
@@ -80,7 +106,8 @@ function install_libbbf()
 	echo "371d530c95a17d1ca223a29b7a6cdc97e1135c1e0959b51106cca91a0b148b5e42742d372a359760742803f2a44bd88fca67ccdcfaeed26d02ce3b6049cb1e04" > /etc/bbfdm/.secure_hash
 	cd ..
 	exec_cmd cp utilities/bbf_configd /usr/sbin/
-	install_ms /usr/lib/libcore.so core
+	exec_cmd cp -f utilities/files/usr/share/bbfdm/scripts/bbf_api /usr/share/bbfdm/scripts/
+	install_ms build/libbbfdm/libcore.so core
 }
 
 function install_libbbf_test()
@@ -96,22 +123,22 @@ function install_libbbf_test()
 
 function install_wifidmd_as_micro_service()
 {
-	[ -d "/opt/dev/wifidmd" ] && return 0
+	[ -d "${BBFDM_PLUGIN_DEST}/wifidmd" ] && return 0
 
-	exec_cmd git clone https://dev.iopsys.eu/bbf/wifidmd.git /opt/dev/wifidmd
+	exec_cmd git clone https://dev.iopsys.eu/bbf/wifidmd.git ${BBFDM_PLUGIN_DEST}/wifidmd
 
-	exec_cmd make -C /opt/dev/wifidmd/src/ clean && make -C /opt/dev/wifidmd/src/ WIFIDMD_ENABLE_WIFI_DATAELEMENTS='y'
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/wifidmd/src/ clean && make -C ${BBFDM_PLUGIN_DEST}/wifidmd/src/ WIFIDMD_ENABLE_WIFI_DATAELEMENTS='y'
-	exec_cmd cp -f /opt/dev/wifidmd/src/wifidmd /usr/sbin/
+	exec_cmd cp -f ${BBFDM_PLUGIN_DEST}/wifidmd/src/wifidmd /usr/sbin/
 }
 
 function install_libeasy()
 {
-	[ -d "/opt/dev/libeasy" ] && return 0
+	[ -d "${BBFDM_PLUGIN_DEST}/libeasy" ] && return 0
 
-	exec_cmd git clone https://dev.iopsys.eu/iopsys/libeasy.git /opt/dev/libeasy
+	exec_cmd git clone https://dev.iopsys.eu/iopsys/libeasy.git ${BBFDM_PLUGIN_DEST}/libeasy
 	(
 
-		cd /opt/dev/libeasy
+		cd ${BBFDM_PLUGIN_DEST}/libeasy
 		exec_cmd make
 		sudo mkdir -p /usr/include/easy
 		sudo cp -a libeasy*.so* /usr/lib
@@ -121,12 +148,12 @@ function install_libeasy()
 
 function install_libqos()
 {
-	[ -d "/opt/dev/libqos" ] && return 0
+	[ -d "${BBFDM_PLUGIN_DEST}/libqos" ] && return 0
 
-	exec_cmd git clone https://dev.iopsys.eu/hal/libqos.git /opt/dev/libqos
+	exec_cmd git clone https://dev.iopsys.eu/hal/libqos.git ${BBFDM_PLUGIN_DEST}/libqos
 	(
 
-		cd /opt/dev/libqos
+		cd ${BBFDM_PLUGIN_DEST}/libqos
 		exec_cmd make
 		sudo mkdir -p /usr/include/
 		sudo cp -a libqos*.so* /usr/lib/
@@ -136,11 +163,11 @@ function install_libqos()
 
 function install_libethernet()
 {
-	[ -d "/opt/dev/libethernet" ] && return 0
+	[ -d "${BBFDM_PLUGIN_DEST}/libethernet" ] && return 0
 
-	exec_cmd git clone https://dev.iopsys.eu/iopsys/libethernet.git /opt/dev/libethernet
+	exec_cmd git clone https://dev.iopsys.eu/iopsys/libethernet.git ${BBFDM_PLUGIN_DEST}/libethernet
 	(
-		 cd /opt/dev/libethernet
+		 cd ${BBFDM_PLUGIN_DEST}/libethernet
 		 make PLATFORM=TEST
 		 sudo cp ethernet.h /usr/include
 		 sudo cp -a libethernet*.so* /usr/lib
@@ -150,56 +177,56 @@ function install_libethernet()
 
 function install_ethmngr_as_micro_service()
 {
-	[ -d "/opt/dev/ethmngr" ] && return 0
+	[ -d "${BBFDM_PLUGIN_DEST}/ethmngr" ] && return 0
 
 	install_libeasy
 	install_libethernet
 	install_libqos
 
-	exec_cmd git clone https://dev.iopsys.eu/hal/ethmngr.git /opt/dev/ethmngr
+	exec_cmd git clone https://dev.iopsys.eu/hal/ethmngr.git ${BBFDM_PLUGIN_DEST}/ethmngr
-	exec_cmd make -C /opt/dev/ethmngr
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/ethmngr
-	exec_cmd sudo cp -f /opt/dev/ethmngr/ethmngr /usr/sbin/ethmngr
+	exec_cmd sudo cp -f ${BBFDM_PLUGIN_DEST}/ethmngr/ethmngr /usr/sbin/ethmngr
 }
 
 function install_netmngr_as_micro_service()
 {
-	[ -d "/opt/dev/netmngr" ] && return 0
+	[ -d "${BBFDM_PLUGIN_DEST}/netmngr" ] && return 0
 
-	exec_cmd git clone -b devel https://dev.iopsys.eu/network/netmngr.git /opt/dev/netmngr
+	exec_cmd git clone -b devel https://dev.iopsys.eu/network/netmngr.git ${BBFDM_PLUGIN_DEST}/netmngr
 	
 	exec_cmd apt install iproute2 -y
 
-	exec_cmd make -C /opt/dev/netmngr/src/ clean
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/netmngr/src/ clean
-	exec_cmd make -C /opt/dev/netmngr/src/ NETMNGR_GRE_OBJ=y NETMNGR_IP_OBJ=y NETMNGR_ROUTING_OBJ=y NETMNGR_PPP_OBJ=y NETMNGR_ROUTER_ADVERTISEMENT_OBJ=y NETMNGR_IPV6RD_OBJ=y
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/netmngr/src/ NETMNGR_GRE_OBJ=y NETMNGR_IP_OBJ=y NETMNGR_ROUTING_OBJ=y NETMNGR_PPP_OBJ=y NETMNGR_ROUTER_ADVERTISEMENT_OBJ=y NETMNGR_IPV6RD_OBJ=y
-	install_ms /opt/dev/netmngr/src/libnetmngr.so netmngr
+	install_ms ${BBFDM_PLUGIN_DEST}/netmngr/src/libnetmngr.so netmngr
 
-	exec_cmd git clone https://dev.iopsys.eu/bbf/tr143d.git /opt/dev/tr143d
+	exec_cmd git clone https://dev.iopsys.eu/bbf/tr143d.git ${BBFDM_PLUGIN_DEST}/tr143d
-	exec_cmd make -C /opt/dev/tr143d/src/ clean && make -C /opt/dev/tr143d/src/
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/tr143d/src/ clean && make -C ${BBFDM_PLUGIN_DEST}/tr143d/src/
 	exec_cmd cp -f utilities/files/usr/share/bbfdm/scripts/bbf_api /usr/share/bbfdm/scripts/
-	exec_cmd cp -rf /opt/dev/tr143d/scripts/* /usr/share/bbfdm/scripts/
+	exec_cmd cp -rf ${BBFDM_PLUGIN_DEST}/tr143d/scripts/* /usr/share/bbfdm/scripts/
-	install_ms_plugin /opt/dev/tr143d/src/libtr143d.so netmngr
+	install_ms_plugin ${BBFDM_PLUGIN_DEST}/tr143d/src/libtr143d.so netmngr
 
-	exec_cmd git clone https://dev.iopsys.eu/bbf/tr471d.git /opt/dev/tr471d
+	exec_cmd git clone https://dev.iopsys.eu/bbf/tr471d.git ${BBFDM_PLUGIN_DEST}/tr471d
-	exec_cmd make -C /opt/dev/tr471d/src/ clean && make -C /opt/dev/tr471d/src/
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/tr471d/src/ clean && make -C ${BBFDM_PLUGIN_DEST}/tr471d/src/
-	install_ms_plugin /opt/dev/tr471d/src/libtr471d.so netmngr
+	install_ms_plugin ${BBFDM_PLUGIN_DEST}/tr471d/src/libtr471d.so netmngr
 
-	exec_cmd git clone https://dev.iopsys.eu/bbf/twamp-light.git /opt/dev/twamp
+	exec_cmd git clone https://dev.iopsys.eu/bbf/twamp-light.git ${BBFDM_PLUGIN_DEST}/twamp
-	exec_cmd make -C /opt/dev/twamp clean && make -C /opt/dev/twamp
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/twamp clean && make -C ${BBFDM_PLUGIN_DEST}/twamp
-	install_ms_plugin /opt/dev/twamp/libtwamp.so netmngr
+	install_ms_plugin ${BBFDM_PLUGIN_DEST}/twamp/libtwamp.so netmngr
 
-	exec_cmd git clone https://dev.iopsys.eu/bbf/udpecho.git /opt/dev/udpecho
+	exec_cmd git clone https://dev.iopsys.eu/bbf/udpecho.git ${BBFDM_PLUGIN_DEST}/udpecho
-	exec_cmd make -C /opt/dev/udpecho/src/ clean && make -C /opt/dev/udpecho/src/
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/udpecho/src/ clean && make -C ${BBFDM_PLUGIN_DEST}/udpecho/src/
-	install_ms_plugin /opt/dev/udpecho/src/libudpechoserver.so netmngr
+	install_ms_plugin ${BBFDM_PLUGIN_DEST}/udpecho/src/libudpechoserver.so netmngr
 }
 
 function install_sysmngr_as_micro_service()
 {
-	[ -d "/opt/dev/sysmngr" ] && return 0
+	[ -d "${BBFDM_PLUGIN_DEST}/sysmngr" ] && return 0
 
-	exec_cmd git clone -b devel https://dev.iopsys.eu/system/sysmngr.git /opt/dev/sysmngr
+	exec_cmd git clone -b devel https://dev.iopsys.eu/system/sysmngr.git ${BBFDM_PLUGIN_DEST}/sysmngr
 
-	exec_cmd make -C /opt/dev/sysmngr/src/ clean && \
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/sysmngr/src/ clean && \
-	exec_cmd make -C /opt/dev/sysmngr/src/ \
+	exec_cmd make -C ${BBFDM_PLUGIN_DEST}/sysmngr/src/ \
 		CFLAGS+="-DBBF_VENDOR_PREFIX=\\\"X_IOWRT_EU_\\\"" \
 		SYSMNGR_VENDOR_CONFIG_FILE='y' \
 		SYSMNGR_MEMORY_STATUS='y' \
@@ -211,7 +238,7 @@ function install_sysmngr_as_micro_service()
 		SYSMNGR_VENDOR_EXTENSIONS='y' \
 		SYSMNGR_FWBANK_UBUS_SUPPORT='y'
 
-	exec_cmd cp -f /opt/dev/sysmngr/src/sysmngr /usr/sbin/
+	exec_cmd cp -f ${BBFDM_PLUGIN_DEST}/sysmngr/src/sysmngr /usr/sbin/
 	exec_cmd mkdir /etc/sysmngr
 }
 
@@ -239,15 +266,14 @@ function generate_report()
 
 function install_cmph()
 {
-	[ -d "/opt/dev/cmph" ] && return 0
+	[ -d "${BBFDM_PLUGIN_DEST}/cmph" ] && return 0
 
-	exec_cmd git clone https://git.code.sf.net/p/cmph/git /opt/dev/cmph
+	exec_cmd git clone https://git.code.sf.net/p/cmph/git ${BBFDM_PLUGIN_DEST}/cmph
 	(
-		cd /opt/dev/cmph
+		cd ${BBFDM_PLUGIN_DEST}/cmph
 		exec_cmd autoreconf -i
 		exec_cmd ./configure
 		exec_cmd make
 		exec_cmd sudo make install
 	)
 }
-

gitlab-ci/tools-test.sh

@@ -32,7 +32,7 @@ for plugin in $(ls -1 test/vendor_test/*); do
 	check_ret $?
 done
 
-echo "Validate Data Model JSON Plugin after generating from TR-181, TR-104 and TR-135 XML Files"
+echo "Validate Data Model JSON Plugin after generating from TR-181 and TR-104 XML Files"
 json_path=$(./tools/convert_dm_xml_to_json.py -d test/tools/)
 ./tools/validate_json_plugin.py $json_path
 check_ret $?

libbbfdm-api/legacy/plugin/dotso_plugin.c

@@ -62,16 +62,9 @@ static void dotso_plugin_disable_requested_entries(DMOBJ *entryobj, DMOBJ *reque
 
 int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path)
 {
-#ifndef BBF_SCHEMA_FULL_TREE
 	void *handle = dlopen(plugin_path, RTLD_NOW|RTLD_LOCAL);
-#else
-	void *handle = dlopen(plugin_path, RTLD_LAZY);
-#endif
 	if (!handle) {
 		char *err_msg = dlerror();
-#ifdef BBF_SCHEMA_FULL_TREE
-		TRACE_FILE("Failed to add DotSo plugin '%s', [%s]\n", plugin_path, err_msg);
-#endif
 		BBF_ERR("Failed to add DotSo plugin '%s', [%s]\n", plugin_path, err_msg);
 		return 0;
 	}
@@ -82,9 +75,6 @@ int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path)
 
 	if (dynamic_obj == NULL) {
 		dlclose(handle);
-#ifdef BBF_SCHEMA_FULL_TREE
-		TRACE_FILE("Plugin %s missing init symbol ...\n", plugin_path);
-#endif
 		BBF_ERR("Plugin %s missing init symbol ...", plugin_path);
 		return 0;
 	}
@@ -93,9 +83,6 @@ int load_dotso_plugins(DMOBJ *entryobj, const char *plugin_path)
 
 		DMOBJ *dm_entryobj = find_entry_obj(entryobj, dynamic_obj[i].path);
 		if (!dm_entryobj) {
-#ifdef BBF_SCHEMA_FULL_TREE
-			TRACE_FILE("Failed to add DotSo plugin '%s' to main tree with parent DM index '%d' => '%s'", plugin_path, i, dynamic_obj[i].path);
-#endif
 			BBF_ERR("Failed to add DotSo plugin '%s' to main tree with parent DM index '%d' => '%s'", plugin_path, i, dynamic_obj[i].path);
 			continue;
 		}

libbbfdm-ubus/CMakeLists.txt

@@ -11,12 +11,6 @@ IF(${BBFDMD_MAX_MSG_LEN})
 	ADD_DEFINITIONS(-DBBFDM_MAX_MSG_LEN=${BBFDMD_MAX_MSG_LEN})
 ENDIF()
 
-OPTION(BBF_SCHEMA_FULL_TREE "build with schema full tree" OFF)
-
-IF(BBF_SCHEMA_FULL_TREE)
-	add_compile_definitions(BBF_SCHEMA_FULL_TREE)
-ENDIF(BBF_SCHEMA_FULL_TREE)
-
 FILE(GLOB BBF_UBUS_SOURCES *.c)
 
 ADD_LIBRARY(bbfdm-ubus SHARED ${BBF_UBUS_SOURCES})

libbbfdm-ubus/bbfdm-ubus.c

@@ -259,10 +259,6 @@ static int bbfdm_schema_handler(struct ubus_context *ctx, struct ubus_object *ob
 	data.bbf_ctx.isinfo = (dm_type == BBFDM_CWMP) ? false : true;
 	data.plist = &paths_list;
 
-#ifdef BBF_SCHEMA_FULL_TREE
-	data.bbf_ctx.isinfo = true;
-	bbfdm_get(&data, BBF_SCHEMA);
-#else
 	if (dm_type == BBFDM_CWMP) {
 		char *service_name = strdup(u->config.service_name);
 		data.bbf_ctx.in_value = (dm_type == BBFDM_CWMP) ? service_name : NULL;
@@ -271,7 +267,6 @@ static int bbfdm_schema_handler(struct ubus_context *ctx, struct ubus_object *ob
 	} else {
 		bbfdm_get(&data, BBF_SCHEMA);
 	}
-#endif
 
 	free_path_list(&paths_list);
 	return 0;

test/files/etc/bbfdm/services/core.json

@@ -19,7 +19,20 @@
       },
       {
         "parent_dm": "Device.",
-        "object": "GatewayInfo"
+        "object": "PacketCaptureDiagnostics"
+      },
+      {
+        "parent_dm": "Device.",
+        "object": "SelfTestDiagnostics"
+      },
+      {
+        "parent_dm": "Device.",
+        "object": "Syslog"
+      },
+      {
+        "parent_dm": "Device.",
+        "object": "{BBF_VENDOR_PREFIX}OpenVPN",
+        "proto": "usp"
       },	  
       {
         "parent_dm": "Device.",
@@ -28,11 +41,11 @@
       {
         "parent_dm": "Device.",
         "object": "Reboot()"
-        },
+      },
-        {
+      {
-          "parent_dm": "Device.",
+        "parent_dm": "Device.",
-          "object": "FactoryReset()"
+        "object": "FactoryReset()"
-        }
+      }
     ],
     "config": {
       "loglevel": "3"

test/tools/dm-cli.c

@@ -1,126 +0,0 @@
-#include <stdio.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <libubox/blobmsg.h>
-
-#include <libbbfdm-api/dmapi.h>
-#include <libbbfdm-ubus/bbfdm-ubus.h>
-#include <libbbfdm-api/dmentry.h>
-
-#include "../../libbbfdm-ubus/plugin.h"
-#include "../../libbbfdm/device.h"
-
-extern struct list_head loaded_json_files;
-extern struct list_head json_list;
-extern struct list_head json_memhead;
-
-static int cli_exec_schema(struct dmctx *bbfdm_ctx, char *in_path)
-{
-	int err = 0;
-
-	bbfdm_ctx->in_param = in_path;
-	bbfdm_ctx->nextlevel = false;
-	bbfdm_ctx->iscommand = true;
-	bbfdm_ctx->isevent = true;
-	bbfdm_ctx->isinfo = true;
-
-	err = bbf_entry_method(bbfdm_ctx, BBF_SCHEMA);
-	if (!err) {
-		struct blob_attr *cur = NULL;
-		size_t rem = 0;
-
-		blobmsg_for_each_attr(cur, bbfdm_ctx->bb.head, rem) {
-			struct blob_attr *tb[3] = {0};
-			const struct blobmsg_policy p[3] = {
-					{ "path", BLOBMSG_TYPE_STRING },
-					{ "data", BLOBMSG_TYPE_STRING },
-					{ "type", BLOBMSG_TYPE_STRING }
-			};
-
-			blobmsg_parse(p, 3, tb, blobmsg_data(cur), blobmsg_len(cur));
-
-			char *name = (tb[0]) ? blobmsg_get_string(tb[0]) : "";
-			char *data = (tb[1]) ? blobmsg_get_string(tb[1]) : "";
-			char *type = (tb[2]) ? blobmsg_get_string(tb[2]) : "";
-
-			printf("%s %s %s\n", name, type, strlen(data) ? data : "0"); // Added a data check to handle events with empty or missing data
-		}
-	} else {
-		printf("ERROR: %d retrieving %s\n", err, bbfdm_ctx->in_param);
-		err = -1;
-	}
-
-	return err;
-}
-
-int main(int argc, char **argv)
-{
-	DMOBJ *CLI_DM_ROOT_OBJ = NULL;
-	void *cli_lib_handle = NULL;
-	struct dmctx bbfdm_ctx = {0};
-	char *plugin_path = NULL, *plugin_dir = NULL, *dm_path = NULL;
-	unsigned int proto = BBFDM_BOTH;
-	int err = 0, ch;
-
-	memset(&bbfdm_ctx, 0, sizeof(struct dmctx));
-
-	while ((ch = getopt(argc, argv, "hc:u:l:p:")) != -1) {
-		switch (ch) {
-		case 'c':
-			bbfdm_ctx.dm_type = BBFDM_CWMP;
-			dm_path = argv[optind - 1];
-			break;
-		case 'u':
-			bbfdm_ctx.dm_type = BBFDM_USP;
-			dm_path = argv[optind - 1];
-			break;
-		case 'l':
-			plugin_path = optarg;
-			break;
-		case 'p':
-			plugin_dir = optarg;
-			break;
-		default:
-			break;
-		}
-	}
-
-	if (plugin_path == NULL) {
-		err = bbfdm_load_internal_plugin(NULL, tDynamicObj, &CLI_DM_ROOT_OBJ);
-	} else {
-		if (strstr(plugin_path, ".json") != NULL)
-			err = bbfdm_load_json_plugin(NULL, &loaded_json_files, &json_list, &json_memhead, plugin_path, &CLI_DM_ROOT_OBJ);
-		else
-			err = bbfdm_load_dotso_plugin(NULL, &cli_lib_handle, plugin_path, &CLI_DM_ROOT_OBJ);
-	}
-
-	if (err || !CLI_DM_ROOT_OBJ) {
-		printf("ERROR: Failed to load plugin\n");
-		return -1;
-	}
-
-	if (!dm_path) {
-		printf("ERROR: Data Model path should be defined\n");
-		return -1;
-	}
-
-	// Initialize global context
-	bbf_global_init(CLI_DM_ROOT_OBJ, plugin_dir);
-
-	// Initialize the bbfdm context
-	bbf_ctx_init(&bbfdm_ctx, CLI_DM_ROOT_OBJ);
-
-	err = cli_exec_schema(&bbfdm_ctx, dm_path);
-
-	// Clean up the context and global resources
-	bbf_ctx_clean(&bbfdm_ctx);
-	bbf_global_clean(CLI_DM_ROOT_OBJ);
-
-	// Free plugin handle
-	bbfdm_free_dotso_plugin(NULL, &cli_lib_handle);
-
-	// Free JSON plugin handle
-	bbfdm_free_json_plugin();
-
-	return err;
-}

tools/bbf_common.py

@@ -1,19 +1,16 @@
 #!/usr/bin/python3
 
-# Copyright (C) 2024 iopsys Software Solutions AB
+# Copyright (C) 2024-2025 iopsys Software Solutions AB
 # Author: Amin Ben Romdhane <amin.benromdhane@iopsys.eu>
 
 import sys
 import os
 import subprocess
 import shutil
-import glob
 
 # Constants
 BBF_ERROR_CODE = 0
 CURRENT_PATH = os.getcwd()
-BBF_MS_CORE_DIR = "/usr/share/bbfdm/micro_services/core/"
-BBF_MS_DIR = "/usr/share/bbfdm/micro_services/"
 
 DM_JSON_FILE = os.path.join(CURRENT_PATH, "tools", "datamodel.json")
 
@@ -35,25 +32,6 @@ Array_Types = {
 }
 
 
-def rename_file(old_path, new_path):
-    try:
-        os.rename(old_path, new_path)
-    except OSError:
-        pass
-
-
-def move_file(source_path, destination_path):
-    shutil.move(source_path, destination_path)
-
-
-def install_json_plugin(source_path, destination_path, vendor_extn):
-    with open(source_path, 'r', encoding='UTF-8') as src, open(destination_path, 'w', encoding='UTF-8') as dest:
-        data = src.read()
-        data = data.replace("{BBF_VENDOR_PREFIX}", vendor_extn)
-
-        dest.write(data)
-
-
 def remove_file(file_path):
     try:
         os.remove(file_path)
@@ -73,13 +51,6 @@ def remove_folder(folder_path):
         shutil.rmtree(folder_path)
 
 
-def cd_dir(path):
-    try:
-        os.chdir(path)
-    except OSError:
-        pass
-
-
 def obj_has_child(value):
     if isinstance(value, dict):
         for _obj, val in value.items():
@@ -138,146 +109,37 @@ def is_proto_exist(value, proto):
     return proto in protocols
 
 
-def clear_list(input_list):
+def build_command(plugin, proto):
-    input_list.clear()
+    service_name = get_option_value(plugin, "service_name")
+    unified = get_option_value(plugin, "unified_daemon", False)
+    daemon_name = get_option_value(plugin, "daemon_name", "")
 
+    if not service_name:
+        return None  # skip this plugin
 
-def generate_shared_library(dm_name, source_files, vendor_prefix,
+    if unified:
-                            extra_dependencies, is_microservice=False):
+        base_cmd = f"{daemon_name}"
-    # Return if source_files (list) is empty
-    if len(source_files) == 0:
-        return
-
-    if is_microservice:
-        outdir = BBF_MS_DIR
     else:
-        outdir = BBF_MS_CORE_DIR
+        base_cmd = f"dm-service -m {service_name}"
 
-    output_library = outdir + dm_name
-
-    # Set vendor prefix
-    if vendor_prefix is not None:
-        VENDOR_PREFIX = vendor_prefix
-    else:
-        VENDOR_PREFIX = "X_IOWRT_EU_"
-
-    # Ensure that the source files exist
-    for source_file in source_files:
-        if not os.path.exists(source_file):
-            print(f"     Error: Source file {source_file} does not exist.")
-            return False
-
-    cmd = ['gcc', '-shared', '-o', output_library, '-fPIC',
-           '-DBBF_VENDOR_PREFIX=\\"{}\\"'.format(VENDOR_PREFIX)]
-    cmd = cmd + source_files + extra_dependencies
-    # Compile the shared library
-    try:
-        cmdstr = ' '.join(str(e) for e in cmd)
-        subprocess.run(cmdstr, shell=True, check=True)
-        print(f"     Shared library {output_library} successfully created.")
-        return True
-    except subprocess.CalledProcessError as e:
-        print(f"     Error during compilation: {e}")
-        sys.exit(-1)
-
-
-def build_and_install_bbfdm(vendor_prefix):
-    print("Compiling and installing bbfdmd in progress ...")
-
-    create_folder(os.path.join(CURRENT_PATH, "build"))
-    cd_dir(os.path.join(CURRENT_PATH, "build"))
-
-    # Set vendor prefix
-    if vendor_prefix is not None:
-        VENDOR_PREFIX = vendor_prefix
-    else:
-        VENDOR_PREFIX = "X_IOWRT_EU_"
-
-    # Build and install bbfdm
-    cmake_command = [
-        "cmake",
-        "../",
-        "-DBBF_SCHEMA_FULL_TREE=ON",
-        f"-DBBF_VENDOR_PREFIX={VENDOR_PREFIX}",
-        "-DBBF_MAX_OBJECT_INSTANCES=255",
-        "-DBBFDMD_MAX_MSG_LEN=1048576",
-        "-DCMAKE_INSTALL_PREFIX=/"
-    ]
-    make_command = ["make"]
-    make_install_command = ["sudo", "make", "install"]
-
-    try:
-        subprocess.check_call(cmake_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        subprocess.check_call(make_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        subprocess.check_call(make_install_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    except subprocess.CalledProcessError as e:
-        print(f"Error running commands: {e}")
-        sys.exit(1)
-
-    cd_dir(CURRENT_PATH)
-    remove_folder(os.path.join(CURRENT_PATH, "build"))
-    print('Compiling and installing bbfdmd done')
-
-
-def build_and_install_dmcli():
-    print("Compiling and installing dm-cli in progress ...")
-
-    create_folder(os.path.join(CURRENT_PATH, "build"))
-    cd_dir(os.path.join(CURRENT_PATH, "build"))
-
-    # GCC command to compile dm-cli
-    gcc_command = [
-        "gcc",
-        "../test/tools/dm-cli.c",
-        "-lbbfdm-api",
-        "-lbbfdm-ubus",
-        "-lubox",
-        "-lblobmsg_json",
-        "-lcore",
-        "-ljson-c",
-        "-lssl",
-        "-lcrypto",
-        "-o", "dm-cli"
-    ]
-
-    try:
-        subprocess.check_call(gcc_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        subprocess.check_call(["sudo", "mv", "dm-cli", "/usr/sbin/dm-cli"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    except subprocess.CalledProcessError as e:
-        print(f"Error running commands: {e}")
-        sys.exit(1)
-
-    cd_dir(CURRENT_PATH)
-    remove_folder(os.path.join(CURRENT_PATH, "build"))
-    print('Compiling and installing dm-cli done')
-
-
-def fill_list_dm(proto, dm_list, dm_name=None):
-    # Determine the base command depending on the presence of dm_name
-    if dm_name:
-        command = f"dm-cli -l {dm_name}"
-    else:
-        command = "dm-cli -p /usr/share/bbfdm/micro_services/core"
-
-    # Add the appropriate flag (-c or -u) based on the proto value
     if proto == "cwmp":
-        command += " -c Device."
+        base_cmd += " -d"
     elif proto == "usp":
-        command += " -u Device."
+        base_cmd += " -dd"
 
+    return base_cmd
+
+
+def fill_list_dm(command, dm_list):
     try:
-        # Run the command
         result = subprocess.run(command, shell=True, text=True, capture_output=True, check=True)
-
-        # Get the output from the result
         output = result.stdout
-
-        # Split the output into lines
         lines = output.strip().split('\n')
 
-        # Iterate through each line and parse the information
         for line in lines:
             parts = line.split()
+            if len(parts) < 3:
+                continue
             path, n_type, data = parts[0], parts[1], parts[2]
             permission = "readWrite" if data == "1" else "readOnly"
             p_type = n_type[4:]
@@ -289,8 +151,7 @@ def fill_list_dm(proto, dm_list, dm_name=None):
             dm_list.append(entry)
 
     except subprocess.CalledProcessError as e:
-        # Handle subprocess errors here
+        print(f"Error running command '{command}': {e}")
-        print(f"Error running command: {e}")
         sys.exit(1)
 
 
@@ -307,154 +168,30 @@ def remove_duplicate_elements(input_list):
     return result_list
 
 
-def fill_list_supported_dm():
+def fill_list_supported_dm(plugins):
-    for proto, DB in [("usp", LIST_SUPPORTED_USP_DM), ("cwmp", LIST_SUPPORTED_CWMP_DM)]:
-        fill_list_dm(proto, DB)
-        DB.sort(key=lambda x: x['param'], reverse=False)
-        DB[:] = remove_duplicate_elements(DB)
-
-    for file in os.listdir(BBF_MS_DIR):
-        f = os.path.join(BBF_MS_DIR, file)
-
-        if os.path.isfile(f):
-            for proto, DB in [("usp", LIST_SUPPORTED_USP_DM), ("cwmp", LIST_SUPPORTED_CWMP_DM)]:
-                fill_list_dm(proto, DB, f)
-                DB.sort(key=lambda x: x['param'], reverse=False)
-                DB[:] = remove_duplicate_elements(DB)
-
-
-def clone_git_repository(repo, version=None):
-    repo_path = '/tmp/repo/'+os.path.basename(repo).replace('.git', '')
-    if os.path.exists(repo_path):
-        print(f'    {repo} already exists at {repo_path} !')
-        return True
-    try:
-        cmd = ["git", "clone", repo, repo_path]
-        if version is not None:
-            cmd.extend(["-b", version])
-        subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
-        return True
-    except (OSError, subprocess.SubprocessError):
-        print(f'    Failed to clone {repo} !!!!!')
-        return False
-
-
-def get_repo_version_info(repo, version=None):
-    if version is None:
-        return repo
-    return f'{repo}^{version}'
-
-
-def download_and_build_plugins(plugins, vendor_prefix):
-    global BBF_ERROR_CODE
 
     if plugins is None or not isinstance(plugins, list) or not plugins:
         print("No plugins provided.")
         return
 
-    print("Generating data models from defined plugins...")
+    for proto, DB in [("usp", LIST_SUPPORTED_USP_DM), ("cwmp", LIST_SUPPORTED_CWMP_DM)]:
+        for plugin in plugins:
+            command = build_command(plugin, proto)
+            if command:
+                print(f"Running command for {proto}: {command}")
+                fill_list_dm(command, DB)
 
-    remove_folder("/tmp/repo")
+        DB.sort(key=lambda x: x['param'], reverse=False)
-
+        DB[:] = remove_duplicate_elements(DB)
-    for plugin_index, plugin in enumerate(plugins):
-
-        repo = get_option_value(plugin, "repo")
-        proto = get_option_value(plugin, "proto")
-        dm_files = get_option_value(plugin, "dm_files")
-        is_microservice = get_option_value(plugin, "is_microservice")
-        extra_dependencies = get_option_value(plugin, "extra_dependencies", [])
-        dm_desc_file = get_option_value(plugin, "dm_info_file", "")
-        prefix = get_option_value(plugin, "vendor_prefix", None)
-        repo_path = None
-        name = os.path.basename(repo).replace('.git', '')
-
-        path = os.path.expanduser("~/.netrc")
-        if not os.path.isfile(path):
-            repo = repo.replace("https://dev.iopsys.eu/", "git@dev.iopsys.eu:")
-
-        print(f"## Repo is {repo}")
-        if not prefix:
-            prefix = vendor_prefix
-
-        if repo is None or proto is None or dm_files is None or not isinstance(dm_files, list):
-            BBF_ERROR_CODE += 1
-            print(f"# Necessary input missing {BBF_ERROR_CODE}")
-            continue
-
-        print(f' - Processing plugin: MS({is_microservice}) {plugin}')
-
-        if proto == "git":
-            repo_path = "/tmp/repo/"+name
-            version = get_option_value(plugin, "version")
-
-            if not clone_git_repository(repo, version):
-                BBF_ERROR_CODE += 1
-                print(f"# Failed to clone {repo} {BBF_ERROR_CODE}")
-                continue
-            print(f'    Processing {get_repo_version_info(repo, version)}')
-        elif proto == "local":
-            repo_path = repo
-            print(f'    Processing {get_repo_version_info(repo, proto)}')
-        if repo_path is None:
-            BBF_ERROR_CODE += 1
-            print(f"# Repository path not defined {BBF_ERROR_CODE}!!!")
-            continue
-
-        create_folder("/tmp/repo/dm_info")
-        if dm_desc_file.endswith('.json'):
-            dest_file = "/tmp/repo/dm_info/" + os.path.basename(dm_desc_file).replace('.json', f"_{plugin_index}.json")
-            rename_file(repo_path + "/" + dm_desc_file, dest_file)
-
-        LIST_FILES = []
-        os.chdir(repo_path)
-        for dm_file in dm_files:
-            filename = dm_file
-            if filename.endswith('*.c'):
-                LIST_FILES.extend(glob.glob(filename))
-            else:
-                if os.path.isfile(filename):
-                    if filename.endswith('.c'):
-                        LIST_FILES.append(filename)
-                    elif filename.endswith('.json'):
-                        if is_microservice is True:
-                            install_json_plugin(filename, "/usr/share/bbfdm/micro_services/"+f"{plugin_index}_{name}.json", prefix)
-                        else:
-                            install_json_plugin(filename, "/usr/share/bbfdm/micro_services/core/"+f"{plugin_index}_{name}.json", prefix)
-                    else:
-                        BBF_ERROR_CODE += 1
-                        print(f"# Unknown file format {filename} {BBF_ERROR_CODE}")
-                else:
-                    BBF_ERROR_CODE += 1
-                    print(f"# Error: File not accessible {filename} {BBF_ERROR_CODE}!!!!!!")
-
-        if len(LIST_FILES) > 0:
-            if not generate_shared_library(f"{plugin_index}_{name}.so", LIST_FILES, prefix, extra_dependencies, is_microservice):
-                BBF_ERROR_CODE += 1
-                print(f"# Error: Failed to generate shared library for {plugin_index}_{name}, error {BBF_ERROR_CODE}")
-
-        clear_list(LIST_FILES)
-        cd_dir(CURRENT_PATH)
-
-    print(f'Generating plugins completed, error {BBF_ERROR_CODE}')
 
 
-def generate_supported_dm(vendor_prefix=None, plugins=None):
+def generate_supported_dm(plugins=None):
     '''
     Generates supported data models and performs necessary actions.
 
     Args:
-        vendor_prefix (str, optional): Vendor prefix for shared libraries.
         plugins (list, optional): List of plugin configurations.
     '''
 
-    # Build && Install bbfdm
-    build_and_install_bbfdm(vendor_prefix)
-
-    # Build && Install dm-cli
-    build_and_install_dmcli()
-
-    # Download && Build Plugins Data Models
-    download_and_build_plugins(plugins, vendor_prefix)
-
     # Fill the list supported data model
-    fill_list_supported_dm()
+    fill_list_supported_dm(plugins)

tools/generate_dm.py

@@ -24,7 +24,6 @@ def print_dm_usage():
 if len(sys.argv) < 2:
     print_dm_usage()
 
-VENDOR_PREFIX = None
 PLUGINS = None
 OUTPUT = None
 DM_JSON_FILES = None
@@ -62,10 +61,6 @@ for option, value in json_data.items():
         bbf_xml.SOFTWARE_VERSION = value
         continue
 
-    elif option == "vendor_prefix":
-        VENDOR_PREFIX = value
-        continue
-
     elif option == "dm_json_files":
         DM_JSON_FILES = value
         continue
@@ -82,43 +77,39 @@ for option, value in json_data.items():
         print_dm_usage()
         exit(1)
 
-if OUTPUT is None:
+bbf.generate_supported_dm(PLUGINS)
-    bbf.download_and_build_plugins(PLUGINS, VENDOR_PREFIX)
-else:
-    bbf.generate_supported_dm(VENDOR_PREFIX, PLUGINS)
 
-    file_format = bbf.get_option_value(OUTPUT, "file_format", ['xml'])
+file_format = bbf.get_option_value(OUTPUT, "file_format", ['xml'])
-    output_file_prefix = bbf.get_option_value(OUTPUT, "output_file_prefix", "datamodel")
+output_file_prefix = bbf.get_option_value(OUTPUT, "output_file_prefix", "datamodel")
-    output_dir = bbf.get_option_value(OUTPUT, "output_dir", "./out")
+output_dir = bbf.get_option_value(OUTPUT, "output_dir", "./out")
 
-    bbf.create_folder(output_dir)
+bbf.create_folder(output_dir)
 
-    print("Dumping default DM_JSON_FILES")
+print("Dumping default DM_JSON_FILES")
-    print(DM_JSON_FILES)
+print(DM_JSON_FILES)
-    DM_JSON_FILES.extend(glob.glob('/tmp/repo/dm_info/*.json'))
+DM_JSON_FILES.extend(glob.glob('/tmp/desc_files/*.json'))
-    print("Dumping all")
+print("Dumping all")
-    print(DM_JSON_FILES)
+print(DM_JSON_FILES)
 
+if isinstance(file_format, list):
+    for _format in file_format:
 
-    if isinstance(file_format, list):
+        if _format == "xml":
-        for _format in file_format:
+            acs = bbf.get_option_value(OUTPUT, "acs", ['default'])
+            if isinstance(acs, list):
+                for acs_format in acs:
 
-            if _format == "xml":
+                    output_file_name = output_dir + '/' + output_file_prefix + '_' + acs_format + '.xml'
-                acs = bbf.get_option_value(OUTPUT, "acs", ['default'])
+                    if acs_format == "hdm":
-                if isinstance(acs, list):
+                        bbf_xml.generate_xml('HDM', DM_JSON_FILES, output_file_name)
-                    for acs_format in acs:
 
-                        output_file_name = output_dir + '/' + output_file_prefix + '_' + acs_format + '.xml'
+                    if acs_format == "default":
-                        if acs_format == "hdm":
+                        bbf_xml.generate_xml('default', DM_JSON_FILES, output_file_name)
-                            bbf_xml.generate_xml('HDM', DM_JSON_FILES, output_file_name)
 
-                        if acs_format == "default":
+        if _format == "xls":
-                            bbf_xml.generate_xml('default', DM_JSON_FILES, output_file_name)
+            output_file_name = output_dir + '/' + output_file_prefix + '.xls'
+            bbf_excel.generate_excel(output_file_name)
 
-            if _format == "xls":
+print("Datamodel generation completed, aritifacts shall be available in out directory or as per input json configuration")
-                output_file_name = output_dir + '/' + output_file_prefix + '.xls'
-                bbf_excel.generate_excel(output_file_name)
-    
-    print("Datamodel generation completed, aritifacts shall be available in out directory or as per input json configuration")
 
 sys.exit(bbf.BBF_ERROR_CODE)

tools/generate_dm_excel.py

@@ -196,13 +196,6 @@ if __name__ == '__main__':
         help= 'Includes OBJ/PARAM defined under remote repositories defined as bbf plugin'
     )
 
-    parser.add_argument(
-        '-p', '--vendor-prefix',
-		default = 'X_IOWRT_EU_',
-		metavar = 'X_IOWRT_EU_',
-		help = 'Generate data model tree using provided vendor prefix for vendor defined objects'
-    )
-
     parser.add_argument(
         '-o', '--output',
         default = "datamodel.xls",
@@ -225,7 +218,7 @@ if __name__ == '__main__':
 
             plugins.append(r)
 
-    bbf.generate_supported_dm(args.vendor_prefix, plugins)
+    bbf.generate_supported_dm(plugins)
     generate_excel(args.output)
     print(f'Datamodel generation completed, aritifacts available in {args.output}')
     sys.exit(bbf.BBF_ERROR_CODE)

tools/generate_dm_xml.py

@@ -394,13 +394,6 @@ if __name__ == '__main__':
         help= 'Includes OBJ/PARAM defined under remote repositories defined as bbf plugin'
     )
 
-    parser.add_argument(
-        '-p', '--vendor-prefix',
-		default = 'X_IOWRT_EU_',
-		metavar = 'X_IOWRT_EU_',
-		help = 'Generate data model tree using provided vendor prefix for vendor defined objects.'
-    )
-
     parser.add_argument(
         '-d', '--device-protocol',
 		default = 'DEVICE_PROTOCOL_DSLFTR069v1',
@@ -480,7 +473,7 @@ if __name__ == '__main__':
 
             plugins.append(r)
 
-    bbf.generate_supported_dm(args.vendor_prefix, plugins)
+    bbf.generate_supported_dm(plugins)
     generate_xml(args.format, args.dm_json_files, args.output)
     print(f'Datamodel generation completed, aritifacts available in {args.output}')
     sys.exit(bbf.BBF_ERROR_CODE)