bbfdm: 1.8.0

- New set of APIs to handle plugin installation
- Auto-generate datamodel micro-service plugin to minimize errors
- Added compaction of json datamodel plugin
- Added validation of json plugin while installing
- datamodel service name used in process name and in logging
This commit is contained in:
Vivek Kumar Dutta 2024-04-15 12:23:44 +05:30
parent 67a85ccf49
commit c3f99c0914
8 changed files with 330 additions and 216 deletions

View file

@ -5,13 +5,13 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=bbfdm
PKG_VERSION:=1.7.29
PKG_VERSION:=1.8.0
USE_LOCAL:=0
ifneq ($(USE_LOCAL),1)
PKG_SOURCE_PROTO:=git
PKG_SOURCE_URL:=https://dev.iopsys.eu/bbf/bbfdm.git
PKG_SOURCE_VERSION:=b4aebc9c6d92d63bf2788be53c94196e21590198
PKG_SOURCE_VERSION:=63fad00eeed1a7c181ef51be43174d92be4ad00f
PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION)-$(PKG_SOURCE_VERSION).tar.gz
PKG_MIRROR_HASH:=skip
endif
@ -107,7 +107,8 @@ define Package/libbbfdm/install
$(INSTALL_DIR) $(1)/etc/bbfdm/dmmap
$(INSTALL_DIR) $(1)/etc/uci-defaults
$(INSTALL_DIR) $(1)/lib/upgrade/keep.d
$(CP) $(PKG_BUILD_DIR)/libbbfdm/libbbfdm.so $(1)/lib/
$(INSTALL_DIR) $(1)/usr/share/bbfdm/
$(CP) $(PKG_BUILD_DIR)/libbbfdm/libbbfdm.so $(1)/usr/share/bbfdm/libbbfdm.so
$(INSTALL_DATA) ./files/lib/upgrade/keep.d/bbf $(1)/lib/upgrade/keep.d/bbf
$(INSTALL_BIN) ./files/etc/uci-defaults/95-portmap-firewall $(1)/etc/uci-defaults/95-portmap-firewall
$(INSTALL_BIN) ./files/etc/uci-defaults/97-firewall-service $(1)/etc/uci-defaults/97-firewall-service
@ -117,13 +118,13 @@ define Package/libbbfdm/install
$(INSTALL_BIN) ./files/etc/firewall.portmap $(1)/etc/firewall.portmap
$(INSTALL_BIN) ./files/etc/firewall.service $(1)/etc/firewall.service
ifeq ($(findstring iopsys,$(CONFIG_BBF_VENDOR_LIST)),iopsys)
$(call BbfdmInstallPlugin,$(1),$(PKG_BUILD_DIR)/libbbfdm/dmtree/vendor/iopsys/libbbfdm_iopsys_ext.so)
$(BBFDM_INSTALL_CORE_PLUGIN) $(PKG_BUILD_DIR)/libbbfdm/dmtree/vendor/iopsys/libbbfdm_iopsys_ext.so $(1)
endif
ifeq ($(CONFIG_BBF_TR143),y)
$(INSTALL_DIR) $(1)/usr/share/bbfdm
$(INSTALL_DIR) $(1)/usr/share/bbfdm/scripts/
$(INSTALL_DIR) $(1)/usr/libexec/rpcd
$(CP) $(PKG_BUILD_DIR)/libbbfdm/scripts/* $(1)/usr/share/bbfdm
$(LN) /usr/share/bbfdm/bbf.diag $(1)/usr/libexec/rpcd/bbf.diag
$(CP) $(PKG_BUILD_DIR)/libbbfdm/scripts/* $(1)/usr/share/bbfdm/scripts/
$(LN) /usr/share/bbfdm/scripts/bbf.diag $(1)/usr/libexec/rpcd/bbf.diag
endif
endef
@ -142,6 +143,7 @@ define Package/bbfdmd/install
$(INSTALL_BIN) $(PKG_BUILD_DIR)/bbfdmd/ubus/bbfdmd $(1)/usr/sbin/
$(INSTALL_DATA) ./files/etc/bbfdm/input.json $(1)/etc/bbfdm/
$(INSTALL_BIN) ./files/etc/init.d/bbfdmd $(1)/etc/init.d/bbfdmd
$(INSTALL_BIN) ./files/etc/init.d/bbfdm.services $(1)/etc/init.d/
$(INSTALL_CONF) ./files/etc/config/bbfdm $(1)/etc/config/bbfdm
$(INSTALL_BIN) ./files/etc/hotplug.d/iface/85-bbfdm-sysctl $(1)/etc/hotplug.d/iface/85-bbfdm-sysctl
endef

View file

@ -38,77 +38,3 @@ $ echo -n "Sup3rS3cur3Passw0rd" | sha512sum
bbfdm provides a helper utility [bbfdm.mk](./bbfdm.mk) to install datamodel plugins in bbfdm core or in microservice directory.
### Install datamodel DotSO/JSON plugin in bbfdm core
Utility to install the DotSO/JSON plugin in bbfdm core plugin path
```bash
# inputs:
# $1 => package install directory
# $2 => Plugin artifact
```
Example:
```make
define Package/$(PKG_NAME)/install
$(call BbfdmInstallPlugin,$(1),./files/etc/bbfdm/json/CWMPManagementServer.json)
endef
```
### Install datamodel DotSO/JSON plugin in bbfdm core with priority
Its now possible to overwrite/remove core datamodel with plugin, so, if some datamodel objects/parameters are present in more than one plugin, order in which they loaded into memory becomes crucial, this Utility help to configure a priority order in which they gets loaded in memory.
```bash
# inputs:
# $1 => package install directory
# $2 => Priority of the installed plugin
# $3 => Plugin artifact
```
Example:
```make
define Package/$(PKG_NAME)/install
$(call BbfdmInstallPluginWithPriority,$(1),01,$(PKG_BUILD_DIR)/bbf_plugin/bulkdata.json)
endef
```
> Note: Last loaded plugin gets the highest priority
### Install plugin into micro-service directory
Utility to install the plugin in datamodel microservice directory, ex. user wants to run a datamodel micro-service, it required to install the DotSO/JSON plugin into a non bbf core directory, this utility helps in installing the DotSO/JSON plugin in mentioned directory.
```bash
# inputs:
# $1 => package install directory with micro-service directory
# $2 => Plugin artifact
```
Example:
```make
define Package/$(PKG_NAME)/install
$(call BbfdmInstallPluginInMicroservice,$(1)/etc/bulkdata,$(PKG_BUILD_DIR)/bbf_plugin/bulkdata.json)
endef
```
### Install datamodel micro-service input file
Utility to install the datamodel plugin input file into the bbfdm micro-service directory, so that bbfdm auto start the datamodel micro-service before main bbfdm process.
```bash
# inputs:
# $1 => package install directory
# $2 => Microservice input.json path
```
Example:
```make
define Package/$(PKG_NAME)/install
$(call BbfdmInstallMicroServiceInputFile,$(1),./files/etc/bulkdata/input.json)
endef
```

View file

@ -2,56 +2,95 @@
# Copyright (C) 2023 IOPSYS
#
# Utility to install the plugin in bbfdm core path
# inputs:
# $1 => package install directory
# $2 => Plugin artifact
#
# Example:
# $(call BbfdmInstallPlugin,$(1),./files/etc/bbfdm/json/CWMPManagementServer.json)
#
define BbfdmInstallPlugin
$(INSTALL_DIR) $(1)/etc/bbfdm/plugins
$(INSTALL_DATA) $(2) $(1)/etc/bbfdm/plugins/
endef
BBFDM_BASE_DM_PATH=/usr/share/bbfdm
BBFDM_INPUT_PATH=/etc/bbfdm/micro_services
BBFDM_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
# Utility to install the plugin in bbfdm core path with priority
# inputs:
# $1 => package install directory
# $2 => Priority of the installed plugin
# $3 => Plugin artifact
#
# Example:
# $(call BbfdmInstallPluginWithPriority,$(1),01,$(PKG_BUILD_DIR)/bbf_plugin/bulkdata.json)
#
define BbfdmInstallPluginWithPriority
$(INSTALL_DIR) $(1)/etc/bbfdm/plugins
$(INSTALL_DATA) $(3) $(1)/etc/bbfdm/plugins/$(2)_$(shell basename ${3})
endef
BBFDM_VERSION:=$(shell grep -oP '(?<=^PKG_VERSION:=).*' ${BBFDM_DIR}/Makefile)
BBFDM_TOOLS:=$(BUILD_DIR)/bbfdm-$(BBFDM_VERSION)/tools
# Utility to install the plugin in datamodel microservice directory
# inputs:
# $1 => package install directory with micro-service directory
# $2 => Plugin artifact
# Utility to install the plugin in bbfdm core path with priority.
# Its now possible to overwrite/remove core datamodel with plugin, so, if some
# datamodel objects/parameters are present in more than one plugin, order in
# which they loaded into memory becomes crucial, this Utility help to configure
# a priority order in which they gets loaded in memory.
#
# ARGS:
# $1 => Plugin artifact
# $2 => package install directory
# $3 => Priority of the installed plugin (Optional)
#
# Note:
# - Last loaded plugin gets the highest priority
#
# Example:
# $(call BbfdmInstallPluginInMicroservice,$(1)/etc/bulkdata,$(PKG_BUILD_DIR)/bbf_plugin/bulkdata.json)
# BBFDM_INSTALL_CORE_PLUGIN ./files/etc/bbfdm/json/CWMPManagementServer.json $(1)
#
# Example to install plugin with priority:
# BBFDM_INSTALL_CORE_PLUGIN ./files/etc/bbfdm/json/CWMPManagementServer.json $(1) 01
#
BBFDM_INSTALL_CORE_PLUGIN:=$(BBFDM_DIR)/bbfdm.sh -t $(BBFDM_TOOLS) -p
# Utility to install the micro-service datamodel
# Use Case:
# user wants to run a datamodel micro-service, it required to install the
# DotSO/JSON plugin into a bbf shared directory, this utility helps in
# installing the DotSO/JSON plugin in bbfdm shared directory, and auto-generate
# input file for the micro-service
#
# ARGS:
# $1 => DotSo or Json plugin with complete path
# $2 => package install directory
# $3 => service name
#
# Note:
# - There could be only one main plugin file, so its bind to PKG_NAME
# - Micro-service input.json will be auto generated with this call
#
# Example:
# BBFDM_INSTALL_MS_DM $(PKG_BUILD_DIR)/libcwmp.so $(1) $(PKG_NAME)
#
BBFDM_INSTALL_MS_DM:=$(BBFDM_DIR)/bbfdm.sh -t $(BBFDM_TOOLS) -m
# Utility to install a plugins in datamodel micro-service
#
# ARGS:
# $1 => DotSo or Json plugin with complete path
# $2 => package install directory
# $3 => service name
#
# Note:
# - Use the service_name/PKG_NAME of the service in which this has to run
#
# Example:
# BBFDM_INSTALL_MS_PLUGIN $(PKG_BUILD_DIR)/libxmpp.so $(1) icwmp
#
BBFDM_INSTALL_MS_PLUGIN:=$(BBFDM_DIR)/bbfdm.sh -t $(BBFDM_TOOLS) -m -p
# Deprecated functions errors
define BbfdmInstallPluginInMicroservice
$(warning # BbfdmInstallPluginInMicroservice function is deprecated, use BBFDM_INSTALL_MS_PLUGIN macro #)
$(INSTALL_DIR) $(1)
$(INSTALL_DATA) $(2) $(1)/
endef
# Utility to install the datamodel plugin input file
# inputs:
# $1 => package install directory
# $2 => Microservice input.json path
#
# Example:
# $(call BbfdmInstallMicroServiceInputFile,$(1),./files/etc/bulkdata/input.json)
#
define BbfdmInstallMicroServiceInputFile
$(warning # function BbfdmInstallMicroServiceInputFile deprecated, input file auto generated with BBFDM_INSTALL_MS_DM #)
$(INSTALL_DIR) $(1)/etc/bbfdm/micro_services
$(INSTALL_DATA) $(2) $(1)/etc/bbfdm/micro_services/$(PKG_NAME).json
endef
define BbfdmInstallPlugin
$(warning # function BbfdmInstallPlugin deprecated, use BBFDM_INSTALL_CORE_PLUGIN macro #)
$(INSTALL_DIR) $(1)/etc/bbfdm/plugins
$(INSTALL_DATA) $(2) $(1)/etc/bbfdm/plugins/
endef
define BbfdmInstallPluginWithPriority
$(warning # fucntion BbfdmInstallPluginWithPriority deprecated, use BBFDM_INSTALL_CORE_PLUGIN #)
$(INSTALL_DIR) $(1)/etc/bbfdm/plugins
$(INSTALL_DATA) $(3) $(1)/etc/bbfdm/plugins/$(2)_$(shell basename ${3})
endef

134
bbfdm/bbfdm.sh Executable file
View file

@ -0,0 +1,134 @@
#!/bin/bash
BBFDM_BASE_DM_PATH="usr/share/bbfdm"
BBFDM_INPUT_PATH="etc/bbfdm/micro_services"
INPUT_TEMPLATE='{"daemon":{"service_name":"template","config":{"loglevel":"1"}}}'
MICRO_SERVICE=0
PLUGIN=0
DEST=""
TOOLS=""
SRC=""
while getopts ":t:mp" opt; do
case ${opt} in
t)
TOOLS="${OPTARG}"
;;
m)
MICRO_SERVICE=1
;;
p)
PLUGIN=1
;;
?)
echo "Invalid option: ${OPTARG}"
exit 1
;;
esac
done
shift $((OPTIND-1))
SRC="${1}"
shift
DEST="${1}"
shift
DATA="${1}"
install_bin() {
install -m0755 ${1} ${2}
}
install_dir() {
install -d -m0755 ${1}
}
install_data() {
install -m0644 ${1} ${2}
}
# Installing datamodel
bbfdm_install_dm()
{
local src dest priority minfile
src="$1"
dest="$2"
priority="${3}"
minfile=""
if [ -z ${src} ] || [ -z "${dest}" ] || [ -z "${TOOLS}" ]; then
echo "Invalid input option for install dm $@"
exit 1
fi
if [ "${src##*.}" = "json" ]; then
echo "Compacting BBFDM JSON file"
minfile=$(mktemp)
jq -c 'del(..|.description?)' ${src} > ${minfile}
src=${minfile}
if dpkg -s python3-jsonschema >/dev/null 2>&1; then
echo "Verifying bbfdm Datamodel JSON file"
if ! ${TOOLS}/validate_json_plugin.py ${src}; then
echo "Validation of the plugin failed ${src}"
exit 1
fi
else
echo "## Install python3-jsonschema to verify datamodel plugins"
fi
fi
if [ -n "${priority}" ]; then
install_bin ${src} ${dest}/${priority}_$(basename ${src})
else
install_bin ${src} ${dest}
fi
if [ -f "${minfile}" ]; then
rm ${minfile}
fi
}
bbfdm_generate_input()
{
local dest ser
dest_dir=${1}
ser=${2}
dest=${dest_dir}/${ser}.json
echo ${INPUT_TEMPLATE} | jq --arg service "$ser" '.daemon.service_name = $service' > ${dest}
chmod 466 ${dest}
}
if [ -z "$SRC" ] || [ -z "${DEST}" ] ; then
echo "# BBFDM Null value in src[${SRC}], dest[${DEST}]"
exit 1
fi
if [ "${MICRO_SERVICE}" -eq "1" ]; then
if [ -z "${DATA}" ]; then
echo "# service_name[${DATA}] not provided"
exit 1
fi
if [ "${PLUGIN}" -ne "1" ]; then
extn="$(basename ${SRC})"
install_dir ${DEST}/${BBFDM_BASE_DM_PATH}/micro_services
bbfdm_install_dm ${SRC} ${DEST}/${BBFDM_BASE_DM_PATH}/micro_services/${DATA}.${extn##*.}
# main micro-service datamodel plugin, create an input file as well
install_dir ${DEST}/${BBFDM_INPUT_PATH}
bbfdm_generate_input ${DEST}/${BBFDM_INPUT_PATH}/ ${DATA}
else
install_dir ${DEST}/${BBFDM_BASE_DM_PATH}/micro_services/${DATA}
bbfdm_install_dm ${SRC} ${DEST}/${BBFDM_BASE_DM_PATH}/micro_services/${DATA}/$(basename ${SRC})
fi
else
if [ "${PLUGIN}" -eq "1" ]; then
install_dir ${DEST}/${BBFDM_BASE_DM_PATH}/plugins
bbfdm_install_dm ${SRC} ${DEST}/${BBFDM_BASE_DM_PATH}/plugins/ ${DATA}
fi
fi

View file

@ -4,8 +4,8 @@
},
"input": {
"type": "DotSo",
"name": "/lib/libbbfdm.so",
"plugin_dir": "/etc/bbfdm/plugins"
"name": "/usr/share/bbfdm/libbbfdm.so",
"plugin_dir": "/usr/share/bbfdm/plugins"
},
"output": {
"type": "UBUS",

View file

@ -0,0 +1,106 @@
#!/bin/sh /etc/rc.common
START=40
STOP=8
USE_PROCD=1
PROG=/usr/sbin/bbfdmd
BBFDM_MICROSERVICE_DIR="/etc/bbfdm/micro_services"
. /usr/share/libubox/jshn.sh
log() {
echo "${@}"|logger -t bbfdmd.services -p info
}
validate_bbfdm_micro_service_section()
{
uci_validate_section bbfdm micro_services "micro_services" \
'enable:bool:true' \
'enable_core:bool:false' \
'enable_respawn:bool:true'
}
_add_microservice()
{
local name path
local enable enable_core enable_respawn
# Check enable from micro-service
path="${1}"
enable_respawn="${2}"
enable_core="${3}"
name="$(basename ${path})"
name="${name//.json}"
enable="$(jq '.daemon.enable//1' ${path})"
if [ "${enable}" -eq "0" ]; then
log "datamodel micro-service ${name} not enabled"
return 0
fi
procd_open_instance "${name}"
procd_set_param command ${PROG}
procd_append_param command -m "${name}"
if [ "${enable_core}" -eq "1" ]; then
procd_set_param limits core="unlimited"
procd_set_param stdout 1
procd_set_param stderr 1
fi
if [ "${enable_respawn}" -eq "1" ]; then
procd_set_param respawn "3600" "5" "5"
fi
procd_close_instance "${name}"
}
configure_bbfdm_micro_services()
{
local enable enable_core enable_respawn
config_load bbfdm
validate_bbfdm_micro_service_section || {
log "Validation of micro_service section failed"
return 1;
}
[ "${enable}" -eq "0" ] && return 0
if [ -d "${BBFDM_MICROSERVICE_DIR}" ]; then
FILES="$(ls -1 ${BBFDM_MICROSERVICE_DIR}/*.json)"
for file in $FILES;
do
[ -e "$file" ] || continue
_add_microservice $file "${enable_respawn}" "${enable_core}"
done
fi
}
_start_single_service()
{
local service file
service="${1}"
if [ -d "${BBFDM_MICROSERVICE_DIR}" ]; then
file="$(ls -1 ${BBFDM_MICROSERVICE_DIR}/${service}.json)"
[ -e "$file" ] || return
_add_microservice $file "0" "0"
fi
}
start_service()
{
if [ -n "${1}" ]; then
_start_single_service "${1}"
else
configure_bbfdm_micro_services
fi
}

View file

@ -7,7 +7,6 @@ USE_PROCD=1
PROG=/usr/sbin/bbfdmd
BBFDM_JSON_INPUT="/etc/bbfdm/input.json"
BBFDM_MICROSERVICE_DIR="/etc/bbfdm/micro_services"
BBFDM_TEMP_DIR="/tmp/bbfdm"
. /usr/share/libubox/jshn.sh
@ -28,91 +27,6 @@ validate_bbfdm_bbfdmd_section()
'subprocess_level:uinteger'
}
validate_bbfdm_micro_service_section()
{
uci_validate_section bbfdm micro_services "micro_services" \
'enable:bool:true' \
'enable_core:bool:false' \
'enable_respawn:bool:true'
}
bbfdm_add_micro_service()
{
local name path cmd
local enable enable_core enable_respawn
validate_bbfdm_micro_service_section || {
log "Validation of micro_service section failed"
return 1;
}
[ "${enable}" -eq "0" ] && return 0
path="${1}"
name="$(basename ${path})"
name="${name//.json}"
json_init
json_add_string name "bbfdm.services"
json_add_object "instances"
json_add_object "${name}"
json_add_array "command"
json_add_string "" "${PROG}"
json_add_string "" "-m"
json_add_string "" "${path}"
json_close_array
if [ "${enable_core}" -eq "1" ]; then
json_add_object "limits"
json_add_string "core" "unlimited"
json_close_object
json_add_boolean "stdout" 1
json_add_boolean "stderr" 1
fi
if [ "${enable_respawn}" -eq "1" ]; then
json_add_array "respawn"
json_add_string "" "3600"
json_add_string "" "5"
json_add_string "" "5"
json_close_array
fi
json_close_object
json_close_object
cmd="$(json_dump)"
ubus call service add "${cmd}"
}
_add_microservice()
{
local enable loglevel input_json name
name="${1}"
input_json="$(jq -r '.daemon.input.name' ${name})"
if [ -f "${input_json}" ]; then
bbfdm_add_micro_service "${name}"
else
log "Input json [${input_json}] does not defined/present"
fi
}
configure_bbfdm_micro_services()
{
if [ -d "${BBFDM_MICROSERVICE_DIR}" ]; then
FILES="$(ls -1 ${BBFDM_MICROSERVICE_DIR}/*.json)"
for service in $FILES;
do
[ -e "$service" ] || continue
_add_microservice $service
done
fi
ubus call service state '{"name":"bbfdm.services", "spawn":true}'
}
configure_bbfdmd()
{
local enable debug sock
@ -145,19 +59,12 @@ start_service()
{
mkdir -p ${BBFDM_TEMP_DIR}
configure_bbfdm_micro_services
procd_open_instance "bbfdm"
configure_bbfdmd
procd_set_param respawn
procd_close_instance "bbfdm"
}
stop_service()
{
ubus call service delete '{"name":"bbfdm.services"}'
}
service_triggers()
{
procd_add_reload_trigger "bbfdm"

View file

@ -1,6 +1,6 @@
#!/bin/sh
UNIFIED_PATH="/etc/bbfdm/plugins/"
UNIFIED_PATH="/usr/share/bbfdm/plugins/"
# Link JSON plugins
for f in `ls -1 /etc/bbfdm/json/*.json`; do