From e35bf1d50901c904efe88a56e6f876f968defe2b Mon Sep 17 00:00:00 2001 From: Husaam Mehdi Date: Tue, 5 Aug 2025 11:36:51 +0000 Subject: [PATCH] logmngr: add support for Syslog Source and Template --- logmngr/Config.in | 2 + logmngr/Makefile | 28 +- logmngr/files/etc/config/logmngr | 26 ++ .../{logmngr.init => etc/init.d/logmngr} | 0 .../uci-defaults/10-logmngr_config_migrate | 39 +++ .../uci-defaults/11-logmngr_logrotate_syslog} | 2 +- logmngr/files/lib/logmngr/fluent-bit.sh | 282 ++++++++++++------ logmngr/files/logmngr.uci | 15 - 8 files changed, 284 insertions(+), 110 deletions(-) create mode 100644 logmngr/files/etc/config/logmngr rename logmngr/files/{logmngr.init => etc/init.d/logmngr} (100%) create mode 100644 logmngr/files/etc/uci-defaults/10-logmngr_config_migrate rename logmngr/files/{11-logmngr_logrotate_config_generate => etc/uci-defaults/11-logmngr_logrotate_syslog} (84%) delete mode 100644 logmngr/files/logmngr.uci diff --git a/logmngr/Config.in b/logmngr/Config.in index 0474e0622..b598a78d4 100644 --- a/logmngr/Config.in +++ b/logmngr/Config.in @@ -8,6 +8,7 @@ choice config LOGMNGR_BACKEND_FLUENTBIT bool "Use fluent-bit for log management" + default y help Enable this option to use fluent-bit for log management. @@ -31,4 +32,5 @@ config LOGMNGR_VENDOR_LOG_FILE default y help It adds support for Device.DeviceInfo.VendorLogFile. Object. + endif diff --git a/logmngr/Makefile b/logmngr/Makefile index dc7828799..d1e1a4bfc 100644 --- a/logmngr/Makefile +++ b/logmngr/Makefile @@ -5,13 +5,13 @@ include $(TOPDIR)/rules.mk PKG_NAME:=logmngr -PKG_VERSION:=1.0.20 +PKG_VERSION:=1.1.0 LOCAL_DEV:=0 ifneq ($(LOCAL_DEV),1) PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://dev.iopsys.eu/system/logmngr.git -PKG_SOURCE_VERSION:=ad2636c642d56967e78c0c84bf82cb0e2b6311f2 +PKG_SOURCE_VERSION:=ad043d5d2e96c1b203a9c5b22ad6e6e1488ad8f2 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_MIRROR_HASH:=skip endif @@ -52,30 +52,36 @@ endif define Package/logmngr/install $(INSTALL_DIR) $(1)/etc/init.d - $(INSTALL_BIN) ./files/logmngr.init $(1)/etc/init.d/logmngr - $(INSTALL_DIR) $(1)/etc/config - $(INSTALL_BIN) ./files/logmngr.uci $(1)/etc/config/logmngr + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/etc/init.d/logmngr $(1)/etc/init.d/logmngr + $(INSTALL_BIN) ./files/etc/config/logmngr $(1)/etc/config/logmngr + + $(BBFDM_INSTALL_MS_PLUGIN) $(PKG_BUILD_DIR)/bbf_plugin/libbbfsyslog.so $(1) core 10 + + # Install logmngr service backend $(INSTALL_DIR) $(1)/lib/logmngr ifeq ($(CONFIG_LOGMNGR_BACKEND_FLUENTBIT),y) - $(INSTALL_DATA) ./files/lib/logmngr/fluent-bit.sh $(1)/lib/logmngr/ $(INSTALL_DIR) $(1)/sbin + $(INSTALL_BIN) ./files/logread $(1)/sbin/ -endif -ifeq ($(CONFIG_LOGMNGR_BACKEND_SYSLOG_NG),y) + $(INSTALL_DATA) ./files/lib/logmngr/fluent-bit.sh $(1)/lib/logmngr/ + $(INSTALL_BIN) ./files/etc/uci-defaults/10-logmngr_config_migrate $(1)/etc/uci-defaults/ +else ifeq ($(CONFIG_LOGMNGR_BACKEND_SYSLOG_NG),y) $(INSTALL_DATA) ./files/lib/logmngr/syslog-ng.sh $(1)/lib/logmngr/ endif - $(BBFDM_INSTALL_MS_PLUGIN) $(PKG_BUILD_DIR)/bbf_plugin/libbbfsyslog.so $(1) core 10 + ifeq ($(CONFIG_LOGMNGR_LOGROTATE),y) - $(INSTALL_DIR) $(1)/etc/uci-defaults - $(INSTALL_BIN) ./files/11-logmngr_logrotate_config_generate $(1)/etc/uci-defaults/ $(INSTALL_DATA) ./files/lib/logmngr/logrotate.sh $(1)/lib/logmngr/ + $(INSTALL_BIN) .//files/etc/uci-defaults/11-logmngr_logrotate_syslog $(1)/etc/uci-defaults/ $(BBFDM_INSTALL_MS_PLUGIN) $(PKG_BUILD_DIR)/bbf_plugin/libbbflogrotate.so $(1) sysmngr 11 endif + ifeq ($(CONFIG_LOGMNGR_VENDOR_LOG_FILE),y) $(BBFDM_INSTALL_MS_PLUGIN) $(PKG_BUILD_DIR)/bbf_plugin/libbbfvendorlog.so $(1) sysmngr 12 endif + endef $(eval $(call BuildPackage,logmngr)) diff --git a/logmngr/files/etc/config/logmngr b/logmngr/files/etc/config/logmngr new file mode 100644 index 000000000..d9cf6ee82 --- /dev/null +++ b/logmngr/files/etc/config/logmngr @@ -0,0 +1,26 @@ +config globals 'globals' + option enable '1' + +config source 'default_source' + option name 'default_source' + option system_messages '1' + option kernel_messages '1' + +config template 'default_template' + option name 'default_template' + option expression '{time} {hostname} {ident}: {message}' + +config action 'default_action' + option name 'default_action' + list source 'default_source' + option template 'default_template' + +config log_file 'lf1' + option enable '1' + option action 'default_action' + option file '/var/log/messages' + +config log_remote 'lr1' + option enable '0' + option action 'default_action' + option port '514' diff --git a/logmngr/files/logmngr.init b/logmngr/files/etc/init.d/logmngr similarity index 100% rename from logmngr/files/logmngr.init rename to logmngr/files/etc/init.d/logmngr diff --git a/logmngr/files/etc/uci-defaults/10-logmngr_config_migrate b/logmngr/files/etc/uci-defaults/10-logmngr_config_migrate new file mode 100644 index 000000000..50359a8c1 --- /dev/null +++ b/logmngr/files/etc/uci-defaults/10-logmngr_config_migrate @@ -0,0 +1,39 @@ +#!/bin/sh + +. /lib/functions.sh + +# check if this is a new type UCI or old type UCI +if ! uci -q get logmngr.default_source > /dev/null; then + uci -q set logmngr.default_source=source + uci -q set logmngr.default_source.name='default_source' + uci -q set logmngr.default_source.system_messages='1' + uci -q set logmngr.default_source.kernel_messages='1' +fi + + +if ! uci -q get logmngr.default_template > /dev/null; then + uci -q set logmngr.default_template=template + uci -q set logmngr.default_template.name='default_template' + uci -q set logmngr.default_template.expression='{time} {hostname} {ident}: {message}' +fi + +if uci -q get logmngr.a1 >/dev/null; then + uci -q rename logmngr.a1='default_action' + uci -q set logmngr.default_action.name='default_action' + uci -q set logmngr.default_action.template='default_template' + + uci -q delete logmngr.default_action.source + uci -q add_list logmngr.default_action.source='default_source' +fi + +if uci -q get logmngr.lf1 >/dev/null; then + uci -q rename logmngr.lf1='default_logfile' + uci -q set logmngr.default_logfile.action='default_action' +fi + +if uci -q get logmngr.lr1 >/dev/null; then + uci -q rename logmngr.lr1='default_logremote' + uci -q set logmngr.default_logremote.action='default_action' +fi + +exit 0 diff --git a/logmngr/files/11-logmngr_logrotate_config_generate b/logmngr/files/etc/uci-defaults/11-logmngr_logrotate_syslog similarity index 84% rename from logmngr/files/11-logmngr_logrotate_config_generate rename to logmngr/files/etc/uci-defaults/11-logmngr_logrotate_syslog index c2dd819d9..5e67b3e3e 100644 --- a/logmngr/files/11-logmngr_logrotate_config_generate +++ b/logmngr/files/etc/uci-defaults/11-logmngr_logrotate_syslog @@ -1,7 +1,7 @@ #!/bin/sh # Adds a default log rotate policy if none exists -if uci -q get logmngr.@log_rotate[0] >/dev/null; then +if uci -q get logmngr.lro1 >/dev/null; then # return if there is any valid content exit 0 fi diff --git a/logmngr/files/lib/logmngr/fluent-bit.sh b/logmngr/files/lib/logmngr/fluent-bit.sh index cda7b7f1c..fe6e5ec31 100644 --- a/logmngr/files/lib/logmngr/fluent-bit.sh +++ b/logmngr/files/lib/logmngr/fluent-bit.sh @@ -6,6 +6,37 @@ CONF_FILE=/etc/fluent-bit/fluent-bit.conf TMP_CONF_FILE=/tmp/fluent-bit/fluent-bit.conf FLUENT_BIT_CONF_DIR=/etc/fluent-bit/conf.d +PROCESSED_SYSLOG_TAGS="" +PROCESSED_KMSG_TAGS="" + +# check if syslog source section is already processed +# and add it to the list of processed source sections +syslog_tag_already_processed() { + local tag="$1" + + for t in $PROCESSED_SYSLOG_TAGS; do + [ "$t" = "$tag" ] && return 0 + done + + PROCESSED_SYSLOG_TAGS="$tag $PROCESSED_SYSLOG_TAGS" + + return 1 +} + +# check if kmsg source section is already processed +# and add it to the list of processed source sections +# two separate functions used because we want to populate +# appropriate PROCESSED variable +kmsg_tag_already_processed() { + local tag="$1" + for t in $PROCESSED_KMSG_TAGS; do + [ "$t" = "$tag" ] && return 0 + done + + PROCESSED_KMSG_TAGS="$tag $PROCESSED_KMSG_TAGS" + + return 1 +} append_conf() { echo "$*" >> ${TMP_CONF_FILE} @@ -20,52 +51,56 @@ create_config_file() { # also, if no file is found then fluent-bit aborts # so only add include if any file is present in the FLUENT_BIT_CONF_DIR if [ -d "$FLUENT_BIT_CONF_DIR" ] && [ "$(ls -A "$FLUENT_BIT_CONF_DIR")" ]; then - echo "@INCLUDE ${FLUENT_BIT_CONF_DIR}/*" >> ${TMP_CONF_FILE} + append_conf "@INCLUDE ${FLUENT_BIT_CONF_DIR}/*" fi - echo "" >> ${TMP_CONF_FILE} + append_conf "" } create_service_section() { # the service section of the fluent-bit.conf file has hardcoded values, # no need to lookup any uci section to configure this section - echo "[SERVICE]" >> ${TMP_CONF_FILE} - echo " flush 1" >> ${TMP_CONF_FILE} - echo " daemon off" >> ${TMP_CONF_FILE} - echo " log_level info" >> ${TMP_CONF_FILE} - echo " coro_stack_size 24576" >> ${TMP_CONF_FILE} - echo " parsers_file /etc/fluent-bit/parsers.conf" >> ${TMP_CONF_FILE} - echo " hot_reload on" >> ${TMP_CONF_FILE} - echo "" >> ${TMP_CONF_FILE} - - # Generate default input for kmsg - if [ -c "/dev/kmsg" ]; then - echo "[INPUT]" >> ${TMP_CONF_FILE} - echo " name kmsg" >> ${TMP_CONF_FILE} - echo " Tag KMSG" >> ${TMP_CONF_FILE} - echo "" >> ${TMP_CONF_FILE} - - echo "[OUTPUT]" >> ${TMP_CONF_FILE} - echo " name file" >> ${TMP_CONF_FILE} - echo " match KMSG" >> ${TMP_CONF_FILE} - echo " file /var/log/messages" >> ${TMP_CONF_FILE} - echo " format template" >> ${TMP_CONF_FILE} - echo " template {syslog_ts} {hostname} kernel: {msg}" >> ${TMP_CONF_FILE} - fi - - echo "" >> ${TMP_CONF_FILE} + append_conf "[SERVICE]" + append_conf " flush 1" + append_conf " daemon off" + append_conf " log_level info" + append_conf " coro_stack_size 24576" + append_conf " parsers_file /etc/fluent-bit/parsers.conf" + append_conf " hot_reload on" + append_conf "" } create_input_section() { local tag="$1" + [ -z "$tag" ] && return + + # check if this source section has already been processed + syslog_tag_already_processed "$tag" && return + # the input in our case is always syslog, hence, this section of the # fluent-bit.conf file has hardcoded values as well that do not depend # on any uci value - echo "[INPUT]" >> ${TMP_CONF_FILE} - echo " name syslog" >> ${TMP_CONF_FILE} - echo " tag $tag" >> ${TMP_CONF_FILE} - echo " path /dev/log" >> ${TMP_CONF_FILE} - echo "" >> ${TMP_CONF_FILE} + append_conf "[INPUT]" + append_conf " name syslog" + append_conf " tag $tag" + append_conf " path /dev/log" + append_conf "" +} + +create_kmsg_input_section() { + local tag="$1" + + [ -z "$tag" ] && return + + # check if this source section has already been processed + kmsg_tag_already_processed "$tag" && return + + if [ -c "/dev/kmsg" ]; then + append_conf "[INPUT]" + append_conf " name kmsg" + append_conf " tag $tag" + append_conf "" + fi } generate_facility_regex() { @@ -87,7 +122,7 @@ generate_facility_regex() { # generate regex for each. for sval in 0 1 2 3 4 5 6 7; do pri=`expr $val \* 8 + $sval` - echo " regex pri $pri" >> ${TMP_CONF_FILE} + append_conf " regex pri $pri" done done } @@ -109,7 +144,7 @@ generate_severity_regex() { # generate regex for all facility values, with severity=sev_level while [ $fval -le 23 ] ; do pri=`expr $fval \* 8 + $sev_level` - echo " $param pri $pri" >> ${TMP_CONF_FILE} + append_conf " $param pri $pri" fval=$((fval + 1)) done elif [ "$sev_compare" == "1" ]; then @@ -120,7 +155,7 @@ generate_severity_regex() { sval=0 while [ $sev_level -ge $sval ]; do pri=`expr $fval \* 8 + $sval` - echo " $param pri $pri" >> ${TMP_CONF_FILE} + append_conf " $param pri $pri" sval=$((sval + 1)) done fval=$((fval + 1)) @@ -149,7 +184,7 @@ handle_filter_conf() { local pattern_match config_get pattern_match $section pattern_match if [ -n "$pattern_match" ]; then - echo " regex $pattern_match" >> ${TMP_CONF_FILE} + append_conf " regex $pattern_match" fi local facility_level @@ -180,12 +215,14 @@ handle_filter_conf() { } create_filter_section() { - local match="$1" + local match_regex="$1" - echo "[FILTER]" >> ${TMP_CONF_FILE} - echo " name grep" >> ${TMP_CONF_FILE} - echo " match $match" >> ${TMP_CONF_FILE} - echo " logical_op or" >> ${TMP_CONF_FILE} # handle multiple filters + [ -z "$match_regex" ] && return + + append_conf "[FILTER]" + append_conf " name grep" + append_conf " match_regex $match_regex" + append_conf " logical_op or" # handle multiple filters } handle_filter_ref() { @@ -195,47 +232,55 @@ handle_filter_ref() { handle_log_file() { local section="$1" # out_file section - local match="$2" + local linker="$2" + local match_regex="$3" + local template="$4" local action_ref config_get action_ref $section action - if [ "$action_ref" != "$match" ]; then + if [ "$action_ref" != "$linker" ]; then return fi local enabled - config_get enabled $section enable - if [ "$enabled" == 0 ]; then + config_get_bool enabled $section enable + if [ "$enabled" = "0" ]; then return fi local file config_get file $section file - if [ -z "$file" ]; then + if [ -z "$file" ] || [ -z "$match_regex" ]; then return fi - echo "[OUTPUT]" >> ${TMP_CONF_FILE} - echo " name file" >> ${TMP_CONF_FILE} - echo " match $match" >> ${TMP_CONF_FILE} - echo " file $file" >> ${TMP_CONF_FILE} - echo " format template" >> ${TMP_CONF_FILE} - echo " template {time} {hostname} {ident}: {message}" >> ${TMP_CONF_FILE} + append_conf "[OUTPUT]" + append_conf " name file" + append_conf " match_regex $match_regex" + append_conf " file $file" + + if [ -n "$template" ]; then + append_conf " format template" + append_conf " template ${template}" + fi + + append_conf "" } handle_log_remote() { local section="$1" - local match="$2" + local linker="$2" + local match_regex="$3" local action_ref config_get action_ref $section action - if [ "$action_ref" != "$match" ]; then + if [ "$action_ref" != "$linker" ]; then return fi local enabled - config_get enabled $section enable - if [ "$enabled" == 0 ]; then + config_get_bool enabled $section enable + if [ "$enabled" = "0" ]; then return fi @@ -245,10 +290,10 @@ handle_log_remote() { return fi - echo "[OUTPUT]" >> ${TMP_CONF_FILE} - echo " name syslog" >> ${TMP_CONF_FILE} - echo " match $match" >> ${TMP_CONF_FILE} - echo " host $address" >> ${TMP_CONF_FILE} + append_conf "[OUTPUT]" + append_conf " name syslog" + append_conf " match_regex $match_regex" + append_conf " host $address" append_conf " syslog_appname_key ident" append_conf " syslog_procid_key pid" append_conf " syslog_message_key message" @@ -262,46 +307,114 @@ handle_log_remote() { config_get proto ${section} proto if [ -n "$proto" ]; then if [ "$proto" == "tls" ]; then - echo " mode tcp" >> ${TMP_CONF_FILE} - echo " tls on" >> ${TMP_CONF_FILE} + append_conf " mode tcp" + append_conf " tls on" else - echo " mode $proto" >> ${TMP_CONF_FILE} + append_conf " mode $proto" fi fi local port config_get port $section port if [ -n "$port" ]; then - echo " port $port" >> ${TMP_CONF_FILE} + append_conf " port $port" fi local cert local peer_verify config_get cert $section cert if [ -n "$cert" ]; then - echo " tls.crt_file $cert" >> ${TMP_CONF_FILE} + append_conf " tls.crt_file $cert" - config_get peer_verify $section peer_verify - if [ "$peer_verify" == "1" ]; then - echo " tls.verify on" >> ${TMP_CONF_FILE} + config_get_bool peer_verify $section peer_verify + if [ "$peer_verify" = "1" ]; then + append_conf " tls.verify on" fi fi + append_conf "" +} + +resolve_source_section() { + local src_section="$1" + local linker="$2" + local src_name syslog_en kernel_en source_tag_syslog source_tag_kmsg + + config_get src_name "$src_section" name + [ "$src_name" = "$linker" ] || return + + config_get_bool syslog_en "$src_section" system_messages 1 + config_get_bool kernel_en "$src_section" kernel_messages 1 + + # create an input section using /dev/log or kmsg + # and store the tag in a variable + # so that later a regex can be made to match this tag + # which will be used in output section + if [ "$syslog_en" = "1" ]; then + source_tag_syslog="SL$src_name" + create_input_section "$source_tag_syslog" + action_tags="$source_tag_syslog $action_tags" + fi + + if [ "$kernel_en" = "1" ]; then + source_tag_kmsg="KM$src_name" + create_kmsg_input_section "$source_tag_kmsg" + action_tags="$source_tag_kmsg $action_tags" + fi +} + +# get the value of option expression from the relevant section +resolve_template_section() { + local tmpl_section="$1" + local tmpl_name + + config_get tmpl_name "$tmpl_section" name + [ "$tmpl_name" = "$template_ref" ] || return + + config_get template_expr "$tmpl_section" expression + + [ -n "$template_expr" ] && echo "$template_expr" +} + +# loop over template sections and get the value of option expression from the relevant section +get_template_expression() { + local template_ref="$1" + [ -n "$template_ref" ] && config_foreach resolve_template_section template +} + +# build a regex that will match all the tags supplied to this function +build_match_regex() { + local tags="$1" + local first=1 + local regex="^(" + for tag in $tags; do + [ "$first" -eq 1 ] && first=0 || regex="$regex|" + regex="$regex$tag" + done + regex="$regex)\$" + echo "$regex" } handle_action() { - local section="$1" + local tag_regex filter source_ref template_ref source_sec log_template + local action_section="$1" + local action_tags="" - local filter - config_get filter $section filter + config_get action_name "$action_section" name + config_get filter "$action_section" filter + config_get source_ref "$action_section" source + config_get template_ref "$action_section" template - # use config action option name as tag for input - local tag - config_get tag $section name - if [ -z "$tag" ]; then - return - fi + [ -z "$action_name" ] && return + [ -z "$source_ref" ] && return + + # Resolve referenced source sections + for source_sec in $source_ref; do + config_foreach resolve_source_section source "$source_sec" + done + + # build a regex that will match all the sources for this action + tag_regex=$(build_match_regex "$action_tags") - create_input_section $tag if [ -n "$filter" ]; then # the only fluentbit filter that is useful for the datamodel is # grep. Also, fluentbit does not seem to handle multiple instances @@ -309,7 +422,7 @@ handle_action() { # to an action entry in the uci would translate for us into a set of # regex/exclude values instead of individual FILTER section per uci # section filter is a list, treat according - create_filter_section $tag + create_filter_section "$tag_regex" IFS=" " for finst in $filter; do @@ -317,11 +430,14 @@ handle_action() { done fi - # handle output, each action can be associated with a out_log and out_syslog + # get the template expression if any is present + log_template="$(get_template_expression "$template_ref")" + + # handle output, each action can be associated with an out_log and out_syslog # section so figure out if any out_log or out_syslog section is associated # with this and action and setup output accordingly. - config_foreach handle_log_file log_file "$tag" - config_foreach handle_log_remote log_remote "$tag" + config_foreach handle_log_file log_file "$action_name" "$tag_regex" "$log_template" + config_foreach handle_log_remote log_remote "$action_name" "$tag_regex" } handle_action_section() { @@ -343,7 +459,7 @@ logmngr_init() { logrotate_init fi - if [ "$enabled" == "0" ]; then + if [ "$enabled" = "0" ]; then return fi diff --git a/logmngr/files/logmngr.uci b/logmngr/files/logmngr.uci deleted file mode 100644 index 948ee62c4..000000000 --- a/logmngr/files/logmngr.uci +++ /dev/null @@ -1,15 +0,0 @@ -config globals 'globals' - option enable '1' - -config action 'a1' - option name 'ac1' - -config log_file 'lf1' - option enable '1' - option action 'ac1' - option file '/var/log/messages' - -config log_remote 'lr1' - option enable '0' - option action 'ac1' - option port '514'