mirror/iopsys-feed
1
0
Fork 0
mirror of https://dev.iopsys.eu/feed/iopsys.git synced 2025-12-10 07:44:50 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

logmngr: add support for Syslog Source and Template

Browse source
This commit is contained in:
Husaam Mehdi 2025-08-05 11:36:51 +00:00 • committed by IOPSYS Dev
parent 8e11e5dbc8
commit e35bf1d509
No known key found for this signature in database
8 changed files with 284 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
logmngr/Config.in
View file

@ -8,6 +8,7 @@ choice
config LOGMNGR_BACKEND_FLUENTBIT
bool "Use fluent-bit for log management"
default y
help
Enable this option to use fluent-bit for log management.
@ -31,4 +32,5 @@ config LOGMNGR_VENDOR_LOG_FILE
default y
help
It adds support for Device.DeviceInfo.VendorLogFile. Object.
endif

28
logmngr/Makefile
View file

@ -5,13 +5,13 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=logmngr
PKG_VERSION:=1.0.20
PKG_VERSION:=1.1.0
LOCAL_DEV:=0
ifneq ($(LOCAL_DEV),1)
PKG_SOURCE_PROTO:=git
PKG_SOURCE_URL:=https://dev.iopsys.eu/system/logmngr.git
PKG_SOURCE_VERSION:=ad2636c642d56967e78c0c84bf82cb0e2b6311f2
PKG_SOURCE_VERSION:=ad043d5d2e96c1b203a9c5b22ad6e6e1488ad8f2
PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz
PKG_MIRROR_HASH:=skip
endif
@ -52,30 +52,36 @@ endif
define Package/logmngr/install
$(INSTALL_DIR) $(1)/etc/init.d
$(INSTALL_BIN) ./files/logmngr.init $(1)/etc/init.d/logmngr
$(INSTALL_DIR) $(1)/etc/config
$(INSTALL_BIN) ./files/logmngr.uci $(1)/etc/config/logmngr
$(INSTALL_DIR) $(1)/etc/uci-defaults
$(INSTALL_BIN) ./files/etc/init.d/logmngr $(1)/etc/init.d/logmngr
$(INSTALL_BIN) ./files/etc/config/logmngr $(1)/etc/config/logmngr
$(BBFDM_INSTALL_MS_PLUGIN) $(PKG_BUILD_DIR)/bbf_plugin/libbbfsyslog.so $(1) core 10
# Install logmngr service backend
$(INSTALL_DIR) $(1)/lib/logmngr
ifeq ($(CONFIG_LOGMNGR_BACKEND_FLUENTBIT),y)
$(INSTALL_DATA) ./files/lib/logmngr/fluent-bit.sh $(1)/lib/logmngr/
$(INSTALL_DIR) $(1)/sbin
$(INSTALL_BIN) ./files/logread $(1)/sbin/
endif
ifeq ($(CONFIG_LOGMNGR_BACKEND_SYSLOG_NG),y)
$(INSTALL_DATA) ./files/lib/logmngr/fluent-bit.sh $(1)/lib/logmngr/
$(INSTALL_BIN) ./files/etc/uci-defaults/10-logmngr_config_migrate $(1)/etc/uci-defaults/
else ifeq ($(CONFIG_LOGMNGR_BACKEND_SYSLOG_NG),y)
$(INSTALL_DATA) ./files/lib/logmngr/syslog-ng.sh $(1)/lib/logmngr/
endif
$(BBFDM_INSTALL_MS_PLUGIN) $(PKG_BUILD_DIR)/bbf_plugin/libbbfsyslog.so $(1) core 10
ifeq ($(CONFIG_LOGMNGR_LOGROTATE),y)
$(INSTALL_DIR) $(1)/etc/uci-defaults
$(INSTALL_BIN) ./files/11-logmngr_logrotate_config_generate $(1)/etc/uci-defaults/
$(INSTALL_DATA) ./files/lib/logmngr/logrotate.sh $(1)/lib/logmngr/
$(INSTALL_BIN) .//files/etc/uci-defaults/11-logmngr_logrotate_syslog $(1)/etc/uci-defaults/
$(BBFDM_INSTALL_MS_PLUGIN) $(PKG_BUILD_DIR)/bbf_plugin/libbbflogrotate.so $(1) sysmngr 11
endif
ifeq ($(CONFIG_LOGMNGR_VENDOR_LOG_FILE),y)
$(BBFDM_INSTALL_MS_PLUGIN) $(PKG_BUILD_DIR)/bbf_plugin/libbbfvendorlog.so $(1) sysmngr 12
endif
endef
$(eval $(call BuildPackage,logmngr))

26
logmngr/files/etc/config/logmngr Normal file
View file

@ -0,0 +1,26 @@
config globals 'globals'
option enable '1'
config source 'default_source'
option name 'default_source'
option system_messages '1'
option kernel_messages '1'
config template 'default_template'
option name 'default_template'
option expression '{time} {hostname} {ident}: {message}'
config action 'default_action'
option name 'default_action'
list source 'default_source'
option template 'default_template'
config log_file 'lf1'
option enable '1'
option action 'default_action'
option file '/var/log/messages'
config log_remote 'lr1'
option enable '0'
option action 'default_action'
option port '514'

0
logmngr/files/logmngr.init → logmngr/files/etc/init.d/logmngr
View file

39
logmngr/files/etc/uci-defaults/10-logmngr_config_migrate Normal file
View file

@ -0,0 +1,39 @@
#!/bin/sh
. /lib/functions.sh
# check if this is a new type UCI or old type UCI
if ! uci -q get logmngr.default_source > /dev/null; then
uci -q set logmngr.default_source=source
uci -q set logmngr.default_source.name='default_source'
uci -q set logmngr.default_source.system_messages='1'
uci -q set logmngr.default_source.kernel_messages='1'
fi
if ! uci -q get logmngr.default_template > /dev/null; then
uci -q set logmngr.default_template=template
uci -q set logmngr.default_template.name='default_template'
uci -q set logmngr.default_template.expression='{time} {hostname} {ident}: {message}'
fi
if uci -q get logmngr.a1 >/dev/null; then
uci -q rename logmngr.a1='default_action'
uci -q set logmngr.default_action.name='default_action'
uci -q set logmngr.default_action.template='default_template'
uci -q delete logmngr.default_action.source
uci -q add_list logmngr.default_action.source='default_source'
fi
if uci -q get logmngr.lf1 >/dev/null; then
uci -q rename logmngr.lf1='default_logfile'
uci -q set logmngr.default_logfile.action='default_action'
fi
if uci -q get logmngr.lr1 >/dev/null; then
uci -q rename logmngr.lr1='default_logremote'
uci -q set logmngr.default_logremote.action='default_action'
fi
exit 0

2
logmngr/files/11-logmngr_logrotate_config_generate → logmngr/files/etc/uci-defaults/11-logmngr_logrotate_syslog
View file

@ -1,7 +1,7 @@
#!/bin/sh
# Adds a default log rotate policy if none exists
if uci -q get logmngr.@log_rotate[0] >/dev/null; then
if uci -q get logmngr.lro1 >/dev/null; then
# return if there is any valid content
exit 0
fi

282
logmngr/files/lib/logmngr/fluent-bit.sh
View file

@ -6,6 +6,37 @@
CONF_FILE=/etc/fluent-bit/fluent-bit.conf
TMP_CONF_FILE=/tmp/fluent-bit/fluent-bit.conf
FLUENT_BIT_CONF_DIR=/etc/fluent-bit/conf.d
PROCESSED_SYSLOG_TAGS=""
PROCESSED_KMSG_TAGS=""
# check if syslog source section is already processed
# and add it to the list of processed source sections
syslog_tag_already_processed() {
local tag="$1"
for t in $PROCESSED_SYSLOG_TAGS; do
[ "$t" = "$tag" ] && return 0
done
PROCESSED_SYSLOG_TAGS="$tag $PROCESSED_SYSLOG_TAGS"
return 1
}
# check if kmsg source section is already processed
# and add it to the list of processed source sections
# two separate functions used because we want to populate
# appropriate PROCESSED variable
kmsg_tag_already_processed() {
local tag="$1"
for t in $PROCESSED_KMSG_TAGS; do
[ "$t" = "$tag" ] && return 0
done
PROCESSED_KMSG_TAGS="$tag $PROCESSED_KMSG_TAGS"
return 1
}
append_conf() {
echo "$*" >> ${TMP_CONF_FILE}
@ -20,52 +51,56 @@ create_config_file() {
# also, if no file is found then fluent-bit aborts
# so only add include if any file is present in the FLUENT_BIT_CONF_DIR
if [ -d "$FLUENT_BIT_CONF_DIR" ] && [ "$(ls -A "$FLUENT_BIT_CONF_DIR")" ]; then
echo "@INCLUDE ${FLUENT_BIT_CONF_DIR}/*" >> ${TMP_CONF_FILE}
append_conf "@INCLUDE ${FLUENT_BIT_CONF_DIR}/*"
fi
echo "" >> ${TMP_CONF_FILE}
append_conf ""
}
create_service_section() {
# the service section of the fluent-bit.conf file has hardcoded values,
# no need to lookup any uci section to configure this section
echo "[SERVICE]" >> ${TMP_CONF_FILE}
echo " flush 1" >> ${TMP_CONF_FILE}
echo " daemon off" >> ${TMP_CONF_FILE}
echo " log_level info" >> ${TMP_CONF_FILE}
echo " coro_stack_size 24576" >> ${TMP_CONF_FILE}
echo " parsers_file /etc/fluent-bit/parsers.conf" >> ${TMP_CONF_FILE}
echo " hot_reload on" >> ${TMP_CONF_FILE}
echo "" >> ${TMP_CONF_FILE}
# Generate default input for kmsg
if [ -c "/dev/kmsg" ]; then
echo "[INPUT]" >> ${TMP_CONF_FILE}
echo " name kmsg" >> ${TMP_CONF_FILE}
echo " Tag KMSG" >> ${TMP_CONF_FILE}
echo "" >> ${TMP_CONF_FILE}
echo "[OUTPUT]" >> ${TMP_CONF_FILE}
echo " name file" >> ${TMP_CONF_FILE}
echo " match KMSG" >> ${TMP_CONF_FILE}
echo " file /var/log/messages" >> ${TMP_CONF_FILE}
echo " format template" >> ${TMP_CONF_FILE}
echo " template {syslog_ts} {hostname} kernel: {msg}" >> ${TMP_CONF_FILE}
fi
echo "" >> ${TMP_CONF_FILE}
append_conf "[SERVICE]"
append_conf " flush 1"
append_conf " daemon off"
append_conf " log_level info"
append_conf " coro_stack_size 24576"
append_conf " parsers_file /etc/fluent-bit/parsers.conf"
append_conf " hot_reload on"
append_conf ""
}
create_input_section() {
local tag="$1"
[ -z "$tag" ] && return
# check if this source section has already been processed
syslog_tag_already_processed "$tag" && return
# the input in our case is always syslog, hence, this section of the
# fluent-bit.conf file has hardcoded values as well that do not depend
# on any uci value
echo "[INPUT]" >> ${TMP_CONF_FILE}
echo " name syslog" >> ${TMP_CONF_FILE}
echo " tag $tag" >> ${TMP_CONF_FILE}
echo " path /dev/log" >> ${TMP_CONF_FILE}
echo "" >> ${TMP_CONF_FILE}
append_conf "[INPUT]"
append_conf " name syslog"
append_conf " tag $tag"
append_conf " path /dev/log"
append_conf ""
}
create_kmsg_input_section() {
local tag="$1"
[ -z "$tag" ] && return
# check if this source section has already been processed
kmsg_tag_already_processed "$tag" && return
if [ -c "/dev/kmsg" ]; then
append_conf "[INPUT]"
append_conf " name kmsg"
append_conf " tag $tag"
append_conf ""
fi
}
generate_facility_regex() {
@ -87,7 +122,7 @@ generate_facility_regex() {
# generate regex for each.
for sval in 0 1 2 3 4 5 6 7; do
pri=`expr $val \* 8 + $sval`
echo " regex pri $pri" >> ${TMP_CONF_FILE}
append_conf " regex pri $pri"
done
done
}
@ -109,7 +144,7 @@ generate_severity_regex() {
# generate regex for all facility values, with severity=sev_level
while [ $fval -le 23 ] ; do
pri=`expr $fval \* 8 + $sev_level`
echo " $param pri $pri" >> ${TMP_CONF_FILE}
append_conf " $param pri $pri"
fval=$((fval + 1))
done
elif [ "$sev_compare" == "1" ]; then
@ -120,7 +155,7 @@ generate_severity_regex() {
sval=0
while [ $sev_level -ge $sval ]; do
pri=`expr $fval \* 8 + $sval`
echo " $param pri $pri" >> ${TMP_CONF_FILE}
append_conf " $param pri $pri"
sval=$((sval + 1))
done
fval=$((fval + 1))
@ -149,7 +184,7 @@ handle_filter_conf() {
local pattern_match
config_get pattern_match $section pattern_match
if [ -n "$pattern_match" ]; then
echo " regex $pattern_match" >> ${TMP_CONF_FILE}
append_conf " regex $pattern_match"
fi
local facility_level
@ -180,12 +215,14 @@ handle_filter_conf() {
}
create_filter_section() {
local match="$1"
local match_regex="$1"
echo "[FILTER]" >> ${TMP_CONF_FILE}
echo " name grep" >> ${TMP_CONF_FILE}
echo " match $match" >> ${TMP_CONF_FILE}
echo " logical_op or" >> ${TMP_CONF_FILE} # handle multiple filters
[ -z "$match_regex" ] && return
append_conf "[FILTER]"
append_conf " name grep"
append_conf " match_regex $match_regex"
append_conf " logical_op or" # handle multiple filters
}
handle_filter_ref() {
@ -195,47 +232,55 @@ handle_filter_ref() {
handle_log_file() {
local section="$1" # out_file section
local match="$2"
local linker="$2"
local match_regex="$3"
local template="$4"
local action_ref
config_get action_ref $section action
if [ "$action_ref" != "$match" ]; then
if [ "$action_ref" != "$linker" ]; then
return
fi
local enabled
config_get enabled $section enable
if [ "$enabled" == 0 ]; then
config_get_bool enabled $section enable
if [ "$enabled" = "0" ]; then
return
fi
local file
config_get file $section file
if [ -z "$file" ]; then
if [ -z "$file" ] || [ -z "$match_regex" ]; then
return
fi
echo "[OUTPUT]" >> ${TMP_CONF_FILE}
echo " name file" >> ${TMP_CONF_FILE}
echo " match $match" >> ${TMP_CONF_FILE}
echo " file $file" >> ${TMP_CONF_FILE}
echo " format template" >> ${TMP_CONF_FILE}
echo " template {time} {hostname} {ident}: {message}" >> ${TMP_CONF_FILE}
append_conf "[OUTPUT]"
append_conf " name file"
append_conf " match_regex $match_regex"
append_conf " file $file"
if [ -n "$template" ]; then
append_conf " format template"
append_conf " template ${template}"
fi
append_conf ""
}
handle_log_remote() {
local section="$1"
local match="$2"
local linker="$2"
local match_regex="$3"
local action_ref
config_get action_ref $section action
if [ "$action_ref" != "$match" ]; then
if [ "$action_ref" != "$linker" ]; then
return
fi
local enabled
config_get enabled $section enable
if [ "$enabled" == 0 ]; then
config_get_bool enabled $section enable
if [ "$enabled" = "0" ]; then
return
fi
@ -245,10 +290,10 @@ handle_log_remote() {
return
fi
echo "[OUTPUT]" >> ${TMP_CONF_FILE}
echo " name syslog" >> ${TMP_CONF_FILE}
echo " match $match" >> ${TMP_CONF_FILE}
echo " host $address" >> ${TMP_CONF_FILE}
append_conf "[OUTPUT]"
append_conf " name syslog"
append_conf " match_regex $match_regex"
append_conf " host $address"
append_conf " syslog_appname_key ident"
append_conf " syslog_procid_key pid"
append_conf " syslog_message_key message"
@ -262,46 +307,114 @@ handle_log_remote() {
config_get proto ${section} proto
if [ -n "$proto" ]; then
if [ "$proto" == "tls" ]; then
echo " mode tcp" >> ${TMP_CONF_FILE}
echo " tls on" >> ${TMP_CONF_FILE}
append_conf " mode tcp"
append_conf " tls on"
else
echo " mode $proto" >> ${TMP_CONF_FILE}
append_conf " mode $proto"
fi
fi
local port
config_get port $section port
if [ -n "$port" ]; then
echo " port $port" >> ${TMP_CONF_FILE}
append_conf " port $port"
fi
local cert
local peer_verify
config_get cert $section cert
if [ -n "$cert" ]; then
echo " tls.crt_file $cert" >> ${TMP_CONF_FILE}
append_conf " tls.crt_file $cert"
config_get peer_verify $section peer_verify
if [ "$peer_verify" == "1" ]; then
echo " tls.verify on" >> ${TMP_CONF_FILE}
config_get_bool peer_verify $section peer_verify
if [ "$peer_verify" = "1" ]; then
append_conf " tls.verify on"
fi
fi
append_conf ""
}
resolve_source_section() {
local src_section="$1"
local linker="$2"
local src_name syslog_en kernel_en source_tag_syslog source_tag_kmsg
config_get src_name "$src_section" name
[ "$src_name" = "$linker" ] || return
config_get_bool syslog_en "$src_section" system_messages 1
config_get_bool kernel_en "$src_section" kernel_messages 1
# create an input section using /dev/log or kmsg
# and store the tag in a variable
# so that later a regex can be made to match this tag
# which will be used in output section
if [ "$syslog_en" = "1" ]; then
source_tag_syslog="SL$src_name"
create_input_section "$source_tag_syslog"
action_tags="$source_tag_syslog $action_tags"
fi
if [ "$kernel_en" = "1" ]; then
source_tag_kmsg="KM$src_name"
create_kmsg_input_section "$source_tag_kmsg"
action_tags="$source_tag_kmsg $action_tags"
fi
}
# get the value of option expression from the relevant section
resolve_template_section() {
local tmpl_section="$1"
local tmpl_name
config_get tmpl_name "$tmpl_section" name
[ "$tmpl_name" = "$template_ref" ] || return
config_get template_expr "$tmpl_section" expression
[ -n "$template_expr" ] && echo "$template_expr"
}
# loop over template sections and get the value of option expression from the relevant section
get_template_expression() {
local template_ref="$1"
[ -n "$template_ref" ] && config_foreach resolve_template_section template
}
# build a regex that will match all the tags supplied to this function
build_match_regex() {
local tags="$1"
local first=1
local regex="^("
for tag in $tags; do
[ "$first" -eq 1 ] && first=0 || regex="$regex|"
regex="$regex$tag"
done
regex="$regex)\$"
echo "$regex"
}
handle_action() {
local section="$1"
local tag_regex filter source_ref template_ref source_sec log_template
local action_section="$1"
local action_tags=""
local filter
config_get filter $section filter
config_get action_name "$action_section" name
config_get filter "$action_section" filter
config_get source_ref "$action_section" source
config_get template_ref "$action_section" template
# use config action option name as tag for input
local tag
config_get tag $section name
if [ -z "$tag" ]; then
return
fi
[ -z "$action_name" ] && return
[ -z "$source_ref" ] && return
# Resolve referenced source sections
for source_sec in $source_ref; do
config_foreach resolve_source_section source "$source_sec"
done
# build a regex that will match all the sources for this action
tag_regex=$(build_match_regex "$action_tags")
create_input_section $tag
if [ -n "$filter" ]; then
# the only fluentbit filter that is useful for the datamodel is
# grep. Also, fluentbit does not seem to handle multiple instances
@ -309,7 +422,7 @@ handle_action() {
# to an action entry in the uci would translate for us into a set of
# regex/exclude values instead of individual FILTER section per uci
# section filter is a list, treat according
create_filter_section $tag
create_filter_section "$tag_regex"
IFS=" "
for finst in $filter; do
@ -317,11 +430,14 @@ handle_action() {
done
fi
# handle output, each action can be associated with a out_log and out_syslog
# get the template expression if any is present
log_template="$(get_template_expression "$template_ref")"
# handle output, each action can be associated with an out_log and out_syslog
# section so figure out if any out_log or out_syslog section is associated
# with this and action and setup output accordingly.
config_foreach handle_log_file log_file "$tag"
config_foreach handle_log_remote log_remote "$tag"
config_foreach handle_log_file log_file "$action_name" "$tag_regex" "$log_template"
config_foreach handle_log_remote log_remote "$action_name" "$tag_regex"
}
handle_action_section() {
@ -343,7 +459,7 @@ logmngr_init() {
logrotate_init
fi
if [ "$enabled" == "0" ]; then
if [ "$enabled" = "0" ]; then
return
fi

15
logmngr/files/logmngr.uci
View file

@ -1,15 +0,0 @@
config globals 'globals'
option enable '1'
config action 'a1'
option name 'ac1'
config log_file 'lf1'
option enable '1'
option action 'ac1'
option file '/var/log/messages'
config log_remote 'lr1'
option enable '0'
option action 'ac1'
option port '514'