diff --git a/fluent-bit/Makefile b/fluent-bit/Makefile index 8405882c2..675dfa505 100644 --- a/fluent-bit/Makefile +++ b/fluent-bit/Makefile @@ -1,5 +1,5 @@ # -# Copyright (C) 2023 IOPSYS +# Copyright (C) 2024 IOPSYS # include $(TOPDIR)/rules.mk @@ -165,11 +165,9 @@ CMAKE_OPTIONS += \ -DFLB_OUT_PGSQL=No define Package/fluent-bit/install - $(INSTALL_DIR) $(1)/etc/init.d $(INSTALL_DIR) $(1)/usr/sbin $(INSTALL_DIR) $(1)/etc/fluent-bit $(INSTALL_BIN) $(PKG_BUILD_DIR)/bin/fluent-bit $(1)/usr/sbin/ - $(INSTALL_BIN) ./files/fluent-bit.init $(1)/etc/init.d/fluent-bit $(INSTALL_DATA) ./files/fluent-bit.conf $(1)/etc/fluent-bit/fluent-bit.conf $(INSTALL_DATA) $(PKG_BUILD_DIR)/conf/parsers.conf $(1)/etc/fluent-bit/parsers.conf endef diff --git a/fluent-bit/files/fluent-bit.init b/fluent-bit/files/fluent-bit.init deleted file mode 100644 index b8e2fb606..000000000 --- a/fluent-bit/files/fluent-bit.init +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh /etc/rc.common - -START=12 -STOP=89 -USE_PROCD=1 - -PROG=/usr/sbin/fluent-bit -CONF=/etc/fluent-bit/fluent-bit.conf - -. /lib/functions.sh - -start_service() { - procd_open_instance - procd_set_param command $PROG -c $CONF - procd_set_param respawn - procd_close_instance -} - -service_triggers() { - procd_add_reload_trigger fluent-bit -} diff --git a/logmngr/Config.in b/logmngr/Config.in new file mode 100644 index 000000000..9fae40e1e --- /dev/null +++ b/logmngr/Config.in @@ -0,0 +1,20 @@ +if PACKAGE_logmngr +choice + prompt "Select backend for syslog management" + default LOGMNGR_BACKEND_FLUENTBIT + depends on PACKAGE_logmngr + help + Select which backend daemon to use for syslog management + +config LOGMNGR_BACKEND_FLUENTBIT + bool "Use fluent-bit for log management" + help + Enable this option to use fluent-bit for log management. + +config LOGMNGR_BACKEND_SYSLOG_NG + bool "Use syslog-ng for log management" + help + Enable this option to use syslog-ng for log management. + +endchoice +endif diff --git a/logmngr/Makefile b/logmngr/Makefile new file mode 100644 index 000000000..ac51b2f29 --- /dev/null +++ b/logmngr/Makefile @@ -0,0 +1,61 @@ +# +# Copyright (C) 2024 iopsys +# + +include $(TOPDIR)/rules.mk +include $(INCLUDE_DIR)/kernel.mk + +PKG_NAME:=logmngr +PKG_VERSION:=1.0.0 +LOCAL_DEV:=0 +ifneq ($(LOCAL_DEV),1) +PKG_SOURCE_PROTO:=git +PKG_SOURCE_URL:=https://dev.iopsys.eu/system/logmngr.git +PKG_SOURCE_VERSION:=a5a76071812f6a505c80bd99665f50b30043603c +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_MIRROR_HASH:=skip +endif + +PKG_LICENSE:=GPL-2.0-only +PKG_LICENSE_FILES:=LICENSE + +include $(INCLUDE_DIR)/package.mk +include ../bbfdm/bbfdm.mk + + +define Package/logmngr + CATEGORY:=Utilities + TITLE:=Logging Manager + DEPENDS:=+libbbfdm-api +LOGMNGR_BACKEND_FLUENTBIT:fluent-bit + DEPENDS:=+LOGMNGR_BACKEND_SYSLOG_NG:syslog-ng +endef + +define Package/logmngr/description + Configure log management. This package has the datamodel as well as the + the backend implementation for handling syslog. +endef + +define Package/$(PKG_NAME)/config + source "$(SOURCE)/Config.in" +endef + +ifeq ($(LOCAL_DEV),1) +define Build/Prepare + $(CP) -rf ./logmngr/* $(PKG_BUILD_DIR)/ +endef +endif + +define Build/Compile +endef + +define Package/logmngr/install + $(INSTALL_DIR) $(1)/lib/logmngr + $(INSTALL_DIR) $(1)/etc/init.d + + $(INSTALL_BIN) ./files/logmngr.init $(1)/etc/init.d/logmngr +ifeq ($(CONFIG_LOGMNGR_BACKEND_FLUENTBIT),y) + $(INSTALL_DATA) ./files/lib/logmngr/fluent-bit.sh $(1)/lib/logmngr/. +endif +endef + +$(eval $(call BuildPackage,logmngr)) diff --git a/logmngr/files/lib/logmngr/fluent-bit.sh b/logmngr/files/lib/logmngr/fluent-bit.sh new file mode 100644 index 000000000..f64ccccf3 --- /dev/null +++ b/logmngr/files/lib/logmngr/fluent-bit.sh @@ -0,0 +1,314 @@ +#!/bin/sh + +. /lib/functions.sh + +CONF_FILE=/etc/fluent-bit/fluent-bit.conf +TMP_CONF_FILE=/tmp/fluent-bit/fluent-bit.conf + +create_config_file() { + mkdir -p /tmp/fluent-bit + rm -f ${TMP_CONF_FILE} + touch ${TMP_CONF_FILE} +} + +create_service_section() { + # the service section of the fluent-bit.conf file has hardcoded values, + # no need to lookup any uci section to configure this section + echo "[SERVICE]" >> ${TMP_CONF_FILE} + echo " flush 3" >> ${TMP_CONF_FILE} + echo " daemon off" >> ${TMP_CONF_FILE} + echo " log_level info" >> ${TMP_CONF_FILE} + echo " parsers_file /etc/fluent-bit/parsers.conf" >> ${TMP_CONF_FILE} +} + +create_input_section() { + local tag="$1" + # the input in our case is always syslog, hence, this section of the + # fluent-bit.conf file has hardcoded values as well that do not depend + # on any uci value + echo "[INPUT]" >> ${TMP_CONF_FILE} + echo " name syslog" >> ${TMP_CONF_FILE} + echo " tag $tag" >> ${TMP_CONF_FILE} + echo " path /dev/log" >> ${TMP_CONF_FILE} +} + +generate_facility_regex() { + local facility_level=$1 + local pri=0 + + if [ "$facility_level" == "24" ]; then + # value 24 means all facility level, which is as good as not + # generating a filter section, so return + return + fi + + # facility_level is a list value, hence, generate regex for + # each value + IFS=" " + for val in $facility_level; do + # as per rfc 5424 and 3164, pri in syslog msg is + # facility*8+severity. Severity value can range from 0-7 hence + # generate regex for each. + for sval in 0 1 2 3 4 5 6 7; do + pri=`expr $val \* 8 + $sval` + echo " regex pri $pri" >> ${TMP_CONF_FILE} + done + done + +} + +generate_severity_regex() { + local sev_level="$1" + local sev_compare="$2" + local sev_action="$3" + + local pri=0 + local param="exclude" + + if [ "$sev_action" == "0" ]; then + param="regex" + fi + + local fval=0 + if [ "$sev_compare" == "0" ]; then + # generate regex for all facility values, with severity=sev_level + while [ $fval -le 23 ] ; do + pri=`expr $fval \* 8 + $sev_level` + echo " $param pri $pri" >> ${TMP_CONF_FILE} + fval=$((fval + 1)) + done + elif [ "$sev_compare" == "1" ]; then + # generate regex for all severity value greater than or equal to + # sev_level + while [ $fval -le 23 ] ; do + sval=7 + while [ $sval -ge $sev_level ]; do + pri=`expr $fval \* 8 + $sval` + echo " $param pri $pri" >> ${TMP_CONF_FILE} + sval=$((sval - 1)) + done + fval=$((fval + 1)) + done + fi +} + +handle_filter_conf() { + local section="$1" # config filter + local filter_name="$2" + local name + + # no need to proceed if name of filter section is not one of the values + # listed in option filter in config action section + config_get name $section name + if [ "$name" != "$filter_name" ]; then + return + fi + + # as per data model, at a time either facility_level or severity_level can + # be specified along with pattern_match. hence, first process and generate + # regex for pattern_match which is common in both condition. Next, we will + # process facility_level and return if facility level is defined and not + # process severity related params at all. + + local pattern_match + config_get pattern_match $section pattern_match + if [ -n "$pattern_match" ]; then + echo " regex $pattern_match" >> ${TMP_CONF_FILE} + fi + + local facility_level + config_get facility_level $section facility_level + + if [ -n "$facility_level" ]; then + generate_facility_regex $facility_level + # return from here since if facility_level is defined, then no + # need to process severity_level + return + fi + + local sev_level + local sev_compare + local sev_action + config_get sev_level $section severity_level + + if [ -n "$sev_level" ]; then + # value 1 of severity compare corresponds to data model + # and system default which is EqualorHigher + config_get sev_compare $section severity_compare 1 + # value 0 of severity action corresponds to data model + # and system default that is log + config_get sev_action $section severity_action 0 + + generate_severity_regex $sev_level $sev_compare $sev_action + fi +} + +create_filter_section() { + local match="$1" + + echo "[FILTER]" >> ${TMP_CONF_FILE} + echo " name grep" >> ${TMP_CONF_FILE} + echo " match $match" >> ${TMP_CONF_FILE} + echo " logical_op or" >> ${TMP_CONF_FILE} # handle multiple filters +} + +handle_filter_ref() { + local filter_name="$1" + config_foreach handle_filter_conf filter $filter_name +} + +handle_log_file() { + local section="$1" # out_file section + local match="$2" + local action_ref + + config_get action_ref $section action + if [ "$action_ref" != "$match" ]; then + return + fi + + local enabled + config_get enabled $section enable + if [ "$enabled" == 0 ]; then + return + fi + + local file_name + local file_path + config_get file_name $section file_name + if [ -z "$file_name" ]; then + return + fi + + echo "[OUTPUT]" >> ${TMP_CONF_FILE} + echo " name file" >> ${TMP_CONF_FILE} + echo " match $match" >> ${TMP_CONF_FILE} + echo " file $file_name" >> ${TMP_CONF_FILE} + + config_get file_path $section file_path + if [ -n "$file_path" ]; then + echo " path $file_path" >> ${TMP_CONF_FILE} + fi +} + +handle_log_remote() { + local section="$1" + local match="$2" + local action_ref + + config_get action_ref $section action + if [ "$action_ref" != "$match" ]; then + return + fi + + local enabled + config_get enabled $section enable + if [ "$enabled" == 0 ]; then + return + fi + + + local address + config_get address $section log_ip + if [ -z "$address" ]; then + return + fi + + echo "[OUTPUT]" >> ${TMP_CONF_FILE} + echo " name syslog" >> ${TMP_CONF_FILE} + echo " match $match" >> ${TMP_CONF_FILE} + echo " host $address" >> ${TMP_CONF_FILE} + + local proto # holds value tcp or udp + config_get proto $section proto + if [ -n "$proto" ]; then + echo " mode $proto" >> ${TMP_CONF_FILE} + fi + + local port + config_get port $section port + if [ -n "$port" ]; then + echo " port $port" >> ${TMP_CONF_FILE} + fi + + local cert + local peer_verify + config_get cert $section cert + if [ -n "$cert" ]; then + echo " tls on" >> ${TMP_CONF_FILE} + echo " tls.crt_file $cert" >> ${TMP_CONF_FILE} + + config_get peer_verify $section peer_verify + if [ "$peer_verify" == "1" ]; then + echo " tls.verify on" >> ${TMP_CONF_FILE} + fi + fi +} + +handle_action() { + local section="$1" + + local filter + config_get filter $section filter + + # use config action option name as tag for input + local tag + config_get tag $section name + if [ -z "$tag" ]; then + return + fi + + create_input_section $tag + if [ -n "$filter" ]; then + # the only fluentbit filter that is useful for the datamodel is + # grep. Also, fluentbit does not seem to handle multiple instances + # of FILTER of same kind. Hence, each filter section corresponding + # to an action entry in the uci would translate for us into a set of + # regex/exclude values instead of individual FILTER section per uci + # section filter is a list, treat according + create_filter_section $tag + + IFS=" " + for finst in $filter; do + handle_filter_ref $finst + done + fi + + # handle output, each action can be associated with a out_log and out_syslog + # section so figure out if any out_log or out_syslog section is associated + # with this and action and setup output accordingly. + config_foreach handle_log_file log_file $tag + config_foreach handle_log_remote log_remote $tag + + +} + +handle_action_section() { + config_foreach handle_action action +} + +apply_config_file() { + cp ${TMP_CONF_FILE} ${CONF_FILE} +} + +PROG=/usr/sbin/fluent-bit +logmngr_init() { + create_config_file + + config_load logmngr + local enabled + config_get enabled globals enable + + if [ "$enabled" == "0" ]; then + return + fi + + create_service_section + handle_action_section + apply_config_file + + procd_open_instance + procd_set_param command $PROG -c $CONF_FILE + procd_set_param respawn + procd_close_instance +} diff --git a/logmngr/files/logmngr.init b/logmngr/files/logmngr.init new file mode 100644 index 000000000..af7a82530 --- /dev/null +++ b/logmngr/files/logmngr.init @@ -0,0 +1,16 @@ +#!/bin/sh /etc/rc.common + +START=12 +STOP=89 +USE_PROCD=1 + +. /lib/functions.sh +include /lib/logmngr + +start_service() { + logmngr_init +} + +service_triggers() { + procd_add_reload_trigger logmngr +}