iopsys-feed/logmngr/files/lib/logmngr/fluent-bit.sh

#!/bin/sh

. /lib/functions.sh
. /lib/logmngr/logrotate.sh

CONF_FILE=/etc/fluent-bit/fluent-bit.conf
TMP_CONF_FILE=/tmp/fluent-bit/fluent-bit.conf
FLUENT_BIT_CONF_DIR=/etc/fluent-bit/conf.d
PROCESSED_SYSLOG_TAGS=""
PROCESSED_KMSG_TAGS=""

# check if syslog source section is already processed
# and add it to the list of processed source sections
syslog_tag_already_processed() {
	local tag="$1"

	for t in $PROCESSED_SYSLOG_TAGS; do
		[ "$t" = "$tag" ] && return 0
	done

	PROCESSED_SYSLOG_TAGS="$tag $PROCESSED_SYSLOG_TAGS"

	return 1
}

# check if kmsg source section is already processed
# and add it to the list of processed source sections
# two separate functions used because we want to populate
# appropriate PROCESSED variable
kmsg_tag_already_processed() {
	local tag="$1"
	for t in $PROCESSED_KMSG_TAGS; do
		[ "$t" = "$tag" ] && return 0
	done

	PROCESSED_KMSG_TAGS="$tag $PROCESSED_KMSG_TAGS"

	return 1
}

append_conf() {
	echo "$*" >> ${TMP_CONF_FILE}
}

create_config_file() {
	mkdir -p /tmp/fluent-bit
	rm -f ${TMP_CONF_FILE}
	touch ${TMP_CONF_FILE}
	# include all files placed in FLUENT_BIT_CONF_DIR directory
	# fluent-bit does not support using directory in include directive
	# also, if no file is found then fluent-bit aborts
	# so only add include if any file is present in the FLUENT_BIT_CONF_DIR
	if [ -d "$FLUENT_BIT_CONF_DIR" ] && [ "$(ls -A "$FLUENT_BIT_CONF_DIR")" ]; then
		append_conf "@INCLUDE ${FLUENT_BIT_CONF_DIR}/*"
	fi
	append_conf ""
}

create_service_section() {
	# the service section of the fluent-bit.conf file has hardcoded values,
	# no need to lookup any uci section to configure this section
	append_conf "[SERVICE]"
	append_conf "    flush        1"
	append_conf "    daemon       off"
	append_conf "    log_level    info"
	append_conf "    coro_stack_size    24576"
	append_conf "    parsers_file /etc/fluent-bit/parsers.conf"
	append_conf "    hot_reload        on"
	append_conf ""
}

create_default_filters() {
	append_conf "[FILTER]"
	append_conf "    name            modify"
	append_conf "    match           KM*"
	append_conf "    add             ident   kernel"
	append_conf "    rename          msg     message"
	append_conf ""

	append_conf "[FILTER]"
	append_conf "    name            modify"
	append_conf "    match           *"
	append_conf "    add             pid     0"
	append_conf ""

	append_conf "[FILTER]"
	append_conf "    name            sysinfo"
	append_conf "    match           *"
	append_conf "    hostname_key    hostname"
	append_conf ""
}

create_input_section() {
	local tag="$1"

	[ -z "$tag" ] && return

	# check if this source section has already been processed
	syslog_tag_already_processed "$tag" && return

	append_conf "[INPUT]"
	append_conf "    name        syslog"
	append_conf "    unix_perm   0666"
	append_conf "    tag         $tag"
	append_conf "    path        /dev/log"
	append_conf ""
}

populate_allowed_logs() {
	local facility_level sev_level
	local section="$1"

	[ -z "$section" ] && return

	# reset
	match_pattern=""
	facilities=""
	all_facilities=0
	kern_facility=0
	severities=""
	sev_compare=1
	sev_action=0

	# read config
	config_get match_pattern $section pattern_match

	config_get facility_level $section facility_level
	config_get sev_level $section severity_level
	config_get sev_compare $section severity_compare 1
	config_get sev_action $section severity_action 0

	# normalize facilities
	if [ -n "$facility_level" ]; then
		for f in $facility_level; do
			if [ "$f" = "24" ]; then
				all_facilities=1
				# xargs is used to convert from new line separated numbers to space separated numbers
				facilities="$(seq 0 23 | xargs)"
				break
			fi

			if [ "$f" = "0" ]; then
				kern_facility=1
			fi
		done

		if [ "$all_facilities" -eq 0 ]; then
			facilities="$facility_level"
		fi
	else
		# default to "all facilities" when unset
        	all_facilities=1
	        facilities="$(seq 0 23 | xargs)"
	fi

	# normalize severities
	case "$sev_level" in
		8)  	# all severities
			severities="$(seq 0 7 | xargs)"
			;;
		9)  	# none
			severities="none"
			;;
		"") 	# unset, treat as "all"
			severities="$(seq 0 7 | xargs)"
			;;
		*)
			if [ "$sev_compare" = "0" ]; then
				# equal
				severities="$sev_level"
			else
				# equl or higher
				severities="$(seq 0 $sev_level | xargs)"
			fi
			;;
	esac
}

create_filter_section() {
	local match_regex="$1"
	local pattern="$2"

	[ -z "$match_regex" ] && return

	append_conf "[FILTER]"
	append_conf "    name        grep"
	append_conf "    match_regex       $match_regex"

	# we need "logical_op or" only in non-pattern sections
	if [ "$pattern" = "0" ]; then
		append_conf "    logical_op  or" # handle multiple filters
	fi
}

create_kmsg_input_section() {
	local tag="$1"
	local max_sev=7

	[ -z "$tag" ] && return
	kmsg_tag_already_processed "$tag" && return

	if [ -c "/dev/kmsg" ]; then
		append_conf "[INPUT]"
		append_conf "    name       kmsg"
		append_conf "    tag        $tag"

		# check kern facility (0)
		if [ "$all_facilities" -eq 1 ] || [ "$kern_facility" -eq 1 ]; then
			if [ "$severities" != "none" ]; then
				# severity filtering
				# only EqualOrHigher is supported by Prio_Level
				# and only Log action is supported
				# so set Prio_Level = max severity
				if [ "$sev_action" = "0" ] && [ "$sev_compare" = "1" ]; then
					if [ -n "$severities" ]; then
						max_sev=$(echo $severities | tr ' ' '\n' | sort -n | tail -1)
					fi

					append_conf "    prio_level $max_sev"
				fi
			fi
		fi

		append_conf ""

		# if severities is none, or
		# if kern facility has been excluded
		# then we need to stop kernel logs
		# sev_action and sev_compare is being checked because we don't want to work with rules that exclude logs
		if [ "$severities" = "none" ] || { [ "$kern_facility" -eq 0 ] && [ "$all_facilities" -eq 0 ] && [ "$sev_action" = "0" ] && [ "$sev_compare" = "1" ]; }; then
			# block all
			# create a filter section that matches on KM* tag
			# and excludes all messages
			create_filter_section "KM*" "0"
			append_conf "    exclude    message ^.*$"
			append_conf ""
		fi
	fi
}

generate_syslog_filter() {
	local param="regex"

	[ "$sev_action" = "1" ] && param="exclude"

	# start adding the fluent-bit filter section
	create_filter_section "SL*" "0"

	if [ "$severities" = "none" ]; then
		append_conf "    exclude    pri ^.*$"
		return
	fi

	for fval in $facilities; do
		for sval in $severities; do
			local pri=$((fval * 8 + sval))
			append_conf "    $param        pri ^${pri}$"
		done
	done

	append_conf ""
}

generate_pattern_filter() {
	local match_regex="$1"
	local match_pattern="$2"

	[ -z "$match_regex" ] && return
	[ -z "$match_pattern" ] && return

	# start adding the fluent-bit filter section
	create_filter_section "$match_regex" "1"
	append_conf "    regex        message $match_pattern"
	append_conf ""
}

handle_log_file() {
	local section="$1" # out_file section
	local linker="$2"
	local match_regex="$3"
	local template="$4"
	local action_ref

	config_get action_ref $section action
	if [ "$action_ref" != "$linker" ]; then
		return
	fi

	local enabled
	config_get_bool enabled $section enable
	if [ "$enabled" = "0" ]; then
		return
	fi

	local file
	config_get file $section file
	if [ -z "$file" ] || [ -z "$match_regex" ]; then
		return
	fi

	append_conf "[OUTPUT]"
	append_conf "    name         file"
	append_conf "    workers      2"
	append_conf "    match_regex  $match_regex"
	append_conf "    file         $file"


	if [ -n "$template" ]; then
		append_conf "    format       template"
		append_conf "    template     ${template}"
	fi

	append_conf ""
}

handle_log_remote() {
	local section="$1"
	local linker="$2"
	local match_regex="$3"
	local action_ref

	config_get action_ref $section action
	if [ "$action_ref" != "$linker" ]; then
		return
	fi

	local enabled
	config_get_bool enabled $section enable
	if [ "$enabled" = "0" ]; then
		return
	fi

	local address
	config_get address $section log_ip
	if [ -z "$address" ]; then
		return
	fi

	append_conf "[OUTPUT]"
	append_conf "    name         syslog"
	append_conf "    match_regex  $match_regex"
	append_conf "    host         $address"
	append_conf "    syslog_appname_key   ident"
	append_conf "    syslog_procid_key    pid"
	append_conf "    syslog_message_key   message"
	append_conf "    syslog_hostname_key  hostname"

	local proto # holds value tcp or udp
	config_get proto ${section} proto
	if [ -n "$proto" ]; then
		if [ "$proto" == "tls" ]; then
			append_conf "    mode         tcp"
			append_conf "    tls          on"
		else
			append_conf "    mode         $proto"
		fi
	fi

	local port
	config_get port $section port
	if [ -n "$port" ]; then
		append_conf "    port         $port"
	fi

	local cert
	local peer_verify
	config_get cert $section cert
	if [ -n "$cert" ]; then
		append_conf "    tls.crt_file $cert"

		config_get_bool peer_verify $section peer_verify
		if [ "$peer_verify" = "1" ]; then
			append_conf "    tls.verify     on"
		fi
	fi
	append_conf ""
}

resolve_source_section() {
	local src_section="$1"
	local linker="$2"
	local src_name syslog_en kernel_en

	config_get src_name "$src_section" name
	[ "$src_name" = "$linker" ] || return

	config_get_bool syslog_en "$src_section" system_messages 1
	config_get_bool kernel_en "$src_section" kernel_messages 1

	# create an input section using /dev/log or kmsg
	# and store the tag in a variable
	# so that later a regex can be made to match this tag
	# which will be used in output section
	if [ "$syslog_en" = "1" ]; then
		source_tag_syslog="SL$src_name"
		create_input_section "$source_tag_syslog"
	fi

	if [ "$kernel_en" = "1" ]; then
		source_tag_kmsg="KM$src_name"
		create_kmsg_input_section "$source_tag_kmsg"
	fi
}

# get the value of option expression from the relevant section
resolve_template_section() {
	local tmpl_section="$1"
	local tmpl_name

	config_get tmpl_name "$tmpl_section" name
	[ "$tmpl_name" = "$template_ref" ] || return

	config_get template_expr "$tmpl_section" expression

	[ -n "$template_expr" ] && echo "$template_expr"
}

# loop over template sections and get the value of option expression from the relevant section
get_template_expression() {
	local template_ref="$1"
	[ -n "$template_ref" ] && config_foreach resolve_template_section template
}

# build a regex that will match all the tags supplied to this function
build_match_regex() {
	local tags="$1"
	local first=1
	local regex="^("
	for tag in $tags; do
		[ "$first" -eq 1 ] && first=0 || regex="$regex|"
		regex="$regex$tag"
	done
	regex="$regex)\$"
	echo "$regex"
}

handle_filter_conf() {
	local section="$1" # config filter
	local filter_name="$2"
	local name

	config_get name $section name
	[ "$name" = "$filter_name" ] || return

	populate_allowed_logs "$filter_name"
}

handle_action() {
	local tag_regex filter source_ref template_ref source_sec log_template finst
	local action_section="$1"
	local source_tag_syslog source_tag_kmsg

	# shared variables set by populate_allowed_logs
	match_pattern=""
	facilities=""
	all_facilities=0
	kern_facility=1
	severities=""
	sev_compare=1
	sev_action=0

	config_get action_name "$action_section" name
	config_get filter "$action_section" filter
	config_get source_ref "$action_section" source
	config_get template_ref "$action_section" template

	[ -z "$action_name" ] && return
	[ -z "$source_ref" ] && return

	# read filter section and populate relevant variables
	# these variables will be used by create_kmsg_input_section
	# generate_syslog_filter, and generate_pattern_filter functions
	if [ -n "$filter" ]; then
		for finst in $filter; do
			config_foreach handle_filter_conf filter "$finst"
		done
	fi

	# Resolve referenced source sections
	for source_sec in $source_ref; do
		config_foreach resolve_source_section source "$source_sec"
	done

	# build a regex that will match all the sources for this action
	tag_regex=$(build_match_regex "$source_tag_syslog $source_tag_kmsg")

	if [ -n "$filter" ]; then
		generate_pattern_filter "$tag_regex" "$match_pattern"
		generate_syslog_filter
	fi

	# get the template expression if any is present
	log_template="$(get_template_expression "$template_ref")"

	# handle output, each action can be associated with an out_log and out_syslog
	# section so figure out if any out_log or out_syslog section is associated
	# with this and action and setup output accordingly.
	config_foreach handle_log_file log_file "$action_name" "$tag_regex" "$log_template"
	config_foreach handle_log_remote log_remote "$action_name" "$tag_regex"
}

handle_action_section() {
	config_foreach handle_action action
}

PROG=/usr/sbin/fluent-bit
logmngr_init() {
	local enabled

	config_load logmngr
	config_get_bool enabled globals enable "1"

	create_config_file
	create_service_section
	create_default_filters
	handle_action_section

	if [ -f /lib/logmngr/logrotate.sh ]; then
		logrotate_init
	fi

	if [ "$enabled" = "0" ]; then
		return
	fi

	procd_open_instance logmngr
	if [ -s "${TMP_CONF_FILE}" ]; then
		procd_set_param command $PROG -c ${TMP_CONF_FILE}
		procd_set_param file ${TMP_CONF_FILE}
	elif [ -s "${CONF_FILE}" ]; then
		procd_set_param command $PROG -c ${CONF_FILE}
		procd_set_param file ${CONF_FILE}
	fi
	procd_set_param respawn
	procd_close_instance
}