diff --git a/fluent-bit/Config.in b/fluent-bit/Config.in new file mode 100644 index 000000000..3a6f2fb87 --- /dev/null +++ b/fluent-bit/Config.in @@ -0,0 +1,5 @@ +config FLUENT_BIT_SYSLOG_PREPEND_LENGTH + bool "Enable syslog length prefix (RFC5424)" + default y + help + If enabled, prepends the byte-length of the syslog message to the message output. diff --git a/fluent-bit/Makefile b/fluent-bit/Makefile index 4d3b7fbcd..cb222021a 100644 --- a/fluent-bit/Makefile +++ b/fluent-bit/Makefile @@ -33,6 +33,10 @@ define Package/fluent-bit/description Fluent Bit is a super fast, lightweight, and highly scalable logging and metrics processor and forwarder. endef +define Package/$(PKG_NAME)/config + source "$(SOURCE)/Config.in" +endef + ifeq ($(LOCAL_DEV),1) define Build/Prepare $(CP) -rf ./fluent-bit/* $(PKG_BUILD_DIR)/ @@ -57,6 +61,12 @@ CMAKE_OPTIONS+= \ -DFLB_WASM=No \ -DFLB_LUAJIT=No +ifeq ($(CONFIG_FLUENT_BIT_SYSLOG_PREPEND_LENGTH),y) + CMAKE_OPTIONS += -DFLUENT_BIT_SYSLOG_PREPEND_LENGTH=Yes +else + CMAKE_OPTIONS += -DFLUENT_BIT_SYSLOG_PREPEND_LENGTH=No +endif + # In plugins CMAKE_OPTIONS += \ -DFLB_IN_SYSLOG=Yes \ @@ -170,6 +180,7 @@ CMAKE_OPTIONS += \ define Package/fluent-bit/install $(INSTALL_DIR) $(1)/usr/sbin $(INSTALL_DIR) $(1)/etc/fluent-bit + $(INSTALL_DIR) $(1)/etc/fluent-bit/conf.d $(INSTALL_BIN) $(PKG_BUILD_DIR)/bin/fluent-bit $(1)/usr/sbin/ $(INSTALL_DATA) ./files/fluent-bit.conf $(1)/etc/fluent-bit/fluent-bit.conf $(INSTALL_DATA) $(PKG_BUILD_DIR)/conf/parsers.conf $(1)/etc/fluent-bit/parsers.conf diff --git a/fluent-bit/patches/0003-prepend_number_in_syslog_output.patch b/fluent-bit/patches/0003-prepend_number_in_syslog_output.patch new file mode 100644 index 000000000..2859500d3 --- /dev/null +++ b/fluent-bit/patches/0003-prepend_number_in_syslog_output.patch @@ -0,0 +1,71 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 72b950e7b..58e76b349 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -205,6 +205,9 @@ option(FLB_CUSTOM_CALYPTIA "Enable Calyptia Support" Yes) + # Config formats + option(FLB_CONFIG_YAML "Enable YAML config format" Yes) + ++# out syslog message format ++option(FLUENT_BIT_SYSLOG_PREPEND_LENGTH "Enable prepending of length to syslog message" Yes) ++ + # List of plugins available and defaults for each option + include(cmake/plugins_options.cmake) + +diff --git a/plugins/out_syslog/CMakeLists.txt b/plugins/out_syslog/CMakeLists.txt +index 556d8e1a4..a848020e9 100644 +--- a/plugins/out_syslog/CMakeLists.txt ++++ b/plugins/out_syslog/CMakeLists.txt +@@ -1,3 +1,7 @@ ++if(FLUENT_BIT_SYSLOG_PREPEND_LENGTH) ++ add_definitions(-DFLUENT_BIT_SYSLOG_PREPEND_LENGTH) ++endif() ++ + set(src + syslog.c + syslog_conf.c) +diff --git a/plugins/out_syslog/syslog.c b/plugins/out_syslog/syslog.c +index 4ecc7c4ac..f85c38b51 100644 +--- a/plugins/out_syslog/syslog.c ++++ b/plugins/out_syslog/syslog.c +@@ -156,7 +156,7 @@ static flb_sds_t syslog_rfc5424(flb_sds_t *s, struct flb_time *tms, + return NULL; + } + *s = tmp; +- return *s; ++ goto finalize; + } + + prival = (msg->facility << 3) + msg->severity; +@@ -291,6 +291,31 @@ static flb_sds_t syslog_rfc5424(flb_sds_t *s, struct flb_time *tms, + *s = tmp; + } + ++finalize: ++#ifdef FLUENT_BIT_SYSLOG_PREPEND_LENGTH ++ // Create new SDS with length prefix ++ // Allocate enough space for "%u " ++ flb_sds_t prefix = flb_sds_create_size(32); ++ if (!prefix) { ++ return NULL; ++ } ++ ++ unsigned int msg_len = flb_sds_len(*s); ++ prefix = flb_sds_printf(&prefix, "%u ", msg_len); ++ if (!prefix) { ++ return NULL; ++ } ++ ++ flb_sds_t new_s = flb_sds_cat(prefix, *s, msg_len); ++ if (!new_s) { ++ flb_sds_destroy(prefix); ++ return NULL; ++ } ++ ++ flb_sds_destroy(*s); ++ *s = new_s; ++#endif ++ + return *s; + } + diff --git a/iopsys-analytics/Makefile b/iopsys-analytics/Makefile index 94792cbac..c256b445b 100644 --- a/iopsys-analytics/Makefile +++ b/iopsys-analytics/Makefile @@ -31,7 +31,7 @@ define Package/$(PKG_NAME) +@PACKAGE_COLLECTD_ENCRYPTED_NETWORK \ # remote syslog DEPENDS+= \ - +syslog-ng \ + +@FLUENT_BIT_SYSLOG_PREPEND_LENGTH \ +@SYSLOGNG_LOGROTATE \ endef @@ -44,7 +44,12 @@ endef Build/Compile= define Package/$(PKG_NAME)/install - $(CP) -r $(PKG_BUILD_DIR)/files/* $(1)/ + $(CP) -r ./files/common/* $(1)/ +ifneq ($(CONFIG_PACKAGE_fluent-bit),) + $(CP) -r ./files/fluent-bit/* $(1)/ +else + $(CP) -r ./files/syslog-ng/* $(1)/ +endif endef $(eval $(call BuildPackage,$(PKG_NAME))) diff --git a/iopsys-analytics/files/common/etc/analytics-cert/clientcert.pem b/iopsys-analytics/files/common/etc/analytics-cert/clientcert.pem new file mode 100644 index 000000000..2411c0de3 --- /dev/null +++ b/iopsys-analytics/files/common/etc/analytics-cert/clientcert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJTCCAg0CAQEwDQYJKoZIhvcNAQELBQAwRTELMAkGA1UEBhMCU0UxEzARBgNV +BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 +ZDAeFw0yMjExMjIxMjM1MDNaFw0zMjExMTkxMjM1MDNaMGwxCzAJBgNVBAYTAlVT +MREwDwYDVQQIDAhJbnRlcm5ldDEMMAoGA1UEBwwDRFVUMQwwCgYDVQQKDAMuLi4x +DDAKBgNVBAsMAy4uLjEMMAoGA1UEAwwDLi4uMRIwEAYJKoZIhvcNAQkBFgMuLi4w +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDleMMuJ8AO1DeRmOyE2k2D +DWUKB4yiRQhKxnRyRcF1s+mHcySS1NRlLwyZUuOFvBYHUhE3AfNMN5ywKazl4FwG +MNtx0iUfHmHc0MQ04KZk1ifkS5WPUMksEJDQyM06ZH3gcIhGDlSkprnMsp9q5HlG +umopBlihLTI+WM+8kliFbT7iOICDi9qRHPS+WTT/RClX12ISAejVFMFdpf8aQOGn +klu7mcGyOBij0axcfd2QlHybvkSSksvny19I1/Be7MdtO3fxJODuUy8sHd9HqaOJ +TbjcwWNWFJIR4+Gsz2DeqIPxpaqUjEZgJfUD5FSTOL3ZgtjMmzjYZj8IEf5BD7st +AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAK+WvRYkkM+11ljo1pKwyTtcyz9fiQ9Z +zGAed6UViEPhJNIchFEFukMMqNaWL3Pi0MkvkAiJvyJbGW0YiPm7r0sf2sOD4PPI +UqDskApA9P1aL1uF/CBvP7kViVVVcNDSHq+Hc+6Jrg+Ufycq2kE9KrU/zI2Z0kEd +70k+YNLWTZNjekXfrxlniw7ycmALJuYkw1GXSZ4pph+VXYPJ7c/sm4J3MKDXonTQ +tEI4wTTkfOHUBDppNX/b9gw0A4jL5nAga/6Q2PPNoaTrgLZS0o03GO3HES24iHlG +TBrIW9q2HRcDOfj5oNaoSasOyARFT0owJdcBFlzp7Pe1lMN+bRLz5h0= +-----END CERTIFICATE----- diff --git a/iopsys-analytics/files/common/etc/analytics-cert/clientkey.pem b/iopsys-analytics/files/common/etc/analytics-cert/clientkey.pem new file mode 100644 index 000000000..b375bc913 --- /dev/null +++ b/iopsys-analytics/files/common/etc/analytics-cert/clientkey.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDleMMuJ8AO1DeR +mOyE2k2DDWUKB4yiRQhKxnRyRcF1s+mHcySS1NRlLwyZUuOFvBYHUhE3AfNMN5yw +Kazl4FwGMNtx0iUfHmHc0MQ04KZk1ifkS5WPUMksEJDQyM06ZH3gcIhGDlSkprnM +sp9q5HlGumopBlihLTI+WM+8kliFbT7iOICDi9qRHPS+WTT/RClX12ISAejVFMFd +pf8aQOGnklu7mcGyOBij0axcfd2QlHybvkSSksvny19I1/Be7MdtO3fxJODuUy8s +Hd9HqaOJTbjcwWNWFJIR4+Gsz2DeqIPxpaqUjEZgJfUD5FSTOL3ZgtjMmzjYZj8I +Ef5BD7stAgMBAAECggEATmdMiOCcBnUL53tQlLPh9/0DoMGjk23yv1m/P6wFxamL +CPSWcnS1fTzuGRTxGCiDfkzUBO9tqYT2zFnw8ToPNy5KTEF43YEL/t51q3t2Xs2t +I5uPpj/7/EpMsIlrVaAwQhcw+H68/OTaZs5BGcymgzv+ObYpYiiHGjZOBCgchrl+ +/FyUyrS1M11VcamJ8H6GqNDEy5zvwOX3VjIwHvbONrN6zjNvMRIMzCQC59hEeBDG +j6AYMw2/UJ4sPLpmqObnPvYO4D6qsEuKxQDGFwaPGSulOs5+OHMqTcTLRtktR5qJ +Mjq5C2axy3TvxpbZ5Ou2jRB/zaXWNkK26UP5Iv2MAQKBgQD/4J5lcFBkrZArR5XL +FiAfN4yfBAPcveRP7Zgqb7nlLzzouqz7vyH8Hhnk+XbA9CiKVOMc2Fbe82/exuft +DvsoW5+Ds31SU52q5vWjtnPXeuK0kT6lm+qBbtmsFSAnfCRZUQcF8PhERLfzOEA6 +Uln0xr2EFR3D/UaBjsN3aTKMzQKBgQDllOe+QAf4CZGgCyAF1DDakuZF+iAfsCNz +83dlb6N2ggJinOLs3ExfT0oJSUp4KPAIOQ+QgY01Itk0py2HEfYCgvXim7RrZ1P7 +5Sx++fi48ar3aqgLqg0hur0Yd1kFPjVRWj2/jM6qvLC8nEehNQH1VOh847FcwTrY +UWWMKOnn4QKBgFnApE7Ykpp99u/3cCeIX0QT7BNrCR/Iy+jtYk6gWgh2BVts/3he +lsEkndifWHReMEo1dGSNzgruQPNXucqBUr/ZQ56iuvbBtbBYyxjdysxuok7ME7lr +c1prU5b04g53kRhzP49p8uufXxbFSGIsuLkPbTKDZal1keGn/hBIqh6RAoGAT9pS +HMdSvkA9yaf09HIQqAXOWGqef8cZHbyCO+LvvUmNINWmzHH3knMWslIut8qtan7G +XiHhhqjfWcQIPuHNaqQ4eCBmSbE/8e6NYoD5tQJB0Brl5mC4fpHgMcFZUkyH6Qr1 +TUnhnFtTMdc9xuGUE8v7rWzZBQUkvZbaNj0I1qECgYAesALh2u0r/ayEYjWAcqu8 +XXhQWRhg7fXm0cCODNeK5rhkaqxlw64hSY29tgEKPg/F66jXMDXcsvsBwnPoGN35 +lD6CQlWKR3g2LtRCAHLICQ5aODNL9B321syoIHF1Cdlw78V3uMsQ7av/IWaIX4BL +EEWtDICxzN37PaGuYVSRSQ== +-----END PRIVATE KEY----- diff --git a/iopsys-analytics/files/common/etc/collectd.conf b/iopsys-analytics/files/common/etc/collectd.conf new file mode 100644 index 000000000..2a2f93d49 --- /dev/null +++ b/iopsys-analytics/files/common/etc/collectd.conf @@ -0,0 +1,93 @@ +BaseDir "/var/lib/collectd" +PIDFile "/var/run/collectd.pid" +Interval 60 +ReadThreads 2 +WriteQueueLimitHigh 10000 +WriteQueueLimitLow 8000 + +LoadPlugin "load" +LoadPlugin "network" +LoadPlugin "memory" +LoadPlugin "processes" +LoadPlugin "cpu" +LoadPlugin "exec" + + + + SecurityLevel "encrypt" + Username "iopsys" + Password "kffJDv3CMVO9WHO4CIP8CGM3ZXN4EQv" + + + + + + SecurityLevel "encrypt" + Username "iopsys" + Password "6rgVOHWq9BPCbZFlLVB854jD8WCv4EvK" + + + + + ValuesPercentage true + + + + CollectFileDescriptor true + CollectContextSwitch true + CollectMemoryMaps true + + ProcessMatch "mapagent_instances" "mapagent" + ProcessMatch "wifimngr_instances" "wifimngr" + ProcessMatch "ieee1905d_instances" "ieee1905d" + ProcessMatch "topologyd_topologyd" "topologyd" + ProcessMatch "uspd_instances" "uspd" + ProcessMatch "mapcontroller_instances" "mapcontroller" + ProcessMatch "obuspa_instances" "(.*)obuspa" + ProcessMatch "hostapd_instances" "hostapd" + ProcessMatch "swmodd_instances" "swmodd" + ProcessMatch "ruleng_instances" "rulengd" + ProcessMatch "wpa_supplicant_instances" "wpa_supplicant" + ProcessMatch "dm_bulkdata" "bbfdmd -m bulkdata" + ProcessMatch "dm_ddnsmngr" "bbfdmd -m ddnsmngr" + ProcessMatch "dm_dnsmngr" "bbfdmd -m dnsmngr" + ProcessMatch "dm_ethmngr" "bbfdmd -m ethmngr" + ProcessMatch "dm_hostmngr" "bbfdmd -m hostmngr" + ProcessMatch "dm_icwmp" "bbfdmd -m icwmp" + ProcessMatch "dm_mcastmngr" "bbfdmd -m mcastmngr" + ProcessMatch "dm_periodicstat" "bbfdmd -m periodicstats" + ProcessMatch "dm_portmapping" "bbfdmd -m portmapping" + ProcessMatch "dm_qosmngr" "bbfdmd -m qosmngr" + ProcessMatch "dm_sshmngr" "bbfdmd -m sshmngr" + ProcessMatch "dm_swmodd" "bbfdmd -m swmodd" + ProcessMatch "dm_timemngr" "bbfdmd -m timemngr" + ProcessMatch "dm_tr104" "bbfdmd -m tr104" + ProcessMatch "dm_urlfilter" "bbfdmd -m urlfilter" + ProcessMatch "dm_userinterfac" "bbfdmd -m userinterface" + ProcessMatch "dm_usermngr" "bbfdmd -m usermngr" + ProcessMatch "bbfdmd" "bbfdmd" + ProcessMatch "decollector" "decollector" + ProcessMatch "collectd" "collectd" + ProcessMatch "uwsgi" "uwsgi" + ProcessMatch "syslog-ng" "syslog-ng" + ProcessMatch "mosquitto" "mosquitto" + ProcessMatch "nginx" "nginx" + ProcessMatch "hostmngr" "hostmngr" + ProcessMatch "asterisk" "asterisk" + ProcessMatch "voicemngr" "voicemngr" + + + + + + ReportByState true + ReportByCpu true + ValuesPercentage true + + + +Exec "nobody" "/sbin/get-sn" + + + + diff --git a/iopsys-analytics/files/common/etc/config/core-watcher b/iopsys-analytics/files/common/etc/config/core-watcher new file mode 100644 index 000000000..c798cd7dd --- /dev/null +++ b/iopsys-analytics/files/common/etc/config/core-watcher @@ -0,0 +1,3 @@ +config core-watcher 'globals' + option api_key '1JH7ZKGDtJXKeLt4CfeJt4y' + option url 'https://analytics.iopsys.eu:1337/crash_upload_file' diff --git a/iopsys-analytics/files/common/etc/icwmpd/force_inform.json b/iopsys-analytics/files/common/etc/icwmpd/force_inform.json new file mode 100644 index 000000000..698b561f7 --- /dev/null +++ b/iopsys-analytics/files/common/etc/icwmpd/force_inform.json @@ -0,0 +1,6 @@ +{ + "forced_inform": [ + "Device.DeviceInfo.ModelName" + ] +} + diff --git a/iopsys-analytics/files/common/etc/init.d/core-watcher b/iopsys-analytics/files/common/etc/init.d/core-watcher new file mode 100755 index 000000000..b828362d5 --- /dev/null +++ b/iopsys-analytics/files/common/etc/init.d/core-watcher @@ -0,0 +1,38 @@ +#!/bin/sh /etc/rc.common + +export USE_PROCD=1 + +export START=95 +export STOP=01 + +LOG="logger -t core-watcher[$$] -p" + +start_service() { + + [ -f /etc/config/core-watcher ] || { + $LOG error "UCI config not found" + return 1 + } + + firmware_version=$(db get device.deviceinfo.SoftwareVersion || echo "unknown") + product_class=$(db get device.deviceinfo.ProductClass | awk '{print tolower($0)}') + serial=$(db get device.deviceinfo.SerialNumber || echo "unknown") + config_load core-watcher + + url="" + config_get url "globals" "url" + api_key="" + config_get api_key "globals" "api_key" + + if [ -z "$url" ] || [ -z "$api_key" ]; then + $LOG error "URL or API Key not found in config" + return 2 + fi + + procd_open_instance + procd_set_param env API_KEY="$api_key" SERIAL="$serial" FIRMWARE_VERSION="$firmware_version" PRODUCT_CLASS="$product_class" + procd_set_param command /usr/bin/core-watcher -u "$url" + procd_set_param limits core="unlimited" + procd_close_instance +} + diff --git a/iopsys-analytics/files/common/etc/init.d/ubus-monitor b/iopsys-analytics/files/common/etc/init.d/ubus-monitor new file mode 100755 index 000000000..1803d5ad6 --- /dev/null +++ b/iopsys-analytics/files/common/etc/init.d/ubus-monitor @@ -0,0 +1,26 @@ +#!/bin/sh /etc/rc.common + +START=15 +STOP=20 + +USE_PROCD=1 + +start_service() { + procd_open_instance + + echo "Starting ubus monitor..." + + procd_set_param respawn # Enable automatic restart on failure + procd_set_param stdout 1 # Redirect stdout to procd + procd_set_param stderr 1 # Redirect stderr to procd + procd_set_param command /bin/sh "/usr/sbin/ubus-monitor" + + + procd_close_instance +} + +reload_service(){ + echo "Explicitly restarting service, are you sure you need this?" + stop + start +} diff --git a/iopsys-analytics/files/common/etc/uci-defaults/89-cwmp-setforceinform b/iopsys-analytics/files/common/etc/uci-defaults/89-cwmp-setforceinform new file mode 100644 index 000000000..ac0a2b0b2 --- /dev/null +++ b/iopsys-analytics/files/common/etc/uci-defaults/89-cwmp-setforceinform @@ -0,0 +1,3 @@ +#!/bin/sh +uci set cwmp.cpe.forced_inform_json='/etc/icwmpd/force_inform.json' +uci commit cwmp diff --git a/iopsys-analytics/files/common/lib/preinit/95_set_core_limit_unlimited b/iopsys-analytics/files/common/lib/preinit/95_set_core_limit_unlimited new file mode 100644 index 000000000..5df8f035f --- /dev/null +++ b/iopsys-analytics/files/common/lib/preinit/95_set_core_limit_unlimited @@ -0,0 +1,5 @@ +set_core_unlimited() { + prlimit --core=unlimited: -p 1 +} + +boot_hook_add preinit_main set_core_unlimited diff --git a/iopsys-analytics/files/common/sbin/get-sn b/iopsys-analytics/files/common/sbin/get-sn new file mode 100755 index 000000000..c0dba204a --- /dev/null +++ b/iopsys-analytics/files/common/sbin/get-sn @@ -0,0 +1,11 @@ +#!/bin/sh + +INTERVAL="${COLLECTD_INTERVAL:-60}" + +while sleep "$INTERVAL"; do + SN=$(db get device.deviceinfo.SerialNumber) + HOSTNAME=$(uname -n) + FORMATTED_RESULT=$(echo "1337") + echo "PUTVAL \"$HOSTNAME/exec-$SN/gauge-INFO\" interval=$INTERVAL N:\"$FORMATTED_RESULT\"" + +done diff --git a/iopsys-analytics/files/common/usr/bin/core-watcher b/iopsys-analytics/files/common/usr/bin/core-watcher new file mode 100755 index 000000000..9231e8f65 --- /dev/null +++ b/iopsys-analytics/files/common/usr/bin/core-watcher @@ -0,0 +1,150 @@ +#!/bin/sh + +# shellcheck shell=busybox + +# This shell script watches for *.core files created in +# /tmp and uploads them + +CORE_DIR="/tmp" + +LOG="logger -t core-watcher[$$] -p" +POLL_INTERVAL="5" + +upload_file() { + FILEPATH=$1 + $LOG info "Uploading $UPLOAD_TYPE $FILEPATH" + + curl -s --fail -X "POST" \ + "$API_URL" \ + -H "accept: application/json" \ + -H "Content-Type: multipart/form-data" \ + -F "api_key=$API_KEY" \ + -F "dump_md5=$COREDUMP_MD5" \ + -F "device_serial=$SERIAL" \ + -F "upload_type=$UPLOAD_TYPE" \ + -F "incoming_file=@$FILEPATH;type=text/plain" + + status=$? + + if [ "$status" != "0" ]; then + $LOG error "$UPLOAD_TYPE upload failed. curl returned status $status" + else + $LOG info "$UPLOAD_TYPE upload success" + if [ $UPLOAD_TYPE = "core" ]; then + $LOG info "Renaming core $ORIGINAL_CORE_FILEPATH" + mv "$ORIGINAL_CORE_FILEPATH" "$ORIGINAL_CORE_FILEPATH"_UPLOADED + fi + fi +} + +usage() { + cat < API Gateway URL (required) +EOM + exit 1 +} + +main() { + if [ -z "$API_KEY" ]; then + $LOG error "API_KEY env variable not set" + exit 1 + fi + + if [ -z "$SERIAL" ]; then + $LOG error "SERIAL env variable not set" + exit 1 + fi + + if [ -z "$FIRMWARE_VERSION" ]; then + $LOG error "FIRMWARE_VERSION env variable not set" + exit 1 + fi + + while getopts "u:" opt; do + case $opt in + u) + API_URL=${OPTARG} + ;; + *) + usage + ;; + esac + done + + if [ -z "$API_URL" ]; then + usage + fi + # Source system information from /etc/os-release + . /etc/os-release + + # Convert OPENWRT_DEVICE_PRODUCT to uppercase and use it as the board name + BOARD_NAME=$(echo "$OPENWRT_DEVICE_PRODUCT" | tr 'a-z' 'A-Z') + BASE_MAC="$(db get device.deviceinfo.BaseMACAddress)" + + while true; do + if ping -c1 -4 analytics.iopsys.eu; then + CORES=$(find "$CORE_DIR" -type f -name "*.core") + for core in $CORES; do + $LOG error "Core found: $core" + + COREDUMP_MD5="$(md5sum "$core" | cut -f1 -d' ')" + + UPLOAD_TYPE="file" + + if [ -e /tmp/ctfw.meta ]; then + . /tmp/ctfw.meta + fi + + echo "$CTFW_TEST_CASE" > /tmp/ctfw-test-case.txt + upload_file "/tmp/ctfw-test-case.txt" + + echo "$CTFW_TAG" > /tmp/ctfw-job-tag.txt + upload_file "/tmp/ctfw-job-tag.txt" + + echo "$CTFW_LOCK_USERNAME" > /tmp/ctfw-lock-username.txt + upload_file "/tmp/ctfw-lock-username.txt" + + opkg list > /tmp/opkg-list.txt + upload_file "/tmp/opkg-list.txt" + + free > /tmp/free-mem.txt + upload_file "/tmp/free-mem.txt" + + cat /proc/slabinfo > /tmp/slabinfo.txt + upload_file "/tmp/slabinfo.txt" + + echo "$BOARD_NAME" > /tmp/board-name.txt + upload_file "/tmp/board-name.txt" + + echo "$BASE_MAC" > /tmp/base-mac.txt + upload_file "/tmp/base-mac.txt" + + uname -n > /tmp/hostname.txt + upload_file "/tmp/hostname.txt" + + echo "$FIRMWARE_VERSION" > /tmp/fw-version.txt + upload_file "/tmp/fw-version.txt" + + cp "$(strings "$core" | grep ^/ | head -1 | cut -d' ' -f1)" /tmp/binary + upload_file /tmp/binary + + UPLOAD_TYPE="core" + ORIGINAL_CORE_FILEPATH="$core" + cp "$core" /tmp/core + upload_file /tmp/core + rm /tmp/core + done + fi + + sleep $POLL_INTERVAL + done + + +} + +main "$@" + diff --git a/iopsys-analytics/files/common/usr/sbin/system-report b/iopsys-analytics/files/common/usr/sbin/system-report new file mode 100755 index 000000000..5eefde259 --- /dev/null +++ b/iopsys-analytics/files/common/usr/sbin/system-report @@ -0,0 +1,231 @@ +#!/bin/sh +# shellcheck shell=dash + +# Location of system report temporary directory +SRDIR="/tmp/system-report" + + +configure() { + local preconf_path + preconf_path="$(realpath "$0")" + + if ! cmp -s /usr/sbin/system-report "${preconf_path}" >/dev/null; then + echo "iopsys-test system-report differs from included with firmware, symlinking" + ln -sf "${preconf_path}" /usr/sbin/system-report + fi + + if ! cmp -s /rom/usr/sbin/system-report /usr/sbin/system-report >/dev/null && ! grep -Fq "/usr/sbin/system-report" /etc/sysupgrade.conf; then + echo "iopsys-test system-report differs from included with firmware, adding to sysupgrade.conf" + echo "/usr/sbin/system-report" >>/etc/sysupgrade.conf + fi +} + +run() { + set -x + + # Alias ubus to have a smaller 5-second timeout on all subsequent calls + ubus() { command ubus -t 5 "$@"; } + + mkdir -p "$SRDIR" + + SRCF="$SRDIR/config" + SRSY="$SRDIR/system" + SRNT="$SRDIR/network" + SRTR="$SRDIR/tr069" + SRMD="$SRDIR/mem_dump" + + # Config + echo "Gathering configuration data" + { + cat /etc/banner + + echo "Installed Packages" + opkg list-installed + + echo "Database" + db show + + echo "Configuration" + uci show + } > "$SRCF" + + # System + echo "Gathering system information" + { + echo "System Info" + ubus call system info + + echo "Board Info" + ubus call system board + + echo "Running Processes" + top -b -n 1 + + echo "Kernel Parameters" + sysctl -A 2>/dev/null + + echo "System Log" + timeout 5 logread -l 1000 + + echo "Driver Message" + timeout 5 dmesg + + echo "PCI Devices" + lspci -k + + echo "Installed Modules" + lsmod + + echo "CPU Info" + cat /proc/cpuinfo + + echo "Memory Info" + cat /proc/meminfo + + echo "Slab Info" + cat /proc/slabinfo + + echo "Firmware banks" + ubus call fwbank dump + } > "$SRSY" + + # Network + echo "Gathering network state" + { + echo "Device Status" + ubus call network.device status + + echo "Hosts Status" + ubus call hosts show + + echo "Topology" + ubus call ieee1905.topology dump + + echo "Network Status" + ubus call network.interface dump + + if [ -d /sys/class/ieee80211 ]; then + echo "Wireless Status" + ubus call network.wireless status + + echo "Wireless Radio Status" + ubus call wifi status + + echo "Get radio scan" + for radio_if in $(ubus list 'wifi.radio.*'); do + ubus call "${radio_if}" scan + sleep 2 + ubus call "${radio_if}" scanresults + done + + echo "Get Assoc List" + interfaces=$(uci show wireless | grep "ifname=" | awk -F'[.,=]' '{print$2}') + for int in $interfaces; do + mode=$(uci get "wireless.${int}.mode") + if [ "$mode" = "ap" ] ; then + ap_int=$(uci get "wireless.${int}.ifname") + echo "Get assoc list" + ubus call "wifi.ap.${ap_int}" assoclist + echo "Get station info" + ubus call "wifi.ap.${ap_int}" stations + fi + done + + fi + + if [ "$(db get hw.board.hasVoice)" = 1 ]; then + echo "voice status" + ubus call voice.asterisk status + fi + + echo "IPv4 Routes" + ip -d r + + echo "IPv6 Routes" + ip -d -6 r + + echo "Neighbor Table" + ip n + + echo "ARP Table" + cat /proc/net/arp + + echo "IGMP Snooping Table" + ubus call mcast stats + + echo "Conntrack Table" + cat /proc/net/nf_conntrack + + echo "Network Interface Status" + ip -d a + + echo "IPv4 Firewall Status" + iptables -xvnL + + echo "IPv6 Firewall Status" + ip6tables -xvnL + + echo "Bridge Info" + brctl show + + if [ -f /tmp/multiap.backhaul ]; then + echo "Multi-AP Backhaul status" + cat /tmp/multiap.backhaul + fi + + echo "TCP/UDP listened ports" + netstat -tlnp + netstat -ulnp + + echo "MAC layer firewall status" + ebtables -t broute -L + + echo "MAC layer firewall status list" + ebtables -L + + echo "IEEE1905 info" + ubus call ieee1905 info + + if [ -f /usr/bin/iwinfo ]; then + echo "iwinfo interface details" + /usr/bin/iwinfo + fi + + } > "$SRNT" + + # TR-069 + echo "Gathering TR-069 state" + { + echo "TR-069 status" + ubus call tr069 status + + echo "get all TR-181 Parameters" + ubus call bbfdm get '{"path":"Device."}' + + echo "TR-069 Option 43 URL" + uci -P /var/state -q get cwmp.acs.url + + echo "TR-069 Logs" + [ -f /var/log/icwmp ] && cat /var/log/icwmp + } > "$SRTR" + + # Firmware RAM dump (IOP-8118) + if [ -f /tmp/mem_dump ]; then + mv /tmp/mem_dump "$SRMD" + fi + + REPORT_PATH="$(dirname "${SRDIR}")" + REPORT_DIRNAME="$(basename "${SRDIR}")" + FILENAME="${REPORT_PATH}/system-report-$(date +%Y-%m-%d).tar.gz" + tar -cz -C "${REPORT_PATH}" -f "${FILENAME}" "${REPORT_DIRNAME}" + echo "Report is located at ${FILENAME}" + + #ubus send "system-report" "{ \"filename\" : \"$FILENAME\" }" + + rm -r "$SRDIR" +} + +case "${1:-}" in + configure) configure ;; + *) run ;; +esac diff --git a/iopsys-analytics/files/common/usr/sbin/ubus-monitor b/iopsys-analytics/files/common/usr/sbin/ubus-monitor new file mode 100755 index 000000000..427e3fd33 --- /dev/null +++ b/iopsys-analytics/files/common/usr/sbin/ubus-monitor @@ -0,0 +1,7 @@ +#!/bin/ash + +rm -f /tmp/ubus-calls-pipe + +mkfifo /tmp/ubus-calls-pipe || echo "Pipe wasn't created" >&2 +logger -t ubus-logging "Pipe created sucessfully" +/bin/ubus listen > /tmp/ubus-calls-pipe diff --git a/iopsys-analytics/files/fluent-bit/etc/fluent-bit/conf.d/analytics.conf b/iopsys-analytics/files/fluent-bit/etc/fluent-bit/conf.d/analytics.conf new file mode 100644 index 000000000..db0637d0f --- /dev/null +++ b/iopsys-analytics/files/fluent-bit/etc/fluent-bit/conf.d/analytics.conf @@ -0,0 +1,42 @@ +[INPUT] + name syslog + tag logs + path /dev/log + parser syslog-rfc3164-local + +[INPUT] + Name head + Tag ubus_listen + file /tmp/ubus-calls-pipe + Interval_Sec 1 + +[OUTPUT] + name syslog + match logs + host analytics.iopsys.eu + port 6514 + mode tcp + tls on + tls.verify off + tls.key_file /etc/analytics-cert/clientkey.pem + tls.crt_file /etc/analytics-cert/clientcert.pem + syslog_format rfc5424 + syslog_appname_preset dummy + syslog_message_key message + syslog_hostname_key hostname + syslog_hostname_preset dummy + +[OUTPUT] + name syslog + match ubus_listen + host analytics.iopsys.eu + port 6515 + mode tcp + tls on + tls.verify off + tls.key_file /etc/analytics-cert/clientkey.pem + tls.crt_file /etc/analytics-cert/clientcert.pem + syslog_format rfc5424 + syslog_appname_preset dummy + syslog_message_key head + syslog_hostname_preset dummy diff --git a/iopsys-analytics/files/fluent-bit/etc/uci-defaults/01-fluent-bit-init b/iopsys-analytics/files/fluent-bit/etc/uci-defaults/01-fluent-bit-init new file mode 100644 index 000000000..9a490fcb6 --- /dev/null +++ b/iopsys-analytics/files/fluent-bit/etc/uci-defaults/01-fluent-bit-init @@ -0,0 +1,25 @@ +#!/bin/sh + +# Check if fluent-bit is installed +if command -v fluent-bit >/dev/null 2>&1; then + conf_file="/etc/fluent-bit/conf.d/analytics.conf" + + # Check if the configuration file exists + if [ -f "$conf_file" ]; then + # Get hostname and serial number + hostname="$(db -q get device.deviceinfo.ModelName)" + serialnum="$(db -q get device.deviceinfo.SerialNumber)" + + # Replace syslog_hostname_preset line + sed -i "s/^.*syslog_hostname_preset.*/ syslog_hostname_preset ${hostname}/" "$conf_file" + + # Replace syslog_appname_preset line + sed -i "s/^.*syslog_appname_preset.*/ syslog_appname_preset ${serialnum}/" "$conf_file" + fi +fi + +# Clear logs periodically +grep -q 'logrotate' /etc/crontabs/root || echo '*/15 * * * * /usr/sbin/logrotate /etc/logrotate.conf' >> /etc/crontabs/root + +# Exit successfully +exit 0 diff --git a/iopsys-analytics/files/syslog-ng/etc/logrotate.d/analytics b/iopsys-analytics/files/syslog-ng/etc/logrotate.d/analytics new file mode 100644 index 000000000..68ef2a1e5 --- /dev/null +++ b/iopsys-analytics/files/syslog-ng/etc/logrotate.d/analytics @@ -0,0 +1,9 @@ + +/var/log/messages { +    missingok +    rotate 1 +    size 1M +    postrotate +        /usr/sbin/syslog-ng-ctl reopen +    endscript +} diff --git a/iopsys-analytics/files/syslog-ng/etc/syslog-ng.d/analytics b/iopsys-analytics/files/syslog-ng/etc/syslog-ng.d/analytics new file mode 100644 index 000000000..3c618b49f --- /dev/null +++ b/iopsys-analytics/files/syslog-ng/etc/syslog-ng.d/analytics @@ -0,0 +1,46 @@ +source s_file { + file("/var/log/messages" program_override("`ENV_VAR_SYSLOG_NG_DUT_SERIAL_NUMBER`")); +}; + +source ubus_calls { + pipe("/tmp/ubus-calls-pipe" + flags(no-parse) + ); + +}; + +destination d_tls_syslog { + syslog( + "analytics.iopsys.eu" + transport("tls") + port(6514) + tls( + key-file("/etc/analytics-cert/clientkey.pem") + cert-file("/etc/analytics-cert/clientcert.pem") + peer-verify(no) + ) + ); +}; + +destination d_tls_ubus { + syslog( + "analytics.iopsys.eu" + transport("tls") + port(6515) + tls( + key-file("/etc/analytics-cert/clientkey.pem") + cert-file("/etc/analytics-cert/clientcert.pem") + peer-verify(no) + ) + ); +}; + +log { + source(s_file); + destination(d_tls_syslog); +}; + +log { + source(ubus_calls); + destination(d_tls_ubus); +}; diff --git a/iopsys-analytics/files/syslog-ng/etc/uci-defaults/01-syslog-ng-init b/iopsys-analytics/files/syslog-ng/etc/uci-defaults/01-syslog-ng-init new file mode 100644 index 000000000..7fb43093b --- /dev/null +++ b/iopsys-analytics/files/syslog-ng/etc/uci-defaults/01-syslog-ng-init @@ -0,0 +1,4 @@ +sed -i '/procd_open_instance.*/a \\tprocd_set_param env ENV_VAR_SYSLOG_NG_DUT_SERIAL_NUMBER="$(db get hw.board.serial_number)"' /etc/init.d/syslog-ng + +# Clear logs periodically +grep -q 'logrotate' /etc/crontabs/root || echo '*/15 * * * * /usr/sbin/logrotate /etc/logrotate.conf' >> /etc/crontabs/root diff --git a/logmngr/files/lib/logmngr/fluent-bit.sh b/logmngr/files/lib/logmngr/fluent-bit.sh index 947b08164..30c7352a5 100644 --- a/logmngr/files/lib/logmngr/fluent-bit.sh +++ b/logmngr/files/lib/logmngr/fluent-bit.sh @@ -5,6 +5,7 @@ CONF_FILE=/etc/fluent-bit/fluent-bit.conf TMP_CONF_FILE=/tmp/fluent-bit/fluent-bit.conf +CONF_DIR=/etc/fluent-bit/conf.d append_conf() { echo "$*" >> ${TMP_CONF_FILE} @@ -14,6 +15,14 @@ create_config_file() { mkdir -p /tmp/fluent-bit rm -f ${TMP_CONF_FILE} touch ${TMP_CONF_FILE} + # include all files placed in analytics directory + # fluent-bit does not support using directory in include directive + # also, if no file is found then fluent-bit aborts + # so only add include if any file is present in the CONF_DIR + if [ -d "$CONF_DIR" ] && [ "$(ls -A "$CONF_DIR")" ]; then + echo "@INCLUDE ${CONF_DIR}/*" >> ${TMP_CONF_FILE} + fi + echo "" >> ${TMP_CONF_FILE} } create_service_section() {