From 8bd2d743169388b95fdaba8d113ff42cb2d9baf4 Mon Sep 17 00:00:00 2001 From: Suvendhu Hansa Date: Wed, 10 Jan 2024 14:03:21 +0000 Subject: [PATCH] obuspa: Add CSV report compliance to RFC4180 for bulkdata --- obuspa/patches/0007-bdc_csv_format.patch | 171 ++++++++++++++---- ...iling-version-of-asprintf-and-use-it.patch | 121 ------------- 2 files changed, 136 insertions(+), 156 deletions(-) delete mode 100644 obuspa/patches/0009-Implement-non-failing-version-of-asprintf-and-use-it.patch diff --git a/obuspa/patches/0007-bdc_csv_format.patch b/obuspa/patches/0007-bdc_csv_format.patch index 82e26f216..207d039f5 100644 --- a/obuspa/patches/0007-bdc_csv_format.patch +++ b/obuspa/patches/0007-bdc_csv_format.patch @@ -1,6 +1,6 @@ --- a/src/core/bdc_exec.c +++ b/src/core/bdc_exec.c -@@ -547,11 +547,19 @@ int StartSendingReport(bdc_connection_t +@@ -547,11 +547,19 @@ // Set the list of headers bc->headers = NULL; @@ -25,7 +25,7 @@ { --- a/src/core/bdc_exec.h +++ b/src/core/bdc_exec.h -@@ -53,6 +53,9 @@ void BDC_EXEC_ScheduleExit(void); +@@ -53,6 +53,9 @@ #define BDC_FLAG_PUT 0x00000001 // If set, HTTP PUT should be used instead of HTTP POST when sending the report to the BDC server #define BDC_FLAG_GZIP 0x00000002 // If set, the reports contants are Gzipped #define BDC_FLAG_DATE_HEADER 0x00000004 // If set, the date header should be included in the HTTP post. @@ -52,7 +52,7 @@ // Definitions for Device.BulkData.Profile.{i}.JSONEncoding.ReportTimestamp -@@ -153,6 +156,7 @@ static char *profile_push_event_args[] = +@@ -153,6 +156,7 @@ typedef struct { int num_retained_failed_reports; @@ -60,7 +60,7 @@ char report_timestamp[33]; char url[1025]; char username[257]; -@@ -161,6 +165,11 @@ typedef struct +@@ -161,6 +165,11 @@ char method[9]; bool use_date_header; char report_format[20]; @@ -72,7 +72,7 @@ } profile_ctrl_params_t; //------------------------------------------------------------------------------ -@@ -208,6 +217,7 @@ int Validate_BulkDataEncodingType(dm_req +@@ -208,6 +217,7 @@ int Validate_BulkDataReportingInterval(dm_req_t *req, char *value); int Validate_BulkDataReference(dm_req_t *req, char *value); int Validate_BulkDataReportFormat(dm_req_t *req, char *value); @@ -80,7 +80,7 @@ int Validate_BulkDataReportTimestamp(dm_req_t *req, char *value); int Validate_BulkDataCompression(dm_req_t *req, char *value); int Validate_BulkDataHTTPMethod(dm_req_t *req, char *value); -@@ -236,6 +246,8 @@ bulkdata_profile_t *bulkdata_find_profil +@@ -236,6 +246,8 @@ int bulkdata_calc_report_map(bulkdata_profile_t *bp, kv_vector_t *report_map); int bulkdata_reduce_to_alt_name(char *spec, char *path, char *alt_name, char *out_buf, int buf_len); char *bulkdata_generate_json_report(bulkdata_profile_t *bp, char *report_timestamp, char *report_format); @@ -89,15 +89,16 @@ unsigned char *bulkdata_compress_report(profile_ctrl_params_t *ctrl, char *input_buf, int input_len, int *p_output_len); int bulkdata_schedule_sending_http_report(profile_ctrl_params_t *ctrl, bulkdata_profile_t *bp, unsigned char *json_report, int report_len); int bulkdata_start_profile(bulkdata_profile_t *bp); -@@ -250,6 +262,7 @@ char *bulkdata_platform_calc_uri_query_s +@@ -250,6 +262,8 @@ int bulkdata_platform_get_param_refs(int profile_id, param_ref_vector_t *param_refs); void bulkdata_expand_param_ref(param_ref_entry_t *pr, group_get_vector_t *ggv); void bulkdata_append_to_result_map(param_ref_entry_t *pr, group_get_vector_t *ggv, kv_vector_t *report_map); +void append_string_to_target(char *str, char **output); ++char *csv_encode(const char *str); /*********************************************************************//** ** -@@ -282,7 +295,7 @@ int DEVICE_BULKDATA_Init(void) +@@ -282,7 +296,7 @@ err |= USP_REGISTER_VendorParam_ReadOnly("Device.BulkData.Status", Get_BulkDataGlobalStatus, DM_STRING); err |= USP_REGISTER_Param_Constant("Device.BulkData.MinReportingInterval", BULKDATA_MINIMUM_REPORTING_INTERVAL_STR, DM_UINT); err |= USP_REGISTER_Param_SupportedList("Device.BulkData.Protocols", bdc_protocols, NUM_ELEM(bdc_protocols)); @@ -106,7 +107,7 @@ err |= USP_REGISTER_Param_Constant("Device.BulkData.ParameterWildCardSupported", "true", DM_BOOL); err |= USP_REGISTER_Param_Constant("Device.BulkData.MaxNumberOfProfiles", BULKDATA_MAX_PROFILES_STR, DM_INT); err |= USP_REGISTER_Param_Constant("Device.BulkData.MaxNumberOfParameterReferences", "-1", DM_INT); -@@ -297,7 +310,7 @@ int DEVICE_BULKDATA_Init(void) +@@ -297,7 +311,7 @@ err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.Name", "", NULL, NULL, DM_STRING); err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.NumberOfRetainedFailedReports", "0", Validate_NumberOfRetainedFailedReports, NULL, DM_INT); err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.Protocol", BULKDATA_PROTOCOL_HTTP, Validate_BulkDataProtocol, NULL, DM_STRING); @@ -115,7 +116,7 @@ err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.ReportingInterval", "86400", Validate_BulkDataReportingInterval, NotifyChange_BulkDataReportingInterval, DM_UINT); err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.TimeReference", UNKNOWN_TIME_STR, NULL, NotifyChange_BulkDataTimeReference, DM_DATETIME); -@@ -312,6 +325,13 @@ int DEVICE_BULKDATA_Init(void) +@@ -312,6 +326,13 @@ err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.JSONEncoding.ReportFormat", BULKDATA_JSON_REPORT_FORMAT_NAME_VALUE, Validate_BulkDataReportFormat, NULL, DM_STRING); err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.JSONEncoding.ReportTimestamp", BULKDATA_JSON_TIMESTAMP_FORMAT_EPOCH, Validate_BulkDataReportTimestamp, NULL, DM_STRING); @@ -129,7 +130,7 @@ // Device.BulkData.Profile.{i}.HTTP err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.HTTP.URL", "", NULL, NotifyChange_BulkDataURL, DM_STRING); err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.HTTP.Username", "", NULL, NULL, DM_STRING); -@@ -591,9 +611,10 @@ int Validate_BulkDataProtocol(dm_req_t * +@@ -591,9 +612,10 @@ int Validate_BulkDataEncodingType(dm_req_t *req, char *value) { // Exit if trying to set a value outside of the range we accept @@ -142,7 +143,7 @@ return USP_ERR_INVALID_VALUE; } -@@ -676,6 +697,32 @@ int Validate_BulkDataReportFormat(dm_req +@@ -676,6 +698,32 @@ /*********************************************************************//** ** @@ -175,7 +176,7 @@ ** Validate_BulkDataReportTimestamp ** ** Validates Device.BulkData.Profile.{i}.JSONEncoding.ReportTimestamp -@@ -1970,6 +2017,14 @@ int bulkdata_platform_get_profile_contro +@@ -1970,6 +2018,14 @@ return err; } @@ -190,7 +191,7 @@ // Exit if unable to get ReportTimestamp USP_SNPRINTF(path, sizeof(path), "Device.BulkData.Profile.%d.JSONEncoding.ReportTimestamp", bp->profile_id); err = DATA_MODEL_GetParameterValue(path, ctrl_params->report_timestamp, sizeof(ctrl_params->report_timestamp), 0); -@@ -1986,6 +2041,46 @@ int bulkdata_platform_get_profile_contro +@@ -1986,6 +2042,46 @@ return err; } @@ -237,7 +238,7 @@ return USP_ERR_OK; } -@@ -2222,7 +2317,7 @@ void bulkdata_process_profile_http(bulkd +@@ -2222,7 +2318,7 @@ { int err; report_t *cur_report; @@ -246,7 +247,7 @@ profile_ctrl_params_t ctrl; unsigned char *compressed_report; int compressed_len; -@@ -2261,10 +2356,23 @@ void bulkdata_process_profile_http(bulkd +@@ -2261,10 +2357,23 @@ } // Exit if unable to generate the report @@ -274,7 +275,7 @@ return; } -@@ -2273,14 +2381,14 @@ void bulkdata_process_profile_http(bulkd +@@ -2273,14 +2382,14 @@ USP_LOG_Info("BULK DATA: using compression method=%s", ctrl.compression); if (enable_protocol_trace) { @@ -293,7 +294,7 @@ } // NOTE: From this point on, only the compressed_report exists -@@ -2310,9 +2418,15 @@ void bulkdata_process_profile_usp_event( +@@ -2310,9 +2419,15 @@ kv_vector_t event_args; kv_pair_t kv; report_t *cur_report; @@ -310,7 +311,7 @@ // Exit if the MTP has not been connected to successfully after bootup // This is to prevent BDC events being enqueued before the Boot! event is sent (the Boot! event is only sent after successfully connecting to the MTP). -@@ -2321,20 +2435,62 @@ void bulkdata_process_profile_usp_event( +@@ -2321,20 +2436,62 @@ goto exit; } @@ -384,7 +385,7 @@ } // When sending via USP events, only one report is ever sent in each USP event -@@ -2354,10 +2510,16 @@ void bulkdata_process_profile_usp_event( +@@ -2354,10 +2511,16 @@ bp->num_retained_reports = 1; // Exit if unable to generate the report @@ -404,7 +405,7 @@ return; } -@@ -2365,15 +2527,15 @@ void bulkdata_process_profile_usp_event( +@@ -2365,15 +2528,15 @@ // Construct event_args manually to avoid the overhead of a malloc and copy of the report in KV_VECTOR_Add() kv.key = "Data"; @@ -423,10 +424,37 @@ // From the point of view of this code, the report(s) have been successfully sent, so don't retain them // NOTE: Sending of the reports successfully is delegated to the USP notification retry mechanism -@@ -2835,6 +2997,219 @@ char *bulkdata_generate_json_report(bulk +@@ -2835,6 +2998,319 @@ /*********************************************************************//** ** ++** safe_asprintf ++** ++** Wrapper around asprintf that calls terminate in case of error ++** ++** \param strp -- pointer to the output string ++** \param fmt -- printing format ++** ++** \return None ++** ++**************************************************************************/ ++static void safe_asprintf(char **strp, const char *fmt, ...) ++{ ++ int ret; ++ va_list argp; ++ ++ va_start(argp, fmt); ++ ret = vasprintf(strp, fmt, argp); ++ va_end(argp); ++ ++ if (ret == -1) { ++ USP_ERR_Terminate("%s (%d): asprintf(%s) failed", ++ __FUNCTION__, __LINE__, fmt); ++ } ++} ++ ++/*********************************************************************//** ++** +** append_string_to_target +** +** concatenates the src string with target string in newly allocated memory @@ -454,10 +482,70 @@ + } + + assert(tmp != NULL); -+ asprintf(output, "%s%s", tmp, str); ++ safe_asprintf(output, "%s%s", tmp, str); + free(tmp); +} + ++/************************************************************************* ++** ++** csv_encode ++** ++** encodes the src string to CSV specification RFC4180 compliance string ++** and assign back the new pointer. ++** ++** \param str - pointer to the src string ++** \return address of the pointer that points to the target string or NULL ++** ++**************************************************************************/ ++char *csv_encode(const char *str) ++{ ++ if (str == NULL) ++ return NULL; ++ ++ int len = strlen(str); ++ if (len == 0) ++ return strdup(str); ++ ++ char *temp = NULL; ++ // Get the number of '\"' present in the string ++ int quote_count = 0; ++ ++ temp = strchr(str, '\"'); ++ while (temp) { ++ quote_count++; ++ temp = strchr(temp+1, '\"'); ++ } ++ ++ int encode_size = len + quote_count + 3; // added 3 for initial quote, end quote & null at end ++ temp = (char *)malloc(sizeof(char) * encode_size); ++ ++ if (!temp) ++ return NULL; ++ ++ memset(temp, 0, sizeof(char) * encode_size); ++ ++ int i = 0, j = 0; ++ ++ temp[j++] = '\"'; ++ for (i = 0; i < len; i++) { ++ if (str[i] == '\"') { ++ if (j > (encode_size - 3)) ++ break; ++ ++ temp[j++] = '\"'; ++ } ++ ++ if (j > (encode_size - 3)) ++ break; ++ ++ temp[j++] = str[i]; ++ } ++ ++ temp[j] = '\"'; ++ ++ return temp; ++} ++ +/*********************************************************************//** +** +** bulkdata_generate_csv_report @@ -502,9 +590,9 @@ + + if (strcasecmp(csv_format, "ParameterPerRow") == 0) { + if (strcmp(row_timestamp, "None") == 0) -+ asprintf(&str, "ParameterName%cParameterValue%cParameterType%c", separator, separator, rowseparator); ++ safe_asprintf(&str, "ParameterName%cParameterValue%cParameterType%c", separator, separator, rowseparator); + else -+ asprintf(&str, "ReportTimestamp%cParameterName%cParameterValue%cParameterType%c", separator, separator, separator, rowseparator); ++ safe_asprintf(&str, "ReportTimestamp%cParameterName%cParameterValue%cParameterType%c", separator, separator, separator, rowseparator); + + assert(str != NULL); + append_string_to_target(str, &output); @@ -521,14 +609,18 @@ + // Add Collection time to each csv report element (only if specified and not 'None') + if (strcmp(row_timestamp, "Unix-Epoch")==0) + { -+ asprintf(×tamp, "%lld", (long long int)report->collection_time); ++ safe_asprintf(×tamp, "%lld", (long long int)report->collection_time); + } + else if (strcmp(row_timestamp, "ISO-8601")==0) + { + char *result = iso8601_from_unix_time(report->collection_time, buf, sizeof(buf)); + if (result != NULL) + { -+ asprintf(×tamp, "%s", buf); ++ char *val = csv_encode(buf); ++ if (val) { ++ safe_asprintf(×tamp, "%s", buf); ++ free(val); ++ } + } + } + @@ -579,11 +671,14 @@ + } + + if (type) { ++ char *p_path = csv_encode(param_path); ++ char *p_value = csv_encode(param_value); ++ + if (strcasecmp(csv_format, "ParameterPerRow") == 0) { + if (timestamp == NULL) -+ asprintf(&str, "%s%c%s%c%s%c", param_path, separator, param_value, separator, type, rowseparator); ++ safe_asprintf(&str, "%s%c%s%c%s%c", p_path, separator, p_value, separator, type, rowseparator); + else -+ asprintf(&str, "%s%c%s%c%s%c%s%c", timestamp, separator, param_path, separator, param_value, separator, type, rowseparator); ++ safe_asprintf(&str, "%s%c%s%c%s%c%s%c", timestamp, separator, p_path, separator, p_value, separator, type, rowseparator); + + assert(str != NULL); + append_string_to_target(str, &output); @@ -591,9 +686,9 @@ + str = NULL; + } else { + if (str1 == NULL || strlen(str1) == 0) -+ asprintf(&str, "%s", param_path); ++ safe_asprintf(&str, "%s", p_path); + else -+ asprintf(&str, "%c%s", separator, param_path); ++ safe_asprintf(&str, "%c%s", separator, p_path); + + assert(str != NULL); + append_string_to_target(str, &str1); @@ -601,15 +696,21 @@ + str = NULL; + + if (str2 == NULL || strlen(str2) == 0) -+ asprintf(&str, "%s", param_value); ++ safe_asprintf(&str, "%s", p_value); + else -+ asprintf(&str, "%c%s", separator, param_value); ++ safe_asprintf(&str, "%c%s", separator, p_value); + + assert(str != NULL); + append_string_to_target(str, &str2); + free(str); + str = NULL; + } ++ ++ if (p_path) ++ free(p_path); ++ ++ if (p_value) ++ free(p_value); + } + } + @@ -620,7 +721,7 @@ + } + + if (strcasecmp(csv_format, "ParameterPerColumn") == 0) { -+ asprintf(&str, "%c", rowseparator); ++ safe_asprintf(&str, "%c", rowseparator); + assert(str != NULL); + append_string_to_target(str, &str1); + append_string_to_target(str, &str2); @@ -643,7 +744,7 @@ ** bulkdata_compress_report ** ** Compresses the report to send -@@ -2986,9 +3361,18 @@ int bulkdata_schedule_sending_http_repor +@@ -2986,9 +3462,18 @@ flags |= BDC_FLAG_DATE_HEADER; } diff --git a/obuspa/patches/0009-Implement-non-failing-version-of-asprintf-and-use-it.patch b/obuspa/patches/0009-Implement-non-failing-version-of-asprintf-and-use-it.patch deleted file mode 100644 index f620efe2e..000000000 --- a/obuspa/patches/0009-Implement-non-failing-version-of-asprintf-and-use-it.patch +++ /dev/null @@ -1,121 +0,0 @@ -diff --git a/src/core/device_bulkdata.c b/src/core/device_bulkdata.c -index 8e44f59..a77bd71 100755 ---- a/src/core/device_bulkdata.c -+++ b/src/core/device_bulkdata.c -@@ -2995,6 +2995,33 @@ char *bulkdata_generate_json_report(bulkdata_profile_t *bp, char *report_timesta - return result; - } - -+/*********************************************************************//** -+** -+** safe_asprintf -+** -+** Wrapper around asprintf that calls terminate in case of error -+** -+** \param strp -- pointer to the output string -+** \param fmt -- printing format -+** -+** \return None -+** -+**************************************************************************/ -+static void safe_asprintf(char **strp, const char *fmt, ...) -+{ -+ int ret; -+ va_list argp; -+ -+ va_start(argp, fmt); -+ ret = vasprintf(strp, fmt, argp); -+ va_end(argp); -+ -+ if (ret == -1) { -+ USP_ERR_Terminate("%s (%d): asprintf(%s) failed", -+ __FUNCTION__, __LINE__, fmt); -+ } -+} -+ - /*********************************************************************//** - ** - ** append_string_to_target -@@ -3024,7 +3051,7 @@ void append_string_to_target(char *str, char **output) - } - - assert(tmp != NULL); -- asprintf(output, "%s%s", tmp, str); -+ safe_asprintf(output, "%s%s", tmp, str); - free(tmp); - } - -@@ -3072,9 +3099,9 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator - - if (strcasecmp(csv_format, "ParameterPerRow") == 0) { - if (strcmp(row_timestamp, "None") == 0) -- asprintf(&str, "ParameterName%cParameterValue%cParameterType%c", separator, separator, rowseparator); -+ safe_asprintf(&str, "ParameterName%cParameterValue%cParameterType%c", separator, separator, rowseparator); - else -- asprintf(&str, "ReportTimestamp%cParameterName%cParameterValue%cParameterType%c", separator, separator, separator, rowseparator); -+ safe_asprintf(&str, "ReportTimestamp%cParameterName%cParameterValue%cParameterType%c", separator, separator, separator, rowseparator); - - assert(str != NULL); - append_string_to_target(str, &output); -@@ -3091,14 +3118,14 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator - // Add Collection time to each csv report element (only if specified and not 'None') - if (strcmp(row_timestamp, "Unix-Epoch")==0) - { -- asprintf(×tamp, "%lld", (long long int)report->collection_time); -+ safe_asprintf(×tamp, "%lld", (long long int)report->collection_time); - } - else if (strcmp(row_timestamp, "ISO-8601")==0) - { - char *result = iso8601_from_unix_time(report->collection_time, buf, sizeof(buf)); - if (result != NULL) - { -- asprintf(×tamp, "%s", buf); -+ safe_asprintf(×tamp, "%s", buf); - } - } - -@@ -3151,9 +3178,9 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator - if (type) { - if (strcasecmp(csv_format, "ParameterPerRow") == 0) { - if (timestamp == NULL) -- asprintf(&str, "%s%c%s%c%s%c", param_path, separator, param_value, separator, type, rowseparator); -+ safe_asprintf(&str, "%s%c%s%c%s%c", param_path, separator, param_value, separator, type, rowseparator); - else -- asprintf(&str, "%s%c%s%c%s%c%s%c", timestamp, separator, param_path, separator, param_value, separator, type, rowseparator); -+ safe_asprintf(&str, "%s%c%s%c%s%c%s%c", timestamp, separator, param_path, separator, param_value, separator, type, rowseparator); - - assert(str != NULL); - append_string_to_target(str, &output); -@@ -3161,9 +3188,9 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator - str = NULL; - } else { - if (str1 == NULL || strlen(str1) == 0) -- asprintf(&str, "%s", param_path); -+ safe_asprintf(&str, "%s", param_path); - else -- asprintf(&str, "%c%s", separator, param_path); -+ safe_asprintf(&str, "%c%s", separator, param_path); - - assert(str != NULL); - append_string_to_target(str, &str1); -@@ -3171,9 +3198,9 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator - str = NULL; - - if (str2 == NULL || strlen(str2) == 0) -- asprintf(&str, "%s", param_value); -+ safe_asprintf(&str, "%s", param_value); - else -- asprintf(&str, "%c%s", separator, param_value); -+ safe_asprintf(&str, "%c%s", separator, param_value); - - assert(str != NULL); - append_string_to_target(str, &str2); -@@ -3190,7 +3217,7 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator - } - - if (strcasecmp(csv_format, "ParameterPerColumn") == 0) { -- asprintf(&str, "%c", rowseparator); -+ safe_asprintf(&str, "%c", rowseparator); - assert(str != NULL); - append_string_to_target(str, &str1); - append_string_to_target(str, &str2);