mirror/iopsys-feed
1
0
Fork 0
mirror of https://dev.iopsys.eu/feed/iopsys.git synced 2026-02-22 04:12:28 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

obuspa: Add CSV report compliance to RFC4180 for bulkdata

Browse source
This commit is contained in:
Suvendhu Hansa 2024-01-10 14:03:21 +00:00 • committed by Vivek Kumar Dutta
parent e6015f5db2
commit 8bd2d74316
2 changed files with 136 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

171
obuspa/patches/0007-bdc_csv_format.patch
View file

@ -1,6 +1,6 @@
--- a/src/core/bdc_exec.c
+++ b/src/core/bdc_exec.c
@@ -547,11 +547,19 @@ int StartSendingReport(bdc_connection_t
@@ -547,11 +547,19 @@
// Set the list of headers
bc->headers = NULL;
@ -25,7 +25,7 @@
{
--- a/src/core/bdc_exec.h
+++ b/src/core/bdc_exec.h
@@ -53,6 +53,9 @@ void BDC_EXEC_ScheduleExit(void);
@@ -53,6 +53,9 @@
#define BDC_FLAG_PUT 0x00000001 // If set, HTTP PUT should be used instead of HTTP POST when sending the report to the BDC server
#define BDC_FLAG_GZIP 0x00000002 // If set, the reports contants are Gzipped
#define BDC_FLAG_DATE_HEADER 0x00000004 // If set, the date header should be included in the HTTP post.
@ -52,7 +52,7 @@
// Definitions for Device.BulkData.Profile.{i}.JSONEncoding.ReportTimestamp
@@ -153,6 +156,7 @@ static char *profile_push_event_args[] =
@@ -153,6 +156,7 @@
typedef struct
{
int num_retained_failed_reports;
@ -60,7 +60,7 @@
char report_timestamp[33];
char url[1025];
char username[257];
@@ -161,6 +165,11 @@ typedef struct
@@ -161,6 +165,11 @@
char method[9];
bool use_date_header;
char report_format[20];
@ -72,7 +72,7 @@
} profile_ctrl_params_t;
//------------------------------------------------------------------------------
@@ -208,6 +217,7 @@ int Validate_BulkDataEncodingType(dm_req
@@ -208,6 +217,7 @@
int Validate_BulkDataReportingInterval(dm_req_t *req, char *value);
int Validate_BulkDataReference(dm_req_t *req, char *value);
int Validate_BulkDataReportFormat(dm_req_t *req, char *value);
@ -80,7 +80,7 @@
int Validate_BulkDataReportTimestamp(dm_req_t *req, char *value);
int Validate_BulkDataCompression(dm_req_t *req, char *value);
int Validate_BulkDataHTTPMethod(dm_req_t *req, char *value);
@@ -236,6 +246,8 @@ bulkdata_profile_t *bulkdata_find_profil
@@ -236,6 +246,8 @@
int bulkdata_calc_report_map(bulkdata_profile_t *bp, kv_vector_t *report_map);
int bulkdata_reduce_to_alt_name(char *spec, char *path, char *alt_name, char *out_buf, int buf_len);
char *bulkdata_generate_json_report(bulkdata_profile_t *bp, char *report_timestamp, char *report_format);
@ -89,15 +89,16 @@
unsigned char *bulkdata_compress_report(profile_ctrl_params_t *ctrl, char *input_buf, int input_len, int *p_output_len);
int bulkdata_schedule_sending_http_report(profile_ctrl_params_t *ctrl, bulkdata_profile_t *bp, unsigned char *json_report, int report_len);
int bulkdata_start_profile(bulkdata_profile_t *bp);
@@ -250,6 +262,7 @@ char *bulkdata_platform_calc_uri_query_s
@@ -250,6 +262,8 @@
int bulkdata_platform_get_param_refs(int profile_id, param_ref_vector_t *param_refs);
void bulkdata_expand_param_ref(param_ref_entry_t *pr, group_get_vector_t *ggv);
void bulkdata_append_to_result_map(param_ref_entry_t *pr, group_get_vector_t *ggv, kv_vector_t *report_map);
+void append_string_to_target(char *str, char **output);
+char *csv_encode(const char *str);
/*********************************************************************//**
**
@@ -282,7 +295,7 @@ int DEVICE_BULKDATA_Init(void)
@@ -282,7 +296,7 @@
err |= USP_REGISTER_VendorParam_ReadOnly("Device.BulkData.Status", Get_BulkDataGlobalStatus, DM_STRING);
err |= USP_REGISTER_Param_Constant("Device.BulkData.MinReportingInterval", BULKDATA_MINIMUM_REPORTING_INTERVAL_STR, DM_UINT);
err |= USP_REGISTER_Param_SupportedList("Device.BulkData.Protocols", bdc_protocols, NUM_ELEM(bdc_protocols));
@ -106,7 +107,7 @@
err |= USP_REGISTER_Param_Constant("Device.BulkData.ParameterWildCardSupported", "true", DM_BOOL);
err |= USP_REGISTER_Param_Constant("Device.BulkData.MaxNumberOfProfiles", BULKDATA_MAX_PROFILES_STR, DM_INT);
err |= USP_REGISTER_Param_Constant("Device.BulkData.MaxNumberOfParameterReferences", "-1", DM_INT);
@@ -297,7 +310,7 @@ int DEVICE_BULKDATA_Init(void)
@@ -297,7 +311,7 @@
err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.Name", "", NULL, NULL, DM_STRING);
err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.NumberOfRetainedFailedReports", "0", Validate_NumberOfRetainedFailedReports, NULL, DM_INT);
err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.Protocol", BULKDATA_PROTOCOL_HTTP, Validate_BulkDataProtocol, NULL, DM_STRING);
@ -115,7 +116,7 @@
err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.ReportingInterval", "86400", Validate_BulkDataReportingInterval, NotifyChange_BulkDataReportingInterval, DM_UINT);
err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.TimeReference", UNKNOWN_TIME_STR, NULL, NotifyChange_BulkDataTimeReference, DM_DATETIME);
@@ -312,6 +325,13 @@ int DEVICE_BULKDATA_Init(void)
@@ -312,6 +326,13 @@
err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.JSONEncoding.ReportFormat", BULKDATA_JSON_REPORT_FORMAT_NAME_VALUE, Validate_BulkDataReportFormat, NULL, DM_STRING);
err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.JSONEncoding.ReportTimestamp", BULKDATA_JSON_TIMESTAMP_FORMAT_EPOCH, Validate_BulkDataReportTimestamp, NULL, DM_STRING);
@ -129,7 +130,7 @@
// Device.BulkData.Profile.{i}.HTTP
err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.HTTP.URL", "", NULL, NotifyChange_BulkDataURL, DM_STRING);
err |= USP_REGISTER_DBParam_ReadWrite("Device.BulkData.Profile.{i}.HTTP.Username", "", NULL, NULL, DM_STRING);
@@ -591,9 +611,10 @@ int Validate_BulkDataProtocol(dm_req_t *
@@ -591,9 +612,10 @@
int Validate_BulkDataEncodingType(dm_req_t *req, char *value)
{
// Exit if trying to set a value outside of the range we accept
@ -142,7 +143,7 @@
return USP_ERR_INVALID_VALUE;
}
@@ -676,6 +697,32 @@ int Validate_BulkDataReportFormat(dm_req
@@ -676,6 +698,32 @@
/*********************************************************************//**
**
@ -175,7 +176,7 @@
** Validate_BulkDataReportTimestamp
**
** Validates Device.BulkData.Profile.{i}.JSONEncoding.ReportTimestamp
@@ -1970,6 +2017,14 @@ int bulkdata_platform_get_profile_contro
@@ -1970,6 +2018,14 @@
return err;
}
@ -190,7 +191,7 @@
// Exit if unable to get ReportTimestamp
USP_SNPRINTF(path, sizeof(path), "Device.BulkData.Profile.%d.JSONEncoding.ReportTimestamp", bp->profile_id);
err = DATA_MODEL_GetParameterValue(path, ctrl_params->report_timestamp, sizeof(ctrl_params->report_timestamp), 0);
@@ -1986,6 +2041,46 @@ int bulkdata_platform_get_profile_contro
@@ -1986,6 +2042,46 @@
return err;
}
@ -237,7 +238,7 @@
return USP_ERR_OK;
}
@@ -2222,7 +2317,7 @@ void bulkdata_process_profile_http(bulkd
@@ -2222,7 +2318,7 @@
{
int err;
report_t *cur_report;
@ -246,7 +247,7 @@
profile_ctrl_params_t ctrl;
unsigned char *compressed_report;
int compressed_len;
@@ -2261,10 +2356,23 @@ void bulkdata_process_profile_http(bulkd
@@ -2261,10 +2357,23 @@
}
// Exit if unable to generate the report
@ -274,7 +275,7 @@
return;
}
@@ -2273,14 +2381,14 @@ void bulkdata_process_profile_http(bulkd
@@ -2273,14 +2382,14 @@
USP_LOG_Info("BULK DATA: using compression method=%s", ctrl.compression);
if (enable_protocol_trace)
{
@ -293,7 +294,7 @@
}
// NOTE: From this point on, only the compressed_report exists
@@ -2310,9 +2418,15 @@ void bulkdata_process_profile_usp_event(
@@ -2310,9 +2419,15 @@
kv_vector_t event_args;
kv_pair_t kv;
report_t *cur_report;
@ -310,7 +311,7 @@
// Exit if the MTP has not been connected to successfully after bootup
// This is to prevent BDC events being enqueued before the Boot! event is sent (the Boot! event is only sent after successfully connecting to the MTP).
@@ -2321,20 +2435,62 @@ void bulkdata_process_profile_usp_event(
@@ -2321,20 +2436,62 @@
goto exit;
}
@ -384,7 +385,7 @@
}
// When sending via USP events, only one report is ever sent in each USP event
@@ -2354,10 +2510,16 @@ void bulkdata_process_profile_usp_event(
@@ -2354,10 +2511,16 @@
bp->num_retained_reports = 1;
// Exit if unable to generate the report
@ -404,7 +405,7 @@
return;
}
@@ -2365,15 +2527,15 @@ void bulkdata_process_profile_usp_event(
@@ -2365,15 +2528,15 @@
// Construct event_args manually to avoid the overhead of a malloc and copy of the report in KV_VECTOR_Add()
kv.key = "Data";
@ -423,10 +424,37 @@
// From the point of view of this code, the report(s) have been successfully sent, so don't retain them
// NOTE: Sending of the reports successfully is delegated to the USP notification retry mechanism
@@ -2835,6 +2997,219 @@ char *bulkdata_generate_json_report(bulk
@@ -2835,6 +2998,319 @@
/*********************************************************************//**
**
+** safe_asprintf
+**
+** Wrapper around asprintf that calls terminate in case of error
+**
+** \param strp -- pointer to the output string
+** \param fmt -- printing format
+**
+** \return None
+**
+**************************************************************************/
+static void safe_asprintf(char **strp, const char *fmt, ...)
+{
+ int ret;
+ va_list argp;
+
+ va_start(argp, fmt);
+ ret = vasprintf(strp, fmt, argp);
+ va_end(argp);
+
+ if (ret == -1) {
+ USP_ERR_Terminate("%s (%d): asprintf(%s) failed",
+ __FUNCTION__, __LINE__, fmt);
+ }
+}
+
+/*********************************************************************//**
+**
+** append_string_to_target
+**
+** concatenates the src string with target string in newly allocated memory
@ -454,10 +482,70 @@
+ }
+
+ assert(tmp != NULL);
+ asprintf(output, "%s%s", tmp, str);
+ safe_asprintf(output, "%s%s", tmp, str);
+ free(tmp);
+}
+
+/*************************************************************************
+**
+** csv_encode
+**
+** encodes the src string to CSV specification RFC4180 compliance string
+** and assign back the new pointer.
+**
+** \param str - pointer to the src string
+** \return address of the pointer that points to the target string or NULL
+**
+**************************************************************************/
+char *csv_encode(const char *str)
+{
+ if (str == NULL)
+ return NULL;
+
+ int len = strlen(str);
+ if (len == 0)
+ return strdup(str);
+
+ char *temp = NULL;
+ // Get the number of '\"' present in the string
+ int quote_count = 0;
+
+ temp = strchr(str, '\"');
+ while (temp) {
+ quote_count++;
+ temp = strchr(temp+1, '\"');
+ }
+
+ int encode_size = len + quote_count + 3; // added 3 for initial quote, end quote & null at end
+ temp = (char *)malloc(sizeof(char) * encode_size);
+
+ if (!temp)
+ return NULL;
+
+ memset(temp, 0, sizeof(char) * encode_size);
+
+ int i = 0, j = 0;
+
+ temp[j++] = '\"';
+ for (i = 0; i < len; i++) {
+ if (str[i] == '\"') {
+ if (j > (encode_size - 3))
+ break;
+
+ temp[j++] = '\"';
+ }
+
+ if (j > (encode_size - 3))
+ break;
+
+ temp[j++] = str[i];
+ }
+
+ temp[j] = '\"';
+
+ return temp;
+}
+
+/*********************************************************************//**
+**
+** bulkdata_generate_csv_report
@ -502,9 +590,9 @@
+
+ if (strcasecmp(csv_format, "ParameterPerRow") == 0) {
+ if (strcmp(row_timestamp, "None") == 0)
+ asprintf(&str, "ParameterName%cParameterValue%cParameterType%c", separator, separator, rowseparator);
+ safe_asprintf(&str, "ParameterName%cParameterValue%cParameterType%c", separator, separator, rowseparator);
+ else
+ asprintf(&str, "ReportTimestamp%cParameterName%cParameterValue%cParameterType%c", separator, separator, separator, rowseparator);
+ safe_asprintf(&str, "ReportTimestamp%cParameterName%cParameterValue%cParameterType%c", separator, separator, separator, rowseparator);
+
+ assert(str != NULL);
+ append_string_to_target(str, &output);
@ -521,14 +609,18 @@
+ // Add Collection time to each csv report element (only if specified and not 'None')
+ if (strcmp(row_timestamp, "Unix-Epoch")==0)
+ {
+ asprintf(&timestamp, "%lld", (long long int)report->collection_time);
+ safe_asprintf(&timestamp, "%lld", (long long int)report->collection_time);
+ }
+ else if (strcmp(row_timestamp, "ISO-8601")==0)
+ {
+ char *result = iso8601_from_unix_time(report->collection_time, buf, sizeof(buf));
+ if (result != NULL)
+ {
+ asprintf(&timestamp, "%s", buf);
+ char *val = csv_encode(buf);
+ if (val) {
+ safe_asprintf(&timestamp, "%s", buf);
+ free(val);
+ }
+ }
+ }
+
@ -579,11 +671,14 @@
+ }
+
+ if (type) {
+ char *p_path = csv_encode(param_path);
+ char *p_value = csv_encode(param_value);
+
+ if (strcasecmp(csv_format, "ParameterPerRow") == 0) {
+ if (timestamp == NULL)
+ asprintf(&str, "%s%c%s%c%s%c", param_path, separator, param_value, separator, type, rowseparator);
+ safe_asprintf(&str, "%s%c%s%c%s%c", p_path, separator, p_value, separator, type, rowseparator);
+ else
+ asprintf(&str, "%s%c%s%c%s%c%s%c", timestamp, separator, param_path, separator, param_value, separator, type, rowseparator);
+ safe_asprintf(&str, "%s%c%s%c%s%c%s%c", timestamp, separator, p_path, separator, p_value, separator, type, rowseparator);
+
+ assert(str != NULL);
+ append_string_to_target(str, &output);
@ -591,9 +686,9 @@
+ str = NULL;
+ } else {
+ if (str1 == NULL || strlen(str1) == 0)
+ asprintf(&str, "%s", param_path);
+ safe_asprintf(&str, "%s", p_path);
+ else
+ asprintf(&str, "%c%s", separator, param_path);
+ safe_asprintf(&str, "%c%s", separator, p_path);
+
+ assert(str != NULL);
+ append_string_to_target(str, &str1);
@ -601,15 +696,21 @@
+ str = NULL;
+
+ if (str2 == NULL || strlen(str2) == 0)
+ asprintf(&str, "%s", param_value);
+ safe_asprintf(&str, "%s", p_value);
+ else
+ asprintf(&str, "%c%s", separator, param_value);
+ safe_asprintf(&str, "%c%s", separator, p_value);
+
+ assert(str != NULL);
+ append_string_to_target(str, &str2);
+ free(str);
+ str = NULL;
+ }
+
+ if (p_path)
+ free(p_path);
+
+ if (p_value)
+ free(p_value);
+ }
+ }
+
@ -620,7 +721,7 @@
+ }
+
+ if (strcasecmp(csv_format, "ParameterPerColumn") == 0) {
+ asprintf(&str, "%c", rowseparator);
+ safe_asprintf(&str, "%c", rowseparator);
+ assert(str != NULL);
+ append_string_to_target(str, &str1);
+ append_string_to_target(str, &str2);
@ -643,7 +744,7 @@
** bulkdata_compress_report
**
** Compresses the report to send
@@ -2986,9 +3361,18 @@ int bulkdata_schedule_sending_http_repor
@@ -2986,9 +3462,18 @@
flags |= BDC_FLAG_DATE_HEADER;
}

121
obuspa/patches/0009-Implement-non-failing-version-of-asprintf-and-use-it.patch
View file

@ -1,121 +0,0 @@
diff --git a/src/core/device_bulkdata.c b/src/core/device_bulkdata.c
index 8e44f59..a77bd71 100755
--- a/src/core/device_bulkdata.c
+++ b/src/core/device_bulkdata.c
@@ -2995,6 +2995,33 @@ char *bulkdata_generate_json_report(bulkdata_profile_t *bp, char *report_timesta
return result;
}
+/*********************************************************************//**
+**
+** safe_asprintf
+**
+** Wrapper around asprintf that calls terminate in case of error
+**
+** \param strp -- pointer to the output string
+** \param fmt -- printing format
+**
+** \return None
+**
+**************************************************************************/
+static void safe_asprintf(char **strp, const char *fmt, ...)
+{
+ int ret;
+ va_list argp;
+
+ va_start(argp, fmt);
+ ret = vasprintf(strp, fmt, argp);
+ va_end(argp);
+
+ if (ret == -1) {
+ USP_ERR_Terminate("%s (%d): asprintf(%s) failed",
+ __FUNCTION__, __LINE__, fmt);
+ }
+}
+
/*********************************************************************//**
**
** append_string_to_target
@@ -3024,7 +3051,7 @@ void append_string_to_target(char *str, char **output)
}
assert(tmp != NULL);
- asprintf(output, "%s%s", tmp, str);
+ safe_asprintf(output, "%s%s", tmp, str);
free(tmp);
}
@@ -3072,9 +3099,9 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator
if (strcasecmp(csv_format, "ParameterPerRow") == 0) {
if (strcmp(row_timestamp, "None") == 0)
- asprintf(&str, "ParameterName%cParameterValue%cParameterType%c", separator, separator, rowseparator);
+ safe_asprintf(&str, "ParameterName%cParameterValue%cParameterType%c", separator, separator, rowseparator);
else
- asprintf(&str, "ReportTimestamp%cParameterName%cParameterValue%cParameterType%c", separator, separator, separator, rowseparator);
+ safe_asprintf(&str, "ReportTimestamp%cParameterName%cParameterValue%cParameterType%c", separator, separator, separator, rowseparator);
assert(str != NULL);
append_string_to_target(str, &output);
@@ -3091,14 +3118,14 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator
// Add Collection time to each csv report element (only if specified and not 'None')
if (strcmp(row_timestamp, "Unix-Epoch")==0)
{
- asprintf(&timestamp, "%lld", (long long int)report->collection_time);
+ safe_asprintf(&timestamp, "%lld", (long long int)report->collection_time);
}
else if (strcmp(row_timestamp, "ISO-8601")==0)
{
char *result = iso8601_from_unix_time(report->collection_time, buf, sizeof(buf));
if (result != NULL)
{
- asprintf(&timestamp, "%s", buf);
+ safe_asprintf(&timestamp, "%s", buf);
}
}
@@ -3151,9 +3178,9 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator
if (type) {
if (strcasecmp(csv_format, "ParameterPerRow") == 0) {
if (timestamp == NULL)
- asprintf(&str, "%s%c%s%c%s%c", param_path, separator, param_value, separator, type, rowseparator);
+ safe_asprintf(&str, "%s%c%s%c%s%c", param_path, separator, param_value, separator, type, rowseparator);
else
- asprintf(&str, "%s%c%s%c%s%c%s%c", timestamp, separator, param_path, separator, param_value, separator, type, rowseparator);
+ safe_asprintf(&str, "%s%c%s%c%s%c%s%c", timestamp, separator, param_path, separator, param_value, separator, type, rowseparator);
assert(str != NULL);
append_string_to_target(str, &output);
@@ -3161,9 +3188,9 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator
str = NULL;
} else {
if (str1 == NULL || strlen(str1) == 0)
- asprintf(&str, "%s", param_path);
+ safe_asprintf(&str, "%s", param_path);
else
- asprintf(&str, "%c%s", separator, param_path);
+ safe_asprintf(&str, "%c%s", separator, param_path);
assert(str != NULL);
append_string_to_target(str, &str1);
@@ -3171,9 +3198,9 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator
str = NULL;
if (str2 == NULL || strlen(str2) == 0)
- asprintf(&str, "%s", param_value);
+ safe_asprintf(&str, "%s", param_value);
else
- asprintf(&str, "%c%s", separator, param_value);
+ safe_asprintf(&str, "%c%s", separator, param_value);
assert(str != NULL);
append_string_to_target(str, &str2);
@@ -3190,7 +3217,7 @@ char *bulkdata_generate_csv_report(bulkdata_profile_t *bp, char *field_separator
}
if (strcasecmp(csv_format, "ParameterPerColumn") == 0) {
- asprintf(&str, "%c", rowseparator);
+ safe_asprintf(&str, "%c", rowseparator);
assert(str != NULL);
append_string_to_target(str, &str1);
append_string_to_target(str, &str2);