Skip to content

Commit 9b6b3cd

Browse files
larsewilarsewi
authored
Merge pull request #6267 from victormlg/isvaliddata
 ENT-8193: Added validfiledata policy function
2 parents bf1ba3f + e72fa9a commit 9b6b3cd

3 files changed

Lines changed: 144 additions & 14 deletions

File tree

‎libenv/sysinfo.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1188,7 +1188,7 @@ static void OSReleaseParse(EvalContext *ctx, const char *file_path)
11881188
{
11891189
JsonElement *os_release_json = JsonReadDataFile("system info discovery",
11901190
file_path, DATAFILETYPE_ENV,
1191-
100 * 1024);
1191+
100 * 1024, true);
11921192
if (os_release_json != NULL)
11931193
{
11941194
char *tags;

‎libpromises/evalfunction.c‎

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7779,7 +7779,7 @@ static FnCallResult ReadDataGeneric(const char *const fname,
77797779
assert(fname != NULL);
77807780
assert(input_path != NULL);
77817781

7782-
JsonElement *json = JsonReadDataFile(fname, input_path, requested_mode, size_max);
7782+
JsonElement *json = JsonReadDataFile(fname, input_path, requested_mode, size_max, true);
77837783
if (json == NULL)
77847784
{
77857785
return FnFailure();
@@ -7788,6 +7788,23 @@ static FnCallResult ReadDataGeneric(const char *const fname,
77887788
return FnReturnContainerNoCopy(json);
77897789
}
77907790

7791+
static DataFileType ParseRequestedMode(const char *fname, const char *input_path, const char *const mode_string)
7792+
{
7793+
DataFileType requested_mode = DATAFILETYPE_UNKNOWN;
7794+
if (StringEqual("auto", mode_string))
7795+
{
7796+
requested_mode = GetDataFileTypeFromSuffix(input_path);
7797+
Log(LOG_LEVEL_VERBOSE,
7798+
"%s: automatically selected data type %s from filename %s",
7799+
fname, DataFileTypeToString(requested_mode), input_path);
7800+
}
7801+
else
7802+
{
7803+
requested_mode = GetDataFileTypeFromString(mode_string);
7804+
}
7805+
return requested_mode;
7806+
}
7807+
77917808
static FnCallResult FnCallReadData(ARG_UNUSED EvalContext *ctx,
77927809
ARG_UNUSED const Policy *policy,
77937810
const FnCall *fp,
@@ -7802,20 +7819,32 @@ static FnCallResult FnCallReadData(ARG_UNUSED EvalContext *ctx,
78027819

78037820
const char *input_path = RlistScalarValue(args);
78047821
const char *const mode_string = RlistScalarValue(args->next);
7805-
DataFileType requested_mode = DATAFILETYPE_UNKNOWN;
7806-
if (StringEqual("auto", mode_string))
7807-
{
7808-
requested_mode = GetDataFileTypeFromSuffix(input_path);
7809-
Log(LOG_LEVEL_VERBOSE,
7810-
"%s: automatically selected data type %s from filename %s",
7811-
fp->name, DataFileTypeToString(requested_mode), input_path);
7812-
}
7813-
else
7822+
DataFileType requested_mode = ParseRequestedMode(fp->name, input_path, mode_string);
7823+
7824+
return ReadDataGeneric(fp->name, input_path, CF_INFINITY, requested_mode);
7825+
}
7826+
7827+
static FnCallResult FnCallValidFileData(ARG_UNUSED EvalContext *ctx,
7828+
ARG_UNUSED const Policy *policy,
7829+
const FnCall *fp,
7830+
const Rlist *args)
7831+
{
7832+
assert(fp != NULL);
7833+
if (args == NULL)
78147834
{
7815-
requested_mode = GetDataFileTypeFromString(mode_string);
7835+
Log(LOG_LEVEL_ERR, "Function '%s' requires at least one argument", fp->name);
7836+
return FnFailure();
78167837
}
78177838

7818-
return ReadDataGeneric(fp->name, input_path, CF_INFINITY, requested_mode);
7839+
const char *input_path = RlistScalarValue(args);
7840+
const char *const mode_string = RlistScalarValue(args->next);
7841+
DataFileType requested_mode = ParseRequestedMode(fp->name, input_path, mode_string);
7842+
7843+
JsonElement *json = JsonReadDataFile(fp->name, input_path, requested_mode, CF_INFINITY, false);
7844+
bool is_valid = (json != NULL);
7845+
JsonDestroy(json);
7846+
7847+
return FnReturnContext(is_valid);
78197848
}
78207849

78217850
static FnCallResult ReadGenericDataType(const FnCall *fp,
@@ -10326,7 +10355,7 @@ void ModuleProtocol(EvalContext *ctx, const char *command, const char *line, int
1032610355
Log(LOG_LEVEL_DEBUG, "Module protocol parsing %s file '%s'",
1032710356
DataFileTypeToString(requested_mode), content);
1032810357

10329-
JsonElement *json = JsonReadDataFile("module file protocol", content, requested_mode, size_max);
10358+
JsonElement *json = JsonReadDataFile("module file protocol", content, requested_mode, size_max, true);
1033010359
if (json != NULL)
1033110360
{
1033210361
Buffer *tagbuf = StringSetToBuffer(tags, ',');
@@ -11906,6 +11935,8 @@ const FnCallType CF_FNCALL_TYPES[] =
1190611935
FNCALL_OPTION_NONE, FNCALL_CATEGORY_IO, SYNTAX_STATUS_NORMAL, DEFAULT_ARGC),
1190711936
FnCallTypeNew("readtcp", CF_DATA_TYPE_STRING, READTCP_ARGS, &FnCallReadTcp, "Connect to tcp port, send string and assign result to variable",
1190811937
FNCALL_OPTION_CACHED, FNCALL_CATEGORY_COMM, SYNTAX_STATUS_NORMAL, DEFAULT_ARGC),
11938+
FnCallTypeNew("validfiledata", CF_DATA_TYPE_CONTEXT, READDATA_ARGS, &FnCallValidFileData, "Validate a YAML, JSON, CSV, etc.",
11939+
FNCALL_OPTION_NONE, FNCALL_CATEGORY_IO, SYNTAX_STATUS_NORMAL, DEFAULT_ARGC),
1190911940

1191011941
// reg functions for regex
1191111942
FnCallTypeNew("regarray", CF_DATA_TYPE_CONTEXT, REGARRAY_ARGS, &FnCallRegList, "True if the regular expression in arg1 matches any item in the list or array or data container arg2",

‎tests/acceptance/01_vars/02_functions/validfiledata.cf‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
body common control
2+
{
3+
inputs => { "../../default.sub.cf" };
4+
bundlesequence => { init, test, check };
5+
version => "1.0";
6+
}
7+
8+
bundle agent generate_files(template, data)
9+
{
10+
vars:
11+
"content" string => string_mustache("$(template[mustache])", @(data));
12+
13+
files:
14+
"$(G.testdir)/validfiledata/config.$(template[format])"
15+
create => "true",
16+
content => "$(content)";
17+
18+
"$(G.testdir)/validfiledata/config.x.$(template[format])"
19+
create => "true",
20+
content => "-$(content)", # Add some text that makes the json and yaml invalid
21+
if => strcmp("$(template[create_invalid])", "yes"); # csv and env are invalid ONLY when the file doesn't exist
22+
}
23+
24+
bundle agent init
25+
{
26+
vars:
27+
"config_data"
28+
data => parsejson(
29+
'{"hello" : "world", "bye" : "everyone", "pizza" : "hamburger"}'
30+
);
31+
@if feature(yaml)
32+
"templates"
33+
data => parsejson(
34+
'[
35+
{"format": "json", "mustache": "{{%-top-}}", "create_invalid" : "yes"},
36+
{"format": "yaml", "mustache": "---\\n{{#-top-}}\\n{{@}}: \\\"{{.}}\\\"\\n{{/-top-}}", "create_invalid" : "yes"},
37+
{"format": "csv", "mustache": "Key,Value\\r\\n{{#-top-}}\\r\\n{{@}},\\\"{{.}}\\\"\\r\\n{{/-top-}}", "create_invalid" : "no"},
38+
{"format": "env", "mustache": "{{#-top-}}\\n{{{@}}}={{{.}}}\\n{{/-top-}}", "create_invalid" : "no"}
39+
]'
40+
);
41+
@else
42+
"templates"
43+
data => parsejson(
44+
'[
45+
{"format": "json", "mustache": "{{%-top-}}", "create_invalid" : "yes"},
46+
{"format": "csv", "mustache": "Key,Value\\r\\n{{#-top-}}\\r\\n{{@}},\\\"{{.}}\\\"\\r\\n{{/-top-}}", "create_invalid" : "no"},
47+
{"format": "env", "mustache": "{{#-top-}}\\n{{{@}}}={{{.}}}\\n{{/-top-}}", "create_invalid" : "no"}
48+
]'
49+
);
50+
@endif
51+
"template_idx" slist => getindices(templates);
52+
53+
methods:
54+
"generate"
55+
usebundle => generate_files(
56+
"@(templates[$(template_idx)])", @(config_data)
57+
);
58+
}
59+
60+
bundle agent test
61+
{
62+
vars:
63+
"formats"
64+
slist => maparray("$(init.templates[$(this.k)][format])", init.templates);
65+
66+
"valid_classes" slist => maplist("valid_$(this)", formats);
67+
"invalid_classes" slist => maplist("invalid_$(this)", formats);
68+
69+
classes:
70+
"valid_$(formats)"
71+
expression => validfiledata(
72+
"$(G.testdir)/validfiledata/config.$(formats)", "auto"
73+
),
74+
scope => "namespace";
75+
76+
"invalid_$(formats)"
77+
expression => validfiledata(
78+
"$(G.testdir)/validfiledata/config.x.$(formats)", "auto"
79+
),
80+
scope => "namespace";
81+
}
82+
83+
bundle agent check
84+
{
85+
classes:
86+
"valid" and => { @(test.valid_classes) };
87+
"invalid" or => { @(test.invalid_classes) };
88+
89+
files:
90+
"$(G.testdir)/validfiledata/config.$(test.formats)" delete => tidy;
91+
"$(G.testdir)/validfiledata/config.x.$(test.formats)" delete => tidy;
92+
93+
reports:
94+
valid.!invalid::
95+
"$(this.promise_filename) Pass";
96+
97+
!valid|invalid::
98+
"$(this.promise_filename) FAIL";
99+
}

0 commit comments

Comments
 (0)