Skip to content

ENT-8193: Added validfiledata policy function #6267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
victormlg wants to merge 1 commit into
base: master
Choose a base branch
from
+139 −14
Open

ENT-8193: Added validfiledata policy function #6267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion libenv/sysinfo.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1188,7 +1188,7 @@ static void OSReleaseParse(EvalContext *ctx, const char *file_path)
{
JsonElement *os_release_json = JsonReadDataFile("system info discovery",
file_path, DATAFILETYPE_ENV,
100 * 1024);
100 * 1024, true);
if (os_release_json != NULL)
{
char *tags;
Expand Down
57 changes: 44 additions & 13 deletions libpromises/evalfunction.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7800,7 +7800,7 @@ static FnCallResult ReadDataGeneric(const char *const fname,
assert(fname != NULL);
assert(input_path != NULL);

JsonElement *json = JsonReadDataFile(fname, input_path, requested_mode, size_max);
JsonElement *json = JsonReadDataFile(fname, input_path, requested_mode, size_max, true);
if (json == NULL)
{
return FnFailure();
Expand All @@ -7809,6 +7809,23 @@ static FnCallResult ReadDataGeneric(const char *const fname,
return FnReturnContainerNoCopy(json);
}

static DataFileType ParseRequestedMode(const char *fname, const char *input_path, const char *const mode_string)
{
DataFileType requested_mode = DATAFILETYPE_UNKNOWN;
if (StringEqual("auto", mode_string))
{
requested_mode = GetDataFileTypeFromSuffix(input_path);
Log(LOG_LEVEL_VERBOSE,
"%s: automatically selected data type %s from filename %s",
fname, DataFileTypeToString(requested_mode), input_path);
}
else
{
requested_mode = GetDataFileTypeFromString(mode_string);
}
return requested_mode;
}

static FnCallResult FnCallReadData(ARG_UNUSED EvalContext *ctx,
ARG_UNUSED const Policy *policy,
const FnCall *fp,
Expand All @@ -7823,20 +7840,32 @@ static FnCallResult FnCallReadData(ARG_UNUSED EvalContext *ctx,

const char *input_path = RlistScalarValue(args);
const char *const mode_string = RlistScalarValue(args->next);
DataFileType requested_mode = DATAFILETYPE_UNKNOWN;
if (StringEqual("auto", mode_string))
{
requested_mode = GetDataFileTypeFromSuffix(input_path);
Log(LOG_LEVEL_VERBOSE,
"%s: automatically selected data type %s from filename %s",
fp->name, DataFileTypeToString(requested_mode), input_path);
}
else
DataFileType requested_mode = ParseRequestedMode(fp->name, input_path, mode_string);

return ReadDataGeneric(fp->name, input_path, CF_INFINITY, requested_mode);
}

static FnCallResult FnCallValidFileData(ARG_UNUSED EvalContext *ctx,
ARG_UNUSED const Policy *policy,
const FnCall *fp,
const Rlist *args)
{
assert(fp != NULL);
if (args == NULL)
{
requested_mode = GetDataFileTypeFromString(mode_string);
Log(LOG_LEVEL_ERR, "Function '%s' requires at least one argument", fp->name);
return FnFailure();
}

return ReadDataGeneric(fp->name, input_path, CF_INFINITY, requested_mode);
const char *input_path = RlistScalarValue(args);
const char *const mode_string = RlistScalarValue(args->next);
DataFileType requested_mode = ParseRequestedMode(fp->name, input_path, mode_string);

JsonElement *json = JsonReadDataFile(fp->name, input_path, requested_mode, CF_INFINITY, false);
bool is_valid = (json != NULL);
JsonDestroy(json);
Comment thread
victormlg marked this conversation as resolved.

return FnReturnContext(is_valid);
}

static FnCallResult ReadGenericDataType(const FnCall *fp,
Expand Down Expand Up @@ -10319,7 +10348,7 @@ void ModuleProtocol(EvalContext *ctx, const char *command, const char *line, int
Log(LOG_LEVEL_DEBUG, "Module protocol parsing %s file '%s'",
DataFileTypeToString(requested_mode), content);

JsonElement *json = JsonReadDataFile("module file protocol", content, requested_mode, size_max);
JsonElement *json = JsonReadDataFile("module file protocol", content, requested_mode, size_max, true);
if (json != NULL)
{
Buffer *tagbuf = StringSetToBuffer(tags, ',');
Expand Down Expand Up @@ -11899,6 +11928,8 @@ const FnCallType CF_FNCALL_TYPES[] =
FNCALL_OPTION_NONE, FNCALL_CATEGORY_IO, SYNTAX_STATUS_NORMAL, DEFAULT_ARGC),
FnCallTypeNew("readtcp", CF_DATA_TYPE_STRING, READTCP_ARGS, &FnCallReadTcp, "Connect to tcp port, send string and assign result to variable",
FNCALL_OPTION_CACHED, FNCALL_CATEGORY_COMM, SYNTAX_STATUS_NORMAL, DEFAULT_ARGC),
FnCallTypeNew("validfiledata", CF_DATA_TYPE_CONTEXT, READDATA_ARGS, &FnCallValidFileData, "Validate a YAML, JSON, CSV, etc.",
Comment thread
victormlg marked this conversation as resolved.
FNCALL_OPTION_NONE, FNCALL_CATEGORY_IO, SYNTAX_STATUS_NORMAL, DEFAULT_ARGC),

// reg functions for regex
FnCallTypeNew("regarray", CF_DATA_TYPE_CONTEXT, REGARRAY_ARGS, &FnCallRegList, "True if the regular expression in arg1 matches any item in the list or array or data container arg2",
Expand Down
94 changes: 94 additions & 0 deletions tests/acceptance/01_vars/02_functions/validfiledata.cf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
body common control
{
inputs => { "../../default.sub.cf" };
bundlesequence => { init, test, check };
version => "1.0";
}

bundle agent generate_files(template, data)
{
vars:
"content" string => string_mustache("$(template[mustache])", @(data));

@larsewi larsewi Aug 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems a bit overly complicated to use mustache templates here. Is it not easier to just write the content directly? It may be a little bit more code, but easier to read / understand when debugging this in the future.

Also I'd prefer using the test directory variables from default.sub.cf instead of polluting the /tmp/ directory.

@victormlg victormlg Aug 10, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It just seemed easier to maintain to me: instead of maintaining a hard coded string for each filetype, we have one general data container and one mustache for each filetype. Also, it doesn't pollute the folder with a lot of redundant files:

validfiledata.cf.json
validfiledata.cf.x.json
validfiledata.cf.yaml
valifiledata.cf.x.yaml
validfiledata.cf.csv
validfiledata.cf.x.csv
[...]

@larsewi larsewi Aug 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What I meant is that you write these files to the /tmp/ directory. While you could use $(G.testdir) instead, which gets cleaned up by the test framework if the test passes.


files:
"/tmp/config.$(template[format])"
create => "true",
content => "$(content)";

"/tmp/config.x.$(template[format])"
create => "true",
content => "-$(content)", # Add some text that makes the json and yaml invalid
if => strcmp("$(template[create_invalid])", "yes"); # csv and env are invalid ONLY when the file doesn't exist
}

bundle agent init
{
vars:
"config_data"
data => parsejson(
'{"hello" : "world", "bye" : "everyone", "pizza" : "hamburger"}'
);
@if feature(yaml)
"templates"
data => parsejson(
'[
{"format": "json", "mustache": "{{%-top-}}", "create_invalid" : "yes"},
{"format": "yaml", "mustache": "---\\n{{#-top-}}\\n{{@}}: \\\"{{.}}\\\"\\n{{/-top-}}", "create_invalid" : "yes"},
{"format": "csv", "mustache": "Key,Value\\r\\n{{#-top-}}\\r\\n{{@}},\\\"{{.}}\\\"\\r\\n{{/-top-}}", "create_invalid" : "no"},
{"format": "env", "mustache": "{{#-top-}}\\n{{{@}}}={{{.}}}\\n{{/-top-}}", "create_invalid" : "no"}
]'
);
@else
"templates"
data => parsejson(
'[
{"format": "json", "mustache": "{{%-top-}}", "create_invalid" : "yes"},
{"format": "csv", "mustache": "Key,Value\\r\\n{{#-top-}}\\r\\n{{@}},\\\"{{.}}\\\"\\r\\n{{/-top-}}", "create_invalid" : "no"},
{"format": "env", "mustache": "{{#-top-}}\\n{{{@}}}={{{.}}}\\n{{/-top-}}", "create_invalid" : "no"}
]'
);
@endif
"template_idx" slist => getindices(templates);

methods:
"generate"
usebundle => generate_files(
"@(templates[$(template_idx)])", @(config_data)
);
}

bundle agent test
{
vars:
"formats"
slist => maparray("$(init.templates[$(this.k)][format])", init.templates);

"valid_classes" slist => maplist("valid_$(this)", formats);
"invalid_classes" slist => maplist("invalid_$(this)", formats);

classes:
"valid_$(formats)"
expression => validfiledata("/tmp/config.$(formats)", "auto"),
scope => "namespace";

"invalid_$(formats)"
expression => validfiledata("/tmp/config.x.$(formats)", "auto"),
scope => "namespace";
}

bundle agent check
{
classes:
"valid" and => { @(test.valid_classes) };
"invalid" or => { @(test.invalid_classes) };

files:
"/tmp/config.x.$(test.formats)" delete => tidy;

reports:
valid.!invalid::
"$(this.promise_filename) Pass";

!valid|invalid::
"$(this.promise_filename) FAIL";
}
Loading