diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 60134779a..ed79378bf 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -254,6 +254,18 @@ class VisualizationAssertionError(AssertionError): __tracebackhide__ = True +def _filter_diff(category: str, ev: EvaluationResult) -> str: + """Expected-vs-actual lines for one filter category, or "" when they matched. + + Filters compare by exact equality on their canonical form, so "False" alone leaves + the reader guessing which part differed — the period, the granularity or the dataset. + """ + expected, actual = ev.expected_filters.get(category, []), ev.actual_filters.get(category, []) + if expected == actual: + return "" + return f" expected : {expected or 'none'}\n actual : {actual or 'none'}\n" + + def evaluate_agentic_visualization( host: str, token: str, @@ -393,8 +405,11 @@ def evaluate_agentic_visualization( f" actual : {sorted(ev.actual_dim_uris)}\n" f" Filters Correct : {ev.filters_correct}\n" f" date : {ev.filter_date_score}\n" + f"{_filter_diff('date', ev)}" f" ranking : {ev.filter_ranking_score}\n" + f"{_filter_diff('ranking', ev)}" f" attribute : {ev.filter_attribute_score}\n" + f"{_filter_diff('attribute', ev)}" f" Viz Type Hard : {ev.viz_type_hard}\n" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py index cf9f65746..ad98a2046 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py @@ -10,9 +10,12 @@ check_viz_type, get_dimension_uri_set, get_metric_uri_set, + normalized_filters, validate_cross_references, ) +_NO_FILTERS: dict[str, list[str]] = {"date": [], "ranking": [], "attribute": []} + @dataclass class EvaluationResult: @@ -31,6 +34,11 @@ class EvaluationResult: actual_metric_uris: set[str] expected_dim_uris: set[str] actual_dim_uris: set[str] + # Filters in the canonical form equality is tested on, keyed by the category each + # `filter_*_score` covers. Reported alongside the booleans because a filter mismatch + # is otherwise undiagnosable from a finished run. + expected_filters: dict[str, list[str]] + actual_filters: dict[str, list[str]] @property def strict_pass(self) -> bool: @@ -91,6 +99,8 @@ def _evaluate_visualization( actual_metric_uris=set(), expected_dim_uris=exp_dim_uris, actual_dim_uris=set(), + expected_filters=normalized_filters(expected), + actual_filters={category: values.copy() for category, values in _NO_FILTERS.items()}, ) cross_ref_valid, cross_ref_errors = validate_cross_references(actual) act_metric_uris = get_metric_uri_set(actual) @@ -112,6 +122,8 @@ def _evaluate_visualization( actual_metric_uris=act_metric_uris, expected_dim_uris=exp_dim_uris, actual_dim_uris=act_dim_uris, + expected_filters=normalized_filters(expected), + actual_filters=normalized_filters(actual), ) @@ -174,5 +186,7 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation "actual_metric_uris": sorted(ev.actual_metric_uris), "expected_dim_uris": sorted(ev.expected_dim_uris), "actual_dim_uris": sorted(ev.actual_dim_uris), + "expected_filters": ev.expected_filters, + "actual_filters": ev.actual_filters, }, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/scoring.py b/packages/gooddata-eval/src/gooddata_eval/core/scoring.py index a5913a791..0d554cd85 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/scoring.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/scoring.py @@ -188,6 +188,19 @@ def _split_and_normalize_filters(viz: CreatedVisualization) -> tuple[set[str], s return date_set, ranking_set, attr_set +def normalized_filters(viz: CreatedVisualization) -> dict[str, list[str]]: + """A visualization's filters exactly as `check_filters` compares them. + + Grouped by the three categories it scores separately and sorted for stable output. + Each entry is the canonical JSON string equality is tested on, so reporting the + expected and actual side by side shows precisely why a category did not match — + a `filter_date_score` of False otherwise gives no clue whether the period differed, + the granularity did, or the dataset the filter hangs off did. + """ + date_set, ranking_set, attr_set = _split_and_normalize_filters(viz) + return {"date": sorted(date_set), "ranking": sorted(ranking_set), "attribute": sorted(attr_set)} + + def check_filters(expected: CreatedVisualization, actual: CreatedVisualization) -> FilterScores: exp_date, exp_rank, exp_attr = _split_and_normalize_filters(expected) act_date, act_rank, act_attr = _split_and_normalize_filters(actual) diff --git a/packages/gooddata-eval/tests/test_scoring.py b/packages/gooddata-eval/tests/test_scoring.py index 15f16d383..873e30628 100644 --- a/packages/gooddata-eval/tests/test_scoring.py +++ b/packages/gooddata-eval/tests/test_scoring.py @@ -5,6 +5,7 @@ check_viz_type, get_dimension_uri_set, get_metric_uri_set, + normalized_filters, uri_to_display_name, validate_cross_references, ) @@ -161,3 +162,46 @@ def test_validate_cross_references_accepts_omitted_attribute_but_flags_missing_u ok, errors = validate_cross_references(no_using) assert ok is False assert "is required" in errors[0] + + +def test_normalized_filters_groups_by_scored_category(): + """The three categories `check_filters` scores separately, in the form it compares.""" + viz = CreatedVisualization.model_validate( + { + "id": "x", + "type": "bar_chart", + "query": { + "fields": {"m": {"using": "metric/spend"}, "d": {"using": "label/merchant.name"}}, + "filter_by": { + "f0": { + "type": "date_filter", + "using": "dataset/date", + "granularity": "MONTH", + "from": -1, + "to": -1, + }, + "f1": {"type": "ranking_filter", "using": "m", "top": 5}, + "f2": {"type": "attribute_filter", "using": "label/region", "state": {"include": ["EMEA"]}}, + }, + }, + "metrics": ["m"], + "view_by": ["d"], + } + ) + grouped = normalized_filters(viz) + assert set(grouped) == {"date", "ranking", "attribute"} + assert all(len(v) == 1 for v in grouped.values()) + # The ranking entry carries the substituted sole dimension, matching what equality sees. + assert '"dim_uri": "label/merchant.name"' in grouped["ranking"][0] + + +def test_normalized_filters_is_empty_per_category_when_unfiltered(): + viz = CreatedVisualization.model_validate( + { + "id": "x", + "type": "headline", + "query": {"fields": {"m": {"using": "metric/spend"}}, "filter_by": {}}, + "metrics": ["m"], + } + ) + assert normalized_filters(viz) == {"date": [], "ranking": [], "attribute": []} diff --git a/packages/gooddata-eval/tests/test_visualization_evaluator.py b/packages/gooddata-eval/tests/test_visualization_evaluator.py index 38d89e844..d99d364d6 100644 --- a/packages/gooddata-eval/tests/test_visualization_evaluator.py +++ b/packages/gooddata-eval/tests/test_visualization_evaluator.py @@ -127,3 +127,46 @@ def test_evaluator_passes_when_agent_omits_ranking_attribute_on_single_dim_viz() assert result.detail["filter_ranking_score"] is True assert result.detail["filters_correct"] is True assert result.passed is True + + +def _dated(granularity: str, frm: int, to: int): + return { + "id": "x", + "type": "column_chart", + "query": { + "fields": {"m_rev": {"using": "metric/revenue"}, "d_q": {"using": "label/date.quarter"}}, + "filter_by": { + "f_date": { + "type": "date_filter", + "using": "dataset/date", + "granularity": granularity, + "from": frm, + "to": to, + } + }, + }, + "metrics": ["m_rev"], + "view_by": ["d_q"], + } + + +def test_detail_reports_the_filters_that_were_compared(): + """A `filter_date_score` of False is undiagnosable from a finished run without them: + two encodings of the same period compare unequal and the booleans don't say which.""" + ev = get_evaluator("visualization") + result = ev.evaluate(_item(_dated("MONTH", -11, 0)), _chat_result_with(_dated("MONTH", -12, -1))) + + assert result.detail["filter_date_score"] is False + expected, actual = result.detail["expected_filters"], result.detail["actual_filters"] + assert '"from": -11' in expected["date"][0] + assert '"from": -12' in actual["date"][0] + assert expected["ranking"] == actual["ranking"] == [] + assert expected["attribute"] == actual["attribute"] == [] + + +def test_detail_filters_are_empty_when_no_visualization_was_created(): + ev = get_evaluator("visualization") + empty = ChatResult.model_validate({"textResponse": "what metric?", "toolCallEvents": []}) + result = ev.evaluate(_item(_dated("MONTH", -11, 0)), empty) + assert result.detail["actual_filters"] == {"date": [], "ranking": [], "attribute": []} + assert len(result.detail["expected_filters"]["date"]) == 1