diff --git a/eslint.config.js b/eslint.config.js index d3852e38ecd8..7d95356e7f1d 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -85,6 +85,7 @@ export default [ "web-common/src/components/modal/*.js", "web-common/src/features/dashboards/url-state/filters/expression.js", // generated by nearley "web-common/src/features/dashboards/url-state/time-ranges/rill-time.js", // generated by nearley + "web-common/src/features/dashboards/ephemeral-measures/measure-expression.js", // generated by nearley "web-local/build/*", "web-local/playwright-report/*", "web-local/playwright/*", diff --git a/proto/rill/ui/v1/dashboard.proto b/proto/rill/ui/v1/dashboard.proto index 6a7bf6e9319f..f05187d8f302 100644 --- a/proto/rill/ui/v1/dashboard.proto +++ b/proto/rill/ui/v1/dashboard.proto @@ -159,6 +159,9 @@ message DashboardState { // Per-measure conditional formatting (heatmap / data bar) for pivot cells. repeated PivotConditionalFormat pivot_conditional_formatting = 44; + + // Ephemeral measures defined for the explore. + repeated EphemeralMeasure ephemeral_measures = 45; } message DashboardTimeRange { @@ -184,6 +187,19 @@ message PivotElement { } } +// An ephemeral measure defined ad-hoc for an explore dashboard, +// derived from existing metrics view measures via an arithmetic expression. +message EphemeralMeasure { + // Query alias, e.g. "profit". Must not collide with metrics view field names. + string name = 1; + // Display name shown in the UI, e.g. "Profit". + string display_name = 2; + // Arithmetic expression over existing measure names, e.g. "revenue - cost". + string expression = 3; + // Optional format preset for rendering values. + string format_preset = 4; +} + // Conditional formatting applied to a measure's cells in a pivot table. message PivotConditionalFormat { string measure = 1; diff --git a/runtime/canvas/component.go b/runtime/canvas/component.go index fa215d16f17c..368605cbd672 100644 --- a/runtime/canvas/component.go +++ b/runtime/canvas/component.go @@ -592,28 +592,28 @@ func isEncodedTimeDimension(mv *runtimev1.MetricsViewSpec, fieldName string) boo return ok && v != int32(runtimev1.TimeGrain_TIME_GRAIN_UNSPECIFIED) } -// ephemeralMeasureNames extracts and validates the optional "ephemeral_measures" renderer property. +// ephemeralMeasureNames extracts and validates the optional "adhoc_measures" renderer property. // Each entry defines an ephemeral measure derived from existing measures via an arithmetic expression; // the returned set contains the names that may be referenced alongside the metrics view's own measures. func ephemeralMeasureNames(props map[string]any, mvn string, mv *runtimev1.MetricsViewSpec) (map[string]bool, error) { - raw, ok := props["ephemeral_measures"] + raw, ok := props["adhoc_measures"] if !ok || raw == nil { return nil, nil } list, ok := raw.([]any) if !ok { - return nil, errors.New("renderer property 'ephemeral_measures' must be an array") + return nil, errors.New("renderer property 'adhoc_measures' must be an array") } names := make(map[string]bool, len(list)) for _, item := range list { entry, ok := item.(map[string]any) if !ok { - return nil, errors.New("entries in 'ephemeral_measures' must be objects with 'name' and 'expression'") + return nil, errors.New("entries in 'adhoc_measures' must be objects with 'name' and 'expression'") } name, _ := entry["name"].(string) expression, _ := entry["expression"].(string) if name == "" || expression == "" { - return nil, errors.New("entries in 'ephemeral_measures' must have a non-empty 'name' and 'expression'") + return nil, errors.New("entries in 'adhoc_measures' must have a non-empty 'name' and 'expression'") } // Mirror metricsview.AST.checkNameForComputedField, which also rejects the time dimension. // It is often absent from mv.Dimensions, so checking it here surfaces the collision at parse time rather than at query time. diff --git a/runtime/canvas/component_test.go b/runtime/canvas/component_test.go index e9eef9fd509a..5d6b6662f20c 100644 --- a/runtime/canvas/component_test.go +++ b/runtime/canvas/component_test.go @@ -979,7 +979,7 @@ type: component kpi_grid: metrics_view: mv1 measures: [y, profit] - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: y - z @@ -995,7 +995,7 @@ leaderboard: metrics_view: mv1 measures: [profit] dimensions: [foo] - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: y - z @@ -1010,7 +1010,7 @@ type: component table: metrics_view: mv1 columns: [foo, y, profit] - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: y - z @@ -1026,7 +1026,7 @@ pivot: metrics_view: mv1 measures: [profit] row_dimensions: [foo] - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: y - z @@ -1047,7 +1047,7 @@ bar_chart: field: profit type: quantitative fields: [y, profit] - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: y - z @@ -1067,7 +1067,7 @@ pie_chart: color: field: foo type: nominal - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: y - z @@ -1087,7 +1087,7 @@ heatmap: color: field: profit type: quantitative - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: y - z @@ -1119,7 +1119,7 @@ type: component kpi_grid: metrics_view: mv1 measures: [profit] - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: sum(y) @@ -1135,7 +1135,7 @@ type: component kpi_grid: metrics_view: mv1 measures: [profit] - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: y - unknown @@ -1151,7 +1151,7 @@ type: component kpi_grid: metrics_view: mv1 measures: [missing] - ephemeral_measures: + adhoc_measures: - name: profit display_name: Profit expression: y - z diff --git a/runtime/drivers/clickhouse/dialect.go b/runtime/drivers/clickhouse/dialect.go index b32f9622624b..baa47eaa8565 100644 --- a/runtime/drivers/clickhouse/dialect.go +++ b/runtime/drivers/clickhouse/dialect.go @@ -93,6 +93,26 @@ func (d *dialect) CastToDataType(typ runtimev1.Type_Code) (string, error) { } } +// OrderByExpression and OrderByAliasExpression place NULLs last regardless of the sort direction, +// matching the behavior of the DuckDB dialect. +func (d *dialect) OrderByExpression(name string, desc bool) string { + res := d.EscapeIdentifier(name) + if desc { + res += " DESC" + } + res += " NULLS LAST" + return res +} + +func (d *dialect) OrderByAliasExpression(name string, desc bool) string { + res := d.EscapeAlias(name) + if desc { + res += " DESC" + } + res += " NULLS LAST" + return res +} + func (d *dialect) JoinOnExpression(lhs, rhs string) string { return fmt.Sprintf("isNotDistinctFrom(%s, %s)", lhs, rhs) } diff --git a/runtime/drivers/dialect.go b/runtime/drivers/dialect.go index 445f2ee5323a..d7153c6c5140 100644 --- a/runtime/drivers/dialect.go +++ b/runtime/drivers/dialect.go @@ -275,8 +275,9 @@ func (b *BaseDialect) CastToDataType(typ runtimev1.Type_Code) (string, error) { } } +// SafeDivideExpression returns a division that yields NULL instead of an error, infinity or NaN when the denominator is zero. func (b *BaseDialect) SafeDivideExpression(numExpr, denExpr string) string { - return fmt.Sprintf("(%s)/CAST(%s AS DOUBLE)", numExpr, denExpr) + return fmt.Sprintf("(%s)/NULLIF(CAST(%s AS DOUBLE), 0)", numExpr, denExpr) } func (b *BaseDialect) OrderByExpression(name string, desc bool) string { diff --git a/runtime/metricsview/ast.go b/runtime/metricsview/ast.go index 49b823dde57c..67fd5b9c9496 100644 --- a/runtime/metricsview/ast.go +++ b/runtime/metricsview/ast.go @@ -493,7 +493,7 @@ func (a *AST) ResolveMeasure(qm Measure, visible bool) (*runtimev1.MetricsViewSp } if qm.Compute.ComparisonValue != nil { - m, err := a.LookupMeasure(qm.Compute.ComparisonValue.Measure, visible) + m, err := a.resolveReferencedMeasure(qm.Compute.ComparisonValue.Measure, visible) if err != nil { return nil, err } @@ -513,7 +513,7 @@ func (a *AST) ResolveMeasure(qm Measure, visible bool) (*runtimev1.MetricsViewSp } if qm.Compute.ComparisonDelta != nil { - m, err := a.LookupMeasure(qm.Compute.ComparisonDelta.Measure, visible) + m, err := a.resolveReferencedMeasure(qm.Compute.ComparisonDelta.Measure, visible) if err != nil { return nil, err } @@ -533,7 +533,7 @@ func (a *AST) ResolveMeasure(qm Measure, visible bool) (*runtimev1.MetricsViewSp } if qm.Compute.ComparisonRatio != nil { - m, err := a.LookupMeasure(qm.Compute.ComparisonRatio.Measure, visible) + m, err := a.resolveReferencedMeasure(qm.Compute.ComparisonRatio.Measure, visible) if err != nil { return nil, err } @@ -559,7 +559,7 @@ func (a *AST) ResolveMeasure(qm Measure, visible bool) (*runtimev1.MetricsViewSp return nil, fmt.Errorf("totals not computed for %s", qm.Name) } - m, err := a.LookupMeasure(qm.Compute.PercentOfTotal.Measure, visible) + m, err := a.resolveReferencedMeasure(qm.Compute.PercentOfTotal.Measure, visible) if err != nil { return nil, err } @@ -731,6 +731,27 @@ func (a *AST) LookupMeasure(name string, visible bool) (*runtimev1.MetricsViewSp return nil, fmt.Errorf("measure %q not found", name) } +// resolveReferencedMeasure resolves a measure referenced by name from another measure, +// such as the base measure of a comparison delta or a measure referenced by a derived measure. +// The referenced measure is either a measure in the metrics view or an expression measure defined in the same query, +// which lets ad-hoc measures support comparisons without being declared in the metrics view. +// Expression measures may only reference metrics view measures, so the recursion is at most one level deep. +func (a *AST) resolveReferencedMeasure(name string, visible bool) (*runtimev1.MetricsViewSpec_Measure, error) { + for _, m := range a.MetricsView.Measures { + if m.Name == name { + return a.LookupMeasure(name, visible) + } + } + + for _, qm := range a.Query.Measures { + if qm.Name == name && qm.Compute != nil && qm.Compute.Expression != nil { + return a.ResolveMeasure(qm, visible) + } + } + + return nil, fmt.Errorf("measure %q not found", name) +} + // GenerateIdentifier generates a unique table identifier for use in the AST. func (a *AST) GenerateIdentifier() string { tmp := fmt.Sprintf("t%d", a.nextIdentifier) @@ -1065,7 +1086,7 @@ func (a *AST) addReferencedMeasuresToScope(n *SelectNode, referencedMeasures []s for _, rm := range referencedMeasures { // Note we pass visible==false because the measure won't be projected into the current node's SELECT list, only brought into scope for derived measures. - m, err := a.LookupMeasure(rm, false) + m, err := a.resolveReferencedMeasure(rm, false) if err != nil { return err } diff --git a/runtime/metricsview/executor/executor_rewrite_percent_of_totals.go b/runtime/metricsview/executor/executor_rewrite_percent_of_totals.go index c21d027b78e8..acfee9498861 100644 --- a/runtime/metricsview/executor/executor_rewrite_percent_of_totals.go +++ b/runtime/metricsview/executor/executor_rewrite_percent_of_totals.go @@ -14,9 +14,13 @@ func (e *Executor) rewritePercentOfTotals(ctx context.Context, qry *metricsview. var measureIndices []int for i, measure := range qry.Measures { if measure.Compute != nil && measure.Compute.PercentOfTotal != nil { - measures = append(measures, metricsview.Measure{ - Name: measure.Compute.PercentOfTotal.Measure, - }) + // The referenced measure is usually a metrics view measure, but it may also be an expression measure defined in the same query, + // in which case the totals query must carry its expression compute. + totalOf := metricsview.Measure{Name: measure.Compute.PercentOfTotal.Measure} + if qm, ok := queryExpressionMeasure(qry, totalOf.Name); ok { + totalOf = qm + } + measures = append(measures, totalOf) measureIndices = append(measureIndices, i) } } diff --git a/runtime/metricsview/executor/executor_rewrite_rollup.go b/runtime/metricsview/executor/executor_rewrite_rollup.go index 5e04e2695a33..b2e9fc60bc5d 100644 --- a/runtime/metricsview/executor/executor_rewrite_rollup.go +++ b/runtime/metricsview/executor/executor_rewrite_rollup.go @@ -408,6 +408,12 @@ func rollupEligible(rollup *runtimev1.MetricsViewSpec_Rollup, qry *metricsview.Q return false, rejectComputedMeasure, nil } for _, refName := range refNames { + // A comparison compute may reference an expression measure defined in the same query. + // That measure is not in the rollup, but the measures it references may be, so check those instead. + // Its own rollup safety is checked when the loop reaches it. + if _, ok := queryExpressionMeasure(qry, refName); ok { + continue + } if !rollupMeasures[strings.ToLower(refName)] { return false, rejectMeasureMissing, nil } @@ -583,6 +589,16 @@ func rollupSafeReferencedMeasures(c *metricsview.MeasureCompute) ([]string, bool return nil, false } +// queryExpressionMeasure returns the expression measure with the given name defined in the query, if any. +func queryExpressionMeasure(qry *metricsview.Query, name string) (metricsview.Measure, bool) { + for _, qm := range qry.Measures { + if qm.Name == name && qm.Compute != nil && qm.Compute.Expression != nil { + return qm, true + } + } + return metricsview.Measure{}, false +} + // normalizeTimezone validates and normalizes a timezone string for comparison. // It normalizes UTC variants (empty, "UTC", "Etc/UTC") to "UTC". // Note: Go's time.LoadLocation preserves the input name, so aliases like "US/Eastern" diff --git a/runtime/queries/metricsview_aggregation_expression_test.go b/runtime/queries/metricsview_aggregation_expression_test.go index 3d6fc66868ce..dd72c6f93fb5 100644 --- a/runtime/queries/metricsview_aggregation_expression_test.go +++ b/runtime/queries/metricsview_aggregation_expression_test.go @@ -2,6 +2,7 @@ package queries_test import ( "context" + "math" "testing" "time" @@ -11,6 +12,7 @@ import ( "github.com/rilldata/rill/runtime/queries" "github.com/rilldata/rill/runtime/testruntime" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -348,3 +350,221 @@ func TestMetricsViewTimeSeries_expression_rejects_other_computes(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "only the `expression` compute is supported") } + +func comparisonDeltaMeasure(name, measure string) *runtimev1.MetricsViewAggregationMeasure { + return &runtimev1.MetricsViewAggregationMeasure{ + Name: name, + Compute: &runtimev1.MetricsViewAggregationMeasure_ComparisonDelta{ + ComparisonDelta: &runtimev1.MetricsViewAggregationMeasureComputeComparisonDelta{Measure: measure}, + }, + } +} + +func comparisonRatioMeasure(name, measure string) *runtimev1.MetricsViewAggregationMeasure { + return &runtimev1.MetricsViewAggregationMeasure{ + Name: name, + Compute: &runtimev1.MetricsViewAggregationMeasure_ComparisonRatio{ + ComparisonRatio: &runtimev1.MetricsViewAggregationMeasureComputeComparisonRatio{Measure: measure}, + }, + } +} + +func comparisonValueMeasure(name, measure string) *runtimev1.MetricsViewAggregationMeasure { + return &runtimev1.MetricsViewAggregationMeasure{ + Name: name, + Compute: &runtimev1.MetricsViewAggregationMeasure_ComparisonValue{ + ComparisonValue: &runtimev1.MetricsViewAggregationMeasureComputeComparisonValue{Measure: measure}, + }, + } +} + +func percentOfTotalMeasure(name, measure string) *runtimev1.MetricsViewAggregationMeasure { + return &runtimev1.MetricsViewAggregationMeasure{ + Name: name, + Compute: &runtimev1.MetricsViewAggregationMeasure_PercentOfTotal{ + PercentOfTotal: &runtimev1.MetricsViewAggregationMeasureComputePercentOfTotal{Measure: measure}, + }, + } +} + +// Comparison computes may reference an expression measure defined in the same query. +// "doubled" is 2 * measure_1, so its comparison value and delta are twice those of measure_1 and its ratio is the same. +func TestMetricsViewsAggregation_expression_comparison_of_expression(t *testing.T) { + rt, instanceID := testruntime.NewInstanceForProject(t, "ad_bids") + + limit := int64(10) + q := &queries.MetricsViewAggregation{ + MetricsViewName: "ad_bids_metrics", + Dimensions: []*runtimev1.MetricsViewAggregationDimension{ + {Name: "pub"}, + }, + Measures: []*runtimev1.MetricsViewAggregationMeasure{ + {Name: "measure_1"}, + comparisonValueMeasure("measure_1__prev", "measure_1"), + comparisonDeltaMeasure("measure_1__delta", "measure_1"), + comparisonRatioMeasure("measure_1__ratio", "measure_1"), + expressionMeasure("doubled", "measure_1 * 2"), + comparisonValueMeasure("doubled__prev", "doubled"), + comparisonDeltaMeasure("doubled__delta", "doubled"), + comparisonRatioMeasure("doubled__ratio", "doubled"), + }, + Sort: []*runtimev1.MetricsViewAggregationSort{ + {Name: "doubled__delta", Desc: true}, + }, + TimeRange: &runtimev1.TimeRange{ + Start: timestamppb.New(time.Date(2022, 1, 1, 0, 0, 0, 0, time.UTC)), + End: timestamppb.New(time.Date(2022, 1, 2, 0, 0, 0, 0, time.UTC)), + }, + ComparisonTimeRange: &runtimev1.TimeRange{ + Start: timestamppb.New(time.Date(2022, 1, 2, 0, 0, 0, 0, time.UTC)), + End: timestamppb.New(time.Date(2022, 1, 3, 0, 0, 0, 0, time.UTC)), + }, + Limit: &limit, + SecurityClaims: testClaims(), + } + err := q.Resolve(context.Background(), rt, instanceID, 0) + require.NoError(t, err) + require.NotEmpty(t, q.Result.Data) + + checked := 0 + for _, row := range q.Result.Data { + f := row.Fields + if _, isNull := f["measure_1__delta"].GetKind().(*structpb.Value_NullValue); isNull { + // Rows present in only one of the time ranges have no comparison. Skip them. + continue + } + checked++ + require.InDelta(t, 2*f["measure_1"].GetNumberValue(), f["doubled"].GetNumberValue(), 1e-9) + require.InDelta(t, 2*f["measure_1__prev"].GetNumberValue(), f["doubled__prev"].GetNumberValue(), 1e-9) + require.InDelta(t, 2*f["measure_1__delta"].GetNumberValue(), f["doubled__delta"].GetNumberValue(), 1e-9) + require.InDelta(t, f["measure_1__ratio"].GetNumberValue(), f["doubled__ratio"].GetNumberValue(), 1e-9) + } + require.NotZero(t, checked) + + // The rows are sorted by the expression measure's delta. + prev := math.Inf(1) + for _, row := range q.Result.Data { + v := row.Fields["doubled__delta"] + if _, isNull := v.GetKind().(*structpb.Value_NullValue); isNull { + continue + } + require.LessOrEqual(t, v.GetNumberValue(), prev) + prev = v.GetNumberValue() + } +} + +// Percent of total may reference an expression measure defined in the same query. +// The totals query must carry the expression too. +func TestMetricsViewsAggregation_expression_percent_of_total_of_expression(t *testing.T) { + rt, instanceID := testruntime.NewInstanceForProject(t, "ad_bids_2rows") + + q := &queries.MetricsViewAggregation{ + MetricsViewName: "ad_bids_metrics", + Dimensions: []*runtimev1.MetricsViewAggregationDimension{ + {Name: "domain"}, + }, + Measures: []*runtimev1.MetricsViewAggregationMeasure{ + {Name: "measure_2"}, + percentOfTotalMeasure("measure_2__pot", "measure_2"), + expressionMeasure("tripled", "measure_2 * 3"), + percentOfTotalMeasure("tripled__pot", "tripled"), + }, + Sort: []*runtimev1.MetricsViewAggregationSort{ + {Name: "domain"}, + }, + SecurityClaims: testClaims(), + } + err := q.Resolve(context.Background(), rt, instanceID, 0) + require.NoError(t, err) + require.Len(t, q.Result.Data, 2) + for _, row := range q.Result.Data { + f := row.Fields + require.InDelta(t, f["measure_2__pot"].GetNumberValue(), f["tripled__pot"].GetNumberValue(), 1e-9) + } + // msn.com has 2 of 3 impressions, yahoo.com has 1 of 3. + require.InDelta(t, 2.0/3.0, q.Result.Data[0].Fields["tripled__pot"].GetNumberValue(), 1e-9) + require.InDelta(t, 1.0/3.0, q.Result.Data[1].Fields["tripled__pot"].GetNumberValue(), 1e-9) +} + +func TestMetricsViewsAggregation_expression_comparison_errors(t *testing.T) { + rt, instanceID := testruntime.NewInstanceForProject(t, "ad_bids_2rows") + + cases := []struct { + name string + measures []*runtimev1.MetricsViewAggregationMeasure + errContains string + }{ + { + name: "unknown measure", + measures: []*runtimev1.MetricsViewAggregationMeasure{comparisonDeltaMeasure("x", "unknown_measure")}, + errContains: "not found", + }, + { + name: "expression measure not in the query", + measures: []*runtimev1.MetricsViewAggregationMeasure{comparisonDeltaMeasure("x", "profit")}, + errContains: `measure "profit" not found`, + }, + { + name: "expression measure with an invalid expression", + measures: []*runtimev1.MetricsViewAggregationMeasure{ + expressionMeasure("profit", "sum(measure_1)"), + comparisonRatioMeasure("x", "profit"), + }, + errContains: "aggregate function", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + q := &queries.MetricsViewAggregation{ + MetricsViewName: "ad_bids_metrics", + Measures: c.measures, + TimeRange: &runtimev1.TimeRange{ + Start: timestamppb.New(time.Date(2022, 1, 1, 0, 0, 0, 0, time.UTC)), + End: timestamppb.New(time.Date(2022, 1, 2, 0, 0, 0, 0, time.UTC)), + }, + ComparisonTimeRange: &runtimev1.TimeRange{ + Start: timestamppb.New(time.Date(2022, 1, 2, 0, 0, 0, 0, time.UTC)), + End: timestamppb.New(time.Date(2022, 1, 3, 0, 0, 0, 0, time.UTC)), + }, + SecurityClaims: testClaims(), + } + err := q.Resolve(context.Background(), rt, instanceID, 0) + require.Error(t, err) + require.Contains(t, err.Error(), c.errContains) + }) + } +} + +// A comparison of an expression measure is subject to the same security policy as the expression itself. +func TestMetricsViewsAggregation_expression_comparison_security(t *testing.T) { + rt, instanceID := testruntime.NewInstanceForProject(t, "ad_bids") + + newQuery := func() *queries.MetricsViewAggregation { + return &queries.MetricsViewAggregation{ + MetricsViewName: "ad_bids_mini_metrics_with_policy", + Measures: []*runtimev1.MetricsViewAggregationMeasure{ + expressionMeasure("net", `"total impressions" - "total volume"`), + comparisonDeltaMeasure("net__delta", "net"), + }, + TimeRange: &runtimev1.TimeRange{ + Start: timestamppb.New(time.Date(2022, 1, 1, 0, 0, 0, 0, time.UTC)), + End: timestamppb.New(time.Date(2022, 1, 2, 0, 0, 0, 0, time.UTC)), + }, + ComparisonTimeRange: &runtimev1.TimeRange{ + Start: timestamppb.New(time.Date(2022, 1, 2, 0, 0, 0, 0, time.UTC)), + End: timestamppb.New(time.Date(2022, 1, 3, 0, 0, 0, 0, time.UTC)), + }, + } + } + + q := newQuery() + q.SecurityClaims = &runtime.SecurityClaims{UserAttributes: map[string]any{"domain": "yahoo.com", "email": "user@yahoo.com"}} + err := q.Resolve(context.Background(), rt, instanceID, 0) + require.Error(t, err) + require.ErrorContains(t, err, "total volume") + + q = newQuery() + q.SecurityClaims = &runtime.SecurityClaims{UserAttributes: map[string]any{"domain": "msn.com", "email": "user@msn.com"}} + err = q.Resolve(context.Background(), rt, instanceID, 0) + require.NoError(t, err) +} diff --git a/web-admin/src/features/public-urls/CreatePublicURLForm.svelte b/web-admin/src/features/public-urls/CreatePublicURLForm.svelte index 4dc7d436f99e..008d4f47bff5 100644 --- a/web-admin/src/features/public-urls/CreatePublicURLForm.svelte +++ b/web-admin/src/features/public-urls/CreatePublicURLForm.svelte @@ -82,9 +82,13 @@ dashboardKind, expressionFilterManager, ); - let { fields, sanitizedState, queryTimeStart, queryTimeEnd } = $derived( - $sanitisedFilterState, - ); + let { + fields, + sanitizedState, + droppedEphemeralMeasures, + queryTimeStart, + queryTimeEnd, + } = $derived($sanitisedFilterState); const formId = "create-public-url-form"; @@ -259,6 +263,24 @@ {/if} {/if} + {#if droppedEphemeralMeasures.length > 0} +
+

+ {m.public_url_adhoc_measures_omitted()} +

+ +
+ {/if} + +{/if} diff --git a/web-common/src/components/searchable-filter-menu/SearchableFilterChip.svelte b/web-common/src/components/searchable-filter-menu/SearchableFilterChip.svelte index c17742f9cd30..54af6c978d9f 100644 --- a/web-common/src/components/searchable-filter-menu/SearchableFilterChip.svelte +++ b/web-common/src/components/searchable-filter-menu/SearchableFilterChip.svelte @@ -12,6 +12,9 @@ export let tooltipText: string; export let label: string; export let onSelect: (name: string) => void; + // When set, ephemeral items in the menu get an edit button that closes the + // menu and invokes this with the item name. + export let onEditItem: ((name: string) => void) | undefined = undefined; let open = false; let searchText = ""; @@ -35,8 +38,9 @@ suppress={open} > -
- {label} +
+ {label} +
@@ -54,5 +58,11 @@ {onSelect} selectedItems={[selectedItems]} selectableGroups={[{ name: "", items: selectableItems }]} + onEditItem={onEditItem + ? (name) => { + open = false; + onEditItem?.(name); + } + : undefined} /> diff --git a/web-common/src/components/searchable-filter-menu/SearchableFilterSelectableItem.ts b/web-common/src/components/searchable-filter-menu/SearchableFilterSelectableItem.ts index 7ed2aeb5f033..fc74204b0e44 100644 --- a/web-common/src/components/searchable-filter-menu/SearchableFilterSelectableItem.ts +++ b/web-common/src/components/searchable-filter-menu/SearchableFilterSelectableItem.ts @@ -7,4 +7,8 @@ export interface SearchableFilterSelectableGroup { export interface SearchableFilterSelectableItem { name: string; label: string; + // Shown as a native tooltip on the menu row. + description?: string; + // Marks ephemeral measures with an fx icon. + ephemeral?: boolean; } diff --git a/web-common/src/components/searchable-filter-menu/SearchableMenuContent.svelte b/web-common/src/components/searchable-filter-menu/SearchableMenuContent.svelte index 75595f2c01b5..b791dffa3b42 100644 --- a/web-common/src/components/searchable-filter-menu/SearchableMenuContent.svelte +++ b/web-common/src/components/searchable-filter-menu/SearchableMenuContent.svelte @@ -3,6 +3,7 @@ import * as DropdownMenu from "@rilldata/web-common/components/dropdown-menu"; import type { SearchableFilterSelectableGroup } from "@rilldata/web-common/components/searchable-filter-menu/SearchableFilterSelectableItem"; import { matchSorter } from "match-sorter"; + import { PencilIcon } from "lucide-svelte"; import Button from "../button/Button.svelte"; import { Search } from "../search"; @@ -21,6 +22,8 @@ export let side: "top" | "right" | "bottom" | "left" = "bottom"; export let onSelect: (name: string) => void; export let onToggleSelectAll: () => void = voidFn; + // When set, items flagged `ephemeral` get an edit button invoking this. + export let onEditItem: ((name: string) => void) | undefined = undefined; $: allSelected = selectableGroups.every((g, i) => { return ( @@ -84,41 +87,96 @@ {label ?? name} {/if} - {#each items as { name, label } (name)} + {#each items as { name, label, description, ephemeral } (name)} {@const selected = selectedItems[index]?.includes(name)} - { - if (requireSelection && singleSelection && selected) return; - - onSelect(name); - }} - > - { + if (requireSelection && singleSelection && selected) return; + + onSelect(name); + }} + > + + {#if label.length > 240} + {label.slice(0, 240)}... + {:else} + {label} + {/if} + {#if ephemeral} + ƒx + {/if} + + {#if ephemeral && onEditItem} + + {/if} + + {:else} + { + if (requireSelection && singleSelection && selected) return; + + onSelect(name); + }} > - {#if label.length > 240} - {label.slice(0, 240)}... - {:else} - {label} + + {#if label.length > 240} + {label.slice(0, 240)}... + {:else} + {label} + {/if} + {#if ephemeral} + ƒx + {/if} + + {#if ephemeral && onEditItem} + {/if} - - + + {/if} {:else}
- {#if numSelectedNotShown && showHiddenSelectionsCount}
{m.common_other_values_selected({ count: numSelectedNotShown })} @@ -153,6 +210,12 @@ {/if} {/if} + + {#if $$slots.action} +
+ +
+ {/if} diff --git a/web-common/src/features/dashboards/ephemeral-measures/EphemeralMeasureDialog.svelte b/web-common/src/features/dashboards/ephemeral-measures/EphemeralMeasureDialog.svelte new file mode 100644 index 000000000000..ce17e8984462 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/EphemeralMeasureDialog.svelte @@ -0,0 +1,215 @@ + + + { + if (!open) close(); + }} +> + + + + {editingDef + ? m.dashboard_pivot_ephemeral_edit_title() + : m.dashboard_pivot_ephemeral_new_title()} + + + +
+ + + + + {#if referenceableMeasures.length} +
+ + {m.dashboard_pivot_ephemeral_insert_measure()} + +
+ {#each referenceableMeasures as mes (mes.name)} + + {/each} +
+
+ {/if} + + + + {#if saveError} +

{saveError}

+ {/if} +
+ + + {#if editingDef} +
+ +
+ {/if} + + + + + diff --git a/web-common/src/features/dashboards/ephemeral-measures/canvas.ts b/web-common/src/features/dashboards/ephemeral-measures/canvas.ts new file mode 100644 index 000000000000..90c790bd861f --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/canvas.ts @@ -0,0 +1,34 @@ +import type { EphemeralMeasureDef } from "./types"; + +// YAML shape of an ephemeral measure in a canvas component spec (snake_case). +export interface EphemeralMeasureSpec { + name: string; + display_name: string; + expression: string; + format_preset?: string; +} + +export function ephemeralSpecsToDefs( + specs: EphemeralMeasureSpec[] | undefined, +): EphemeralMeasureDef[] | undefined { + if (!specs?.length) return undefined; + return specs + .filter((spec) => spec?.name && spec?.expression) + .map((spec) => ({ + name: spec.name, + displayName: spec.display_name || spec.name, + expression: spec.expression, + ...(spec.format_preset ? { formatPreset: spec.format_preset } : {}), + })); +} + +export function ephemeralDefsToSpecs( + defs: EphemeralMeasureDef[] | undefined, +): EphemeralMeasureSpec[] { + return (defs ?? []).map((def) => ({ + name: def.name, + display_name: def.displayName, + expression: def.expression, + ...(def.formatPreset ? { format_preset: def.formatPreset } : {}), + })); +} diff --git a/web-common/src/features/dashboards/ephemeral-measures/dialog-store.ts b/web-common/src/features/dashboards/ephemeral-measures/dialog-store.ts new file mode 100644 index 000000000000..8c8c935f5eed --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/dialog-store.ts @@ -0,0 +1,8 @@ +import { writable } from "svelte/store"; +import type { EphemeralMeasureDef } from "./types"; + +// State of the ephemeral measure dialog. null = closed; +// `def` present = editing an existing definition, otherwise creating a new one. +export const ephemeralMeasureDialog = writable<{ + def?: EphemeralMeasureDef; +} | null>(null); diff --git a/web-common/src/features/dashboards/ephemeral-measures/ephemeral-measure-queries.spec.ts b/web-common/src/features/dashboards/ephemeral-measures/ephemeral-measure-queries.spec.ts new file mode 100644 index 000000000000..6b499d46255e --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/ephemeral-measure-queries.spec.ts @@ -0,0 +1,138 @@ +import { DashboardFetchMocks } from "@rilldata/web-common/features/dashboards/dashboard-fetch-mocks"; +import { metricsExplorerStore } from "@rilldata/web-common/features/dashboards/stores/dashboard-stores"; +import { + AD_BIDS_BID_PRICE_MEASURE, + AD_BIDS_EXPLORE_INIT, + AD_BIDS_EXPLORE_NAME, + AD_BIDS_IMPRESSIONS_MEASURE, + AD_BIDS_METRICS_INIT_WITH_TIME, + AD_BIDS_NAME, + AD_BIDS_PUBLISHER_DIMENSION, +} from "@rilldata/web-common/features/dashboards/stores/test-data/data"; +import { initStateManagers } from "@rilldata/web-common/features/dashboards/stores/test-data/helpers"; +import { + AD_BIDS_APPLY_PUB_DIMENSION_FILTER, + applyMutationsToDashboard, +} from "@rilldata/web-common/features/dashboards/stores/test-data/store-mutations"; +import { createTimeDimensionDataStore } from "@rilldata/web-common/features/dashboards/time-dimension-details/time-dimension-data-store"; +import { createTimeSeriesDataStore } from "@rilldata/web-common/features/dashboards/time-series/timeseries-data-store"; +import { asyncWait } from "@rilldata/web-common/lib/waitUtils"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const EPHEMERAL_MEASURE = { + name: "bid_price_per_impression", + displayName: "Bid price per impression", + expression: `${AD_BIDS_BID_PRICE_MEASURE} / ${AD_BIDS_IMPRESSIONS_MEASURE}`, +}; + +describe("ephemeral measures in explore queries", () => { + const dashboardFetchMocks = DashboardFetchMocks.useDashboardFetchMocks(); + + let requestBodies: string[] = []; + + beforeEach(() => { + requestBodies = []; + const inner = globalThis.fetch; + vi.stubGlobal("fetch", (url: any, opts: any) => { + const body = opts?.body; + if (body) { + requestBodies.push( + typeof body === "string" + ? body + : new TextDecoder().decode(body as ArrayBufferView), + ); + } + return inner(url, opts); + }); + + dashboardFetchMocks.mockMetricsExplore( + AD_BIDS_EXPLORE_NAME, + AD_BIDS_METRICS_INIT_WITH_TIME, + AD_BIDS_EXPLORE_INIT, + ); + dashboardFetchMocks.mockTimeRangeSummary(AD_BIDS_NAME, { + min: "2022-01-01", + max: "2022-03-31", + }); + dashboardFetchMocks.mockMetricsViewTimeRanges( + AD_BIDS_NAME, + "2022-01-01T00:00:00Z", + "2022-03-31T00:00:00Z", + ); + dashboardFetchMocks.mockMetricsViewAggregation( + new RegExp(`"name":"${AD_BIDS_PUBLISHER_DIMENSION}"`), + { + schema: { fields: [{ name: AD_BIDS_PUBLISHER_DIMENSION }] }, + data: [ + { [AD_BIDS_PUBLISHER_DIMENSION]: "Google" }, + { [AD_BIDS_PUBLISHER_DIMENSION]: "Facebook" }, + ], + }, + ); + }); + + it("keeps the definition attached in the TDD dimension comparison", async () => { + await runScenario({ expandEphemeralMeasureInTdd: true }); + expectEveryReferenceCarriesTheDefinition(requestBodies); + }); + + it("keeps the definition attached in the explore dimension comparison", async () => { + await runScenario({ expandEphemeralMeasureInTdd: false }); + expectEveryReferenceCarriesTheDefinition(requestBodies); + }); + + async function runScenario({ + expandEphemeralMeasureInTdd, + }: { + expandEphemeralMeasureInTdd: boolean; + }) { + const { stateManagers, destroy } = initStateManagers(); + + metricsExplorerStore.addEphemeralMeasure( + AD_BIDS_EXPLORE_NAME, + EPHEMERAL_MEASURE, + ); + metricsExplorerStore.setComparisonDimension( + AD_BIDS_EXPLORE_NAME, + AD_BIDS_PUBLISHER_DIMENSION, + ); + if (expandEphemeralMeasureInTdd) { + metricsExplorerStore.setExpandedMeasureName( + AD_BIDS_EXPLORE_NAME, + EPHEMERAL_MEASURE.name, + ); + } else { + // The explore chart reads its comparison values from the dimension + // filter, so one has to be applied for those queries to run. + await applyMutationsToDashboard( + AD_BIDS_EXPLORE_NAME, + [AD_BIDS_APPLY_PUB_DIMENSION_FILTER], + stateManagers.expressionFilterManager, + ); + } + + const unsubs = [ + createTimeSeriesDataStore(stateManagers).subscribe(() => {}), + createTimeDimensionDataStore(stateManagers).subscribe(() => {}), + ]; + await asyncWait(500); + unsubs.forEach((u) => u()); + destroy(); + } +}); + +/** + * A request is broken if it names the ephemeral measure but carries neither the + * `expression` compute (aggregation requests) nor an `ephemeralMeasures` entry + * (time series requests): the runtime then resolves the name against the metrics + * view and fails with `measure "..." not found`. + */ +function expectEveryReferenceCarriesTheDefinition(bodies: string[]) { + const referencing = bodies.filter((body) => + body.includes(EPHEMERAL_MEASURE.name), + ); + expect(referencing.length).toBeGreaterThan(0); + expect( + referencing.filter((body) => !body.includes(EPHEMERAL_MEASURE.expression)), + ).toEqual([]); +} diff --git a/web-common/src/features/dashboards/ephemeral-measures/ephemeral-measures.spec.ts b/web-common/src/features/dashboards/ephemeral-measures/ephemeral-measures.spec.ts new file mode 100644 index 000000000000..7567cb360215 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/ephemeral-measures.spec.ts @@ -0,0 +1,158 @@ +import { describe, it, expect } from "vitest"; +import type { V1MetricsViewAggregationMeasure } from "@rilldata/web-common/runtime-client"; +import { prepareMeasuresForRequest } from "../pivot/pivot-utils"; +import type { EphemeralMeasureDef } from "./types"; +import { + slugifyEphemeralMeasureName, + validateEphemeralMeasureDef, + validateEphemeralMeasureName, +} from "./validation"; + +const PROFIT: EphemeralMeasureDef = { + name: "profit", + displayName: "Profit", + expression: "revenue - cost", +}; + +describe("prepareMeasuresForRequest", () => { + it("attaches the expression compute to ephemeral measures", () => { + const measures: V1MetricsViewAggregationMeasure[] = [ + { name: "revenue" }, + { name: "profit" }, + ]; + expect(prepareMeasuresForRequest(measures, [PROFIT])).toEqual([ + { name: "revenue" }, + { + name: "profit", + expression: { expression: "revenue - cost", displayName: "Profit" }, + }, + ]); + }); + + it("still maps comparison suffixes for spec measures", () => { + const measures: V1MetricsViewAggregationMeasure[] = [ + { name: "revenue" }, + { name: "revenue__delta_abs" }, + { name: "revenue__delta_rel" }, + { name: "profit" }, + ]; + expect(prepareMeasuresForRequest(measures, [PROFIT])).toEqual([ + { name: "revenue" }, + { name: "revenue__delta_abs", comparisonDelta: { measure: "revenue" } }, + { name: "revenue__delta_rel", comparisonRatio: { measure: "revenue" } }, + { + name: "profit", + expression: { expression: "revenue - cost", displayName: "Profit" }, + }, + ]); + }); + + it("is a no-op without ephemeral measures", () => { + const measures: V1MetricsViewAggregationMeasure[] = [{ name: "revenue" }]; + expect(prepareMeasuresForRequest(measures, undefined)).toEqual(measures); + }); +}); + +describe("validateEphemeralMeasureName", () => { + const reserved = new Set(["revenue", "domain"]); + + it.each([ + ["profit", undefined], + ["Profit_2", undefined], + ["2profit", "must start with a letter"], + ["pro fit", "must start with a letter"], + ["profit__delta_abs", "comparison suffix"], + ["profit__delta_rel", "comparison suffix"], + ["profit_prev", "comparison suffix"], + ["profit_delta", "comparison suffix"], + ["profit_delta_perc", "comparison suffix"], + ["profit_percent_of_total", "comparison suffix"], + ["profit_rill_day", '"_rill_"'], + ["revenue", "already used"], + ])("%s", (name, expected) => { + const error = validateEphemeralMeasureName(name, reserved); + if (expected === undefined) { + expect(error).toBeUndefined(); + } else { + expect(error).toContain(expected); + } + }); +}); + +describe("validateEphemeralMeasureDef", () => { + const known = new Set(["revenue", "cost"]); + const reserved = new Set(["revenue", "cost", "domain", "timestamp"]); + + it("accepts a valid definition", () => { + expect( + validateEphemeralMeasureDef(PROFIT, known, reserved), + ).toBeUndefined(); + }); + + it("rejects unknown measure references", () => { + const def = { ...PROFIT, expression: "revenue - expenses" }; + expect(validateEphemeralMeasureDef(def, known, reserved)).toContain( + '"expenses" is not a measure', + ); + }); + + it("rejects references to other ephemeral measures", () => { + // "profit2" is not in the known measure set, so a second ephemeral + // measure cannot reference the first. + const def = { + name: "margin", + displayName: "Margin", + expression: "profit2 / revenue", + }; + expect(validateEphemeralMeasureDef(def, known, reserved)).toContain( + '"profit2" is not a measure', + ); + }); + + it("rejects invalid expressions", () => { + const def = { ...PROFIT, expression: "sum(revenue)" }; + expect(validateEphemeralMeasureDef(def, known, reserved)).toContain( + "unsupported function", + ); + }); + + it("rejects an empty display name", () => { + const def = { ...PROFIT, displayName: " " }; + expect(validateEphemeralMeasureDef(def, known, reserved)).toContain( + "display name is required", + ); + }); +}); + +describe("slugifyEphemeralMeasureName", () => { + const reserved = new Set(["profit", "revenue", "cost"]); + + it("derives a plain alias from the display name", () => { + expect(slugifyEphemeralMeasureName("Gross Margin", reserved)).toBe( + "gross_margin", + ); + }); + + it("disambiguates collisions with existing fields", () => { + expect(slugifyEphemeralMeasureName("Profit", reserved)).toBe("profit_2"); + }); + + it("always yields an alias that passes name validation", () => { + for (const label of [ + "Revenue Delta", + "Cost Prev", + "Revenue Percent Of Total", + "Total Rill Value", + "a__previous", + "123 abc", + "Profit", + ]) { + const slug = slugifyEphemeralMeasureName(label, reserved); + expect(validateEphemeralMeasureName(slug, reserved)).toBeUndefined(); + } + expect(slugifyEphemeralMeasureName("Revenue Delta", reserved)).toBe( + "revenue_delta_2", + ); + expect(slugifyEphemeralMeasureName("123 abc", reserved)).toBe("m_123_abc"); + }); +}); diff --git a/web-common/src/features/dashboards/ephemeral-measures/expression-parser.spec.ts b/web-common/src/features/dashboards/ephemeral-measures/expression-parser.spec.ts new file mode 100644 index 000000000000..f7e3adc9f101 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/expression-parser.spec.ts @@ -0,0 +1,193 @@ +import nearley from "nearley"; +import { describe, it, expect } from "vitest"; +import grammar from "./measure-expression.js"; +import { + EPHEMERAL_MEASURE_FUNCTIONS, + MAX_EPHEMERAL_EXPRESSION_LENGTH, + formatMeasureRef, + parseMeasureExpression, +} from "./expression-parser"; + +describe("parseMeasureExpression", () => { + describe("accepts", () => { + const cases: Array<[string, string[]]> = [ + ["revenue - cost", ["revenue", "cost"]], + ["revenue-cost", ["revenue", "cost"]], + ["(a + b) * 2", ["a", "b"]], + ["-a", ["a"]], + ["a % b", ["a", "b"]], + ["a * -1", ["a"]], + ["power(a, 2)", ["a"]], + ["coalesce(a, 0)", ["a"]], + ["coalesce(a, NULL)", ["a"]], + ["nullif(cost, 0)", ["cost"]], + ["round(a / b, 2)", ["a", "b"]], + [ + "abs(a) + floor(b) + ceil(c) + sqrt(d) + ln(e) + exp(f)", + ["a", "b", "c", "d", "e", "f"], + ], + ["greatest(a, b, c)", ["a", "b", "c"]], + ['"count" - a', ["count", "a"]], + ['"weird""name" + 1', ['weird"name']], + ["1e6 * a", ["a"]], + ["0.4 * revenue", ["revenue"]], + ["revenue - revenue", ["revenue"]], + ["((((a))))", ["a"]], + ['"null" + 1', ["null"]], + ["ROUND(a, 2)", ["a"]], + ['"rank" + a', ["rank", "a"]], + ["true + a", ["a"]], + ["a\r+\rb", ["a", "b"]], + ["1. * a", ["a"]], + [".5 * a", ["a"]], + ["1.e5 * a", ["a"]], + [" a ", ["a"]], + ]; + it.each(cases)("%s", (expr, refs) => { + const res = parseMeasureExpression(expr); + expect(res.error).toBeUndefined(); + expect(res.refs).toEqual(refs); + }); + }); + + describe("rejects", () => { + const cases: Array<[string, string]> = [ + ["", "empty"], + [" ", "empty"], + ["revenue); DROP TABLE x;--", "unexpected"], + ["a; SELECT 1", "unexpected"], + ["(select secret from t)", "reserved SQL word"], + ["sum(revenue)", 'unsupported function "sum"'], + ["count(a)", 'unsupported function "count"'], + ["t.revenue", "unexpected character"], + ["a > b", "unexpected"], + ["a and b", "reserved SQL word"], + ["'str'", "string literals"], + ["nullif(a, 'x')", "string literals"], + ["a || b", "unexpected"], + ["@@version", "unexpected character"], + ["foo(a)", 'unsupported function "foo"'], + ["round(a, 1, 2)", "does not accept 3 argument"], + ["coalesce(a)", "does not accept 1 argument"], + ["100", "must reference at least one measure"], + ["1 + 2", "must reference at least one measure"], + ["NULL", "must reference at least one measure"], + ["a as profit", "reserved SQL word"], + ["a /* comment */ + b", "unexpected"], + ['"unterminated', "unterminated quoted identifier"], + ["a +", "unexpected end of expression"], + ["(a", "expected closing parenthesis"], + ["revenue -- cost", "comments are not allowed"], + ["rank + 1", "reserved SQL word"], + ["if(a)", "reserved SQL word"], + ['"" + a', "empty quoted identifier"], + ["1e + a", 'invalid number "1e"'], + ["a * 2.5e+", 'invalid number "2.5e+"'], + ["(", "expected closing parenthesis"], + ["a)", 'unexpected ")"'], + ['"a" "b"', 'unexpected "b"'], + ["round()", "does not accept 0 argument"], + ["a.b", "unexpected character"], + ]; + it.each(cases)("%s", (expr, message) => { + const res = parseMeasureExpression(expr); + expect(res.error).toBeDefined(); + expect(res.error!.message).toContain(message); + }); + }); + + it("reports error positions", () => { + const cases: Array<[string, number]> = [ + ["revenue - 'oops'", 10], + ["a + select", 4], + ["a + b @ c", 6], + ["a + foo(b)", 4], + ["a + round(b, 1, 2)", 4], + ["(a + b", 6], + ["a +", 3], + ]; + for (const [expr, position] of cases) { + expect(parseMeasureExpression(expr).error?.position, expr).toBe(position); + } + }); + + // Lexical problems are found by a pre-scan of the whole input, so make sure + // they never mask a grammar error that appears earlier in the expression. + it("reports the first error in reading order", () => { + expect(parseMeasureExpression('coalesce true "x').error).toEqual({ + message: 'unexpected "true"', + position: 9, + }); + expect(parseMeasureExpression("'x' + select").error?.message).toContain( + "string literals", + ); + }); + + it("parses without ambiguity", () => { + const compiled = nearley.Grammar.fromCompiled(grammar); + for (const expr of [ + "revenue - cost", + "a - -b", + "round(a / b, 2) * -1.5e3 + (c % d)", + 'coalesce("a b", NULL, .5)', + "1.", + "greatest(a, b, c) - least(a, b)", + ]) { + const parser = new nearley.Parser(compiled); + parser.feed(expr); + expect(parser.results, expr).toHaveLength(1); + } + }); + + it("rejects deeply nested expressions", () => { + const expr = "(".repeat(100) + "a" + ")".repeat(100); + const res = parseMeasureExpression(expr); + expect(res.error).toBeDefined(); + expect(res.error!.message).toContain("deeply nested"); + }); + + // Mirrors the server parser (maxMeasureExpressionDepth = 32), which fails + // flat chains of more than 32 binary operators with "too deeply nested". + it("limits total expression complexity like the server", () => { + const ok = Array.from({ length: 33 }, (_, i) => `a${i}`).join("+"); + expect(parseMeasureExpression(ok).error).toBeUndefined(); + const tooLong = Array.from({ length: 34 }, (_, i) => `a${i}`).join("+"); + expect(parseMeasureExpression(tooLong).error?.message).toContain( + "deeply nested", + ); + }); + + it("quotes measure refs that need it", () => { + expect(formatMeasureRef("revenue")).toBe("revenue"); + expect(formatMeasureRef("rank")).toBe('"rank"'); + expect(formatMeasureRef("true")).toBe('"true"'); + expect(formatMeasureRef("total volume")).toBe('"total volume"'); + expect(formatMeasureRef('weird"name')).toBe('"weird""name"'); + }); + + it("rejects expressions over the maximum length", () => { + const expr = "a+".repeat(MAX_EPHEMERAL_EXPRESSION_LENGTH / 2) + "a"; + const res = parseMeasureExpression(expr); + expect(res.error).toBeDefined(); + expect(res.error!.message).toContain("maximum length"); + }); + + // The function allowlist must stay in sync with the server's + // measureExpressionFuncs in runtime/metricsview/measure_expression.go. + it("matches the server function allowlist", () => { + expect(Object.keys(EPHEMERAL_MEASURE_FUNCTIONS).sort()).toEqual([ + "abs", + "ceil", + "coalesce", + "exp", + "floor", + "greatest", + "least", + "ln", + "nullif", + "power", + "round", + "sqrt", + ]); + }); +}); diff --git a/web-common/src/features/dashboards/ephemeral-measures/expression-parser.ts b/web-common/src/features/dashboards/ephemeral-measures/expression-parser.ts new file mode 100644 index 000000000000..9681f1cd0365 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/expression-parser.ts @@ -0,0 +1,549 @@ +/** + * Client-side parser for ephemeral measure expressions. + * + * Mirrors the server-side grammar in `runtime/metricsview/measure_expression.go` + * (which is the authority): references to existing measure names, numeric + * literals, NULL, unary minus, the binary operators + - * / %, parentheses, + * and an allowlist of scalar functions. Everything else is rejected. + * + * The syntax lives in the nearley grammar `measure-expression.ne` (compiled to + * `measure-expression.js`, see the npm script `build-measure-expression-grammar`). + * This module wraps it with the checks that a grammar cannot express well: + * reserved words, string literals and comments (reported with positions before + * parsing), and the function allowlist, arities and nesting limit (checked on + * the parsed tree, mirroring the server's walk). + * + * The client parse exists for inline validation (position-aware errors), + * autocomplete (extracting referenced names) and URL param validation. + * One deliberate difference: SQL comments are rejected here, while the server + * strips them; the server never echoes input, so this is only stricter. + */ +import nearley from "nearley"; +import grammar from "./measure-expression.js"; + +// Keep in sync with `measureExpressionFuncs` in runtime/metricsview/measure_expression.go. +// maxArgs of -1 means unbounded. +export const EPHEMERAL_MEASURE_FUNCTIONS: Record< + string, + { minArgs: number; maxArgs: number } +> = { + abs: { minArgs: 1, maxArgs: 1 }, + round: { minArgs: 1, maxArgs: 2 }, + floor: { minArgs: 1, maxArgs: 1 }, + ceil: { minArgs: 1, maxArgs: 1 }, + sqrt: { minArgs: 1, maxArgs: 1 }, + ln: { minArgs: 1, maxArgs: 1 }, + exp: { minArgs: 1, maxArgs: 1 }, + power: { minArgs: 2, maxArgs: 2 }, + coalesce: { minArgs: 2, maxArgs: -1 }, + nullif: { minArgs: 2, maxArgs: 2 }, + greatest: { minArgs: 2, maxArgs: -1 }, + least: { minArgs: 2, maxArgs: -1 }, +}; + +// Keep in sync with the limits in runtime/metricsview/measure_expression.go. +export const MAX_EPHEMERAL_EXPRESSION_LENGTH = 1024; +const MAX_DEPTH = 32; + +// SQL words the server-side parser treats as reserved: a measure with one of +// these names must be double-quoted in expressions. Generated by probing the +// server parser (see runtime/metricsview/measure_expression.go). +export const RESERVED_SQL_WORDS = new Set([ + "add", + "all", + "alter", + "analyze", + "and", + "array", + "as", + "asc", + "between", + "bigint", + "binary", + "blob", + "both", + "by", + "call", + "cascade", + "case", + "change", + "char", + "character", + "check", + "collate", + "column", + "constraint", + "continue", + "convert", + "create", + "cross", + "cume_dist", + "cursor", + "database", + "databases", + "day_hour", + "day_microsecond", + "day_minute", + "day_second", + "dec", + "decimal", + "default", + "delayed", + "delete", + "dense_rank", + "desc", + "describe", + "distinct", + "distinctrow", + "div", + "double", + "drop", + "dual", + "else", + "elseif", + "enclosed", + "escaped", + "except", + "exists", + "exit", + "explain", + "fetch", + "first_value", + "float", + "float4", + "float8", + "for", + "force", + "foreign", + "from", + "fulltext", + "generated", + "grant", + "group", + "groups", + "having", + "high_priority", + "hour_microsecond", + "hour_minute", + "hour_second", + "if", + "ignore", + "ilike", + "in", + "index", + "infile", + "inner", + "inout", + "insert", + "int", + "int1", + "int2", + "int3", + "int4", + "int8", + "integer", + "intersect", + "interval", + "into", + "is", + "iterate", + "join", + "key", + "keys", + "kill", + "lag", + "last_value", + "lead", + "leading", + "leave", + "left", + "like", + "limit", + "linear", + "lines", + "load", + "lock", + "long", + "longblob", + "longtext", + "low_priority", + "match", + "maxvalue", + "mediumblob", + "mediumint", + "mediumtext", + "middleint", + "minute_microsecond", + "minute_second", + "mod", + "natural", + "no_write_to_binlog", + "not", + "nth_value", + "ntile", + "numeric", + "of", + "on", + "optimize", + "option", + "optionally", + "or", + "order", + "out", + "outer", + "outfile", + "over", + "partition", + "percent_rank", + "precision", + "primary", + "procedure", + "range", + "rank", + "read", + "real", + "recursive", + "references", + "regexp", + "release", + "rename", + "repeat", + "replace", + "require", + "restrict", + "revoke", + "right", + "rlike", + "row", + "row_number", + "rows", + "schema", + "schemas", + "second_microsecond", + "select", + "set", + "show", + "smallint", + "spatial", + "sql", + "sql_big_result", + "sql_calc_found_rows", + "sql_small_result", + "sqlexception", + "sqlstate", + "sqlwarning", + "ssl", + "starting", + "stored", + "straight_join", + "table", + "tablesample", + "terminated", + "then", + "tinyblob", + "tinyint", + "tinytext", + "to", + "trailing", + "trigger", + "union", + "unique", + "unlock", + "unsigned", + "update", + "usage", + "use", + "using", + "values", + "varbinary", + "varchar", + "varcharacter", + "varying", + "virtual", + "when", + "where", + "while", + "window", + "with", + "write", + "xor", + "year_month", + "zerofill", +]); + +// Bare true/false parse as numeric literals server-side, and NULL as the null +// literal; measures with these names must also be quoted. +const LITERAL_WORDS = new Set(["true", "false", "null"]); + +/** + * Formats a measure name for insertion into an expression, double-quoting it + * when a bare reference would not survive the server-side SQL parser. + */ +export function formatMeasureRef(name: string): string { + const lower = name.toLowerCase(); + if ( + /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) && + !RESERVED_SQL_WORDS.has(lower) && + !LITERAL_WORDS.has(lower) + ) { + return name; + } + return `"${name.replace(/"/g, '""')}"`; +} + +export type MeasureExpressionError = { + message: string; + // 0-based character offset into the expression where the error was detected. + position: number; +}; + +export type MeasureExpressionParseResult = { + // Referenced identifiers, deduplicated in order of first appearance. + refs: string[]; + error?: MeasureExpressionError; +}; + +// Parse tree produced by the grammar's postprocessors. +// `pos` is the 0-based offset of the node's first character. +type Node = + | { type: "literal"; literal: string; pos: number } + | { type: "ref"; name: string; pos: number } + // A bare identifier: a measure reference unless it spells a literal. + | { type: "word"; name: string; pos: number } + | { type: "func"; name: string; args: Node[]; pos: number } + | { type: "paren"; expr: Node; pos: number } + | { type: "unary"; expr: Node; pos: number } + | { type: "binary"; op: string; left: Node; right: Node; pos: number }; + +class ParseError extends Error { + position: number; + constructor(message: string, position: number) { + super(message); + this.position = position; + } +} + +const compiledGrammar = nearley.Grammar.fromCompiled(grammar); + +/** + * Parses and validates an ephemeral measure expression. + * Returns the referenced identifiers, or a position-aware error. + * Note this validates the grammar only; callers must separately check that + * each ref is an existing measure name. + */ +export function parseMeasureExpression( + expression: string, +): MeasureExpressionParseResult { + if (expression.trim() === "") { + return { refs: [], error: { message: "expression is empty", position: 0 } }; + } + if (expression.length > MAX_EPHEMERAL_EXPRESSION_LENGTH) { + return { + refs: [], + error: { + message: `expression exceeds the maximum length of ${MAX_EPHEMERAL_EXPRESSION_LENGTH} characters`, + position: MAX_EPHEMERAL_EXPRESSION_LENGTH, + }, + }; + } + try { + const refs: string[] = []; + walk(parse(expression), 0, refs); + if (refs.length === 0) { + return { + refs: [], + error: { + message: "expression must reference at least one measure", + position: 0, + }, + }; + } + return { refs }; + } catch (e) { + if (e instanceof ParseError) { + return { refs: [], error: { message: e.message, position: e.position } }; + } + throw e; + } +} + +/** + * Runs the grammar and turns its failures into position-aware errors. + * Lexical problems the grammar only reports as a generic syntax error (reserved + * words, string literals, comments, malformed quoted identifiers) are found by a + * pre-scan; whichever error comes first in the input wins, so errors are always + * reported left to right. + */ +function parse(expression: string): Node { + const lexical = scanLexicalErrors(expression); + const parser = new nearley.Parser(compiledGrammar); + try { + parser.feed(expression); + } catch (e) { + const offset = (e as { offset?: number }).offset ?? 0; + if (lexical && lexical.position <= offset) throw lexical; + throw syntaxError(expression, offset); + } + if (lexical) throw lexical; + const root = parser.results[0] as Node | undefined; + if (!root) { + // The input is a valid prefix of an expression but ends too early. + const partial = partialNumberError(expression, expression.length); + if (partial) throw partial; + const opens = (expression.match(/\(/g) ?? []).length; + const closes = (expression.match(/\)/g) ?? []).length; + throw new ParseError( + opens > closes + ? "expected closing parenthesis" + : "unexpected end of expression", + expression.length, + ); + } + return root; +} + +/** + * Scans the raw input for constructs the server rejects or interprets + * differently, which the grammar cannot describe with a useful message. + * Returns the first one found, if any. + */ +function scanLexicalErrors(input: string): ParseError | undefined { + let i = 0; + while (i < input.length) { + const ch = input[i]; + if (ch === '"') { + const start = i; + let length = 0; + i++; + for (;;) { + if (i >= input.length) { + return new ParseError("unterminated quoted identifier", start); + } + if (input[i] === '"') { + if (input[i + 1] !== '"') break; + i++; + } + length++; + i++; + } + i++; + if (length === 0) return new ParseError("empty quoted identifier", start); + continue; + } + if (ch === "'") { + return new ParseError( + "string literals are not allowed in the expression", + i, + ); + } + if (ch === "-" && input[i + 1] === "-") { + // The server-side SQL parser treats `--` as a comment start. + return new ParseError( + `comments are not allowed in the expression (found "--"); use parentheses for double negation, e.g. "a - (-b)"`, + i, + ); + } + if (/[A-Za-z_]/.test(ch)) { + const start = i; + while (i < input.length && /[A-Za-z0-9_]/.test(input[i])) i++; + const word = input.slice(start, i); + if (RESERVED_SQL_WORDS.has(word.toLowerCase())) { + return new ParseError( + `"${word}" is a reserved SQL word; wrap it in double quotes to reference a measure with this name`, + start, + ); + } + continue; + } + i++; + } + return undefined; +} + +/** + * Describes the token at `offset` that the grammar could not accept. + */ +function syntaxError(input: string, offset: number): ParseError { + const partial = partialNumberError(input, offset); + if (partial) return partial; + if (offset >= input.length) { + return new ParseError("unexpected end of expression", offset); + } + const rest = input.slice(offset); + const token = + /^[A-Za-z_][A-Za-z0-9_]*/.exec(rest)?.[0] ?? + /^(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?/.exec(rest)?.[0] ?? + /^"(?:[^"]|"")*"/.exec(rest)?.[0].slice(1, -1).replace(/""/g, '"') ?? + (/^[-+*/%(),]/.test(rest) ? rest[0] : undefined); + if (token === undefined) { + return new ParseError(`unexpected character "${rest[0]}"`, offset); + } + return new ParseError(`unexpected "${token}"`, offset); +} + +/** + * The grammar consumes the "e" (and sign) of an exponent before knowing whether + * digits follow, so a failure right after one is really a malformed number, + * e.g. "1e" or "2.5e+"; report it at the start of the number. + */ +function partialNumberError( + input: string, + offset: number, +): ParseError | undefined { + const match = /(?:\d+\.?\d*|\.\d+)[eE][+-]?$/.exec(input.slice(0, offset)); + if (!match) return undefined; + return new ParseError(`invalid number "${match[0]}"`, match.index); +} + +/** + * Validates the parsed tree and collects referenced measure names, in order of + * first appearance. Mirrors the server's `parseNode`: every node, including + * parentheses, counts one level toward the nesting limit, so a flat chain of + * binary operators is bounded as well as deep nesting. + */ +function walk(node: Node, depth: number, refs: string[]): void { + if (depth > MAX_DEPTH) { + throw new ParseError("expression is too deeply nested", node.pos); + } + switch (node.type) { + case "literal": + return; + case "ref": + addRef(refs, node.name); + return; + case "word": + // Bare true/false/null parse as literals server-side. + if (!LITERAL_WORDS.has(node.name.toLowerCase())) addRef(refs, node.name); + return; + case "func": { + const spec = EPHEMERAL_MEASURE_FUNCTIONS[node.name.toLowerCase()]; + if (!spec) { + throw new ParseError(`unsupported function "${node.name}"`, node.pos); + } + const argCount = node.args.length; + if ( + argCount < spec.minArgs || + (spec.maxArgs >= 0 && argCount > spec.maxArgs) + ) { + throw new ParseError( + `function "${node.name}" does not accept ${argCount} argument(s)`, + node.pos, + ); + } + for (const arg of node.args) walk(arg, depth + 1, refs); + return; + } + case "paren": + case "unary": + walk(node.expr, depth + 1, refs); + return; + case "binary": + walk(node.left, depth + 1, refs); + walk(node.right, depth + 1, refs); + return; + } +} + +function addRef(refs: string[], name: string): void { + if (!refs.includes(name)) refs.push(name); +} diff --git a/web-common/src/features/dashboards/ephemeral-measures/format-presets.ts b/web-common/src/features/dashboards/ephemeral-measures/format-presets.ts new file mode 100644 index 000000000000..85e699853ff4 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/format-presets.ts @@ -0,0 +1,30 @@ +import { m } from "@rilldata/web-common/lib/i18n/gen/messages"; +import { FormatPreset } from "@rilldata/web-common/lib/number-formatting/humanizer-types"; + +/** + * Format presets offered when defining an ephemeral measure. + */ +export function ephemeralFormatPresetOptions(): { + value: string; + label: string; +}[] { + return [ + { + value: FormatPreset.HUMANIZE, + label: m.dashboard_pivot_ephemeral_format_humanize(), + }, + { value: FormatPreset.NONE, label: m.common_none() }, + { + value: FormatPreset.CURRENCY_USD, + label: m.dashboard_pivot_ephemeral_format_currency_usd(), + }, + { + value: FormatPreset.CURRENCY_EUR, + label: m.dashboard_pivot_ephemeral_format_currency_eur(), + }, + { + value: FormatPreset.PERCENTAGE, + label: m.dashboard_pivot_ephemeral_format_percentage(), + }, + ]; +} diff --git a/web-common/src/features/dashboards/ephemeral-measures/library.spec.ts b/web-common/src/features/dashboards/ephemeral-measures/library.spec.ts new file mode 100644 index 000000000000..b0a3b159ad09 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/library.spec.ts @@ -0,0 +1,104 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + loadEphemeralMeasureLibrary, + mergeEphemeralMeasureDefs, + saveEphemeralMeasureLibrary, + syncEphemeralMeasureLibrary, + upsertIntoEphemeralMeasureLibrary, +} from "./library"; +import type { EphemeralMeasureDef } from "./types"; + +const profit: EphemeralMeasureDef = { + name: "profit", + displayName: "Profit", + expression: "revenue - cost", +}; +const arpu: EphemeralMeasureDef = { + name: "arpu", + displayName: "ARPU", + expression: "revenue / users", + formatPreset: "currency_usd", +}; + +describe("ephemeral measure library", () => { + beforeEach(() => { + localStorage.clear(); + }); + + it("round-trips definitions per metrics view and namespace", () => { + saveEphemeralMeasureLibrary("mv", "org__proj__", [profit, arpu]); + expect(loadEphemeralMeasureLibrary("mv", "org__proj__")).toEqual([ + profit, + arpu, + ]); + expect(loadEphemeralMeasureLibrary("mv", undefined)).toEqual([]); + expect(loadEphemeralMeasureLibrary("other", "org__proj__")).toEqual([]); + }); + + it("ignores malformed stored values", () => { + localStorage.setItem("rill:app:adhoc-measures:mv", "not json"); + expect(loadEphemeralMeasureLibrary("mv", undefined)).toEqual([]); + + localStorage.setItem( + "rill:app:adhoc-measures:mv", + JSON.stringify([profit, { name: "broken" }, "x"]), + ); + expect(loadEphemeralMeasureLibrary("mv", undefined)).toEqual([profit]); + }); + + it("upsert adds and replaces without removing", () => { + saveEphemeralMeasureLibrary("mv", undefined, [profit]); + const editedProfit = { ...profit, displayName: "Net profit" }; + upsertIntoEphemeralMeasureLibrary("mv", undefined, [editedProfit, arpu]); + expect(loadEphemeralMeasureLibrary("mv", undefined)).toEqual([ + editedProfit, + arpu, + ]); + + // A load that lacks a definition must not delete it. + upsertIntoEphemeralMeasureLibrary("mv", undefined, [arpu]); + expect(loadEphemeralMeasureLibrary("mv", undefined)).toEqual([ + editedProfit, + arpu, + ]); + upsertIntoEphemeralMeasureLibrary("mv", undefined, undefined); + expect(loadEphemeralMeasureLibrary("mv", undefined)).toEqual([ + editedProfit, + arpu, + ]); + }); + + it("sync removes definitions dropped since the previous state", () => { + saveEphemeralMeasureLibrary("mv", undefined, [profit, arpu]); + // profit deleted by the user, arpu edited. + const editedArpu = { ...arpu, expression: "revenue / active_users" }; + syncEphemeralMeasureLibrary("mv", undefined, [profit, arpu], [editedArpu]); + expect(loadEphemeralMeasureLibrary("mv", undefined)).toEqual([editedArpu]); + + // Deleting the last definition clears the entry. + syncEphemeralMeasureLibrary("mv", undefined, [editedArpu], undefined); + expect(loadEphemeralMeasureLibrary("mv", undefined)).toEqual([]); + expect(localStorage.getItem("rill:app:adhoc-measures:mv")).toBeNull(); + }); + + it("sync keeps definitions from other explores on the same metrics view", () => { + // arpu was created in another explore and is not in this state at all. + saveEphemeralMeasureLibrary("mv", undefined, [arpu]); + syncEphemeralMeasureLibrary("mv", undefined, undefined, [profit]); + expect(loadEphemeralMeasureLibrary("mv", undefined)).toEqual([ + arpu, + profit, + ]); + }); + + it("merge unions by name with base order and overrides", () => { + const editedProfit = { ...profit, displayName: "Net profit" }; + expect(mergeEphemeralMeasureDefs([profit], [arpu, profit])).toEqual([ + profit, + arpu, + ]); + expect(mergeEphemeralMeasureDefs([profit], [arpu], [editedProfit])).toEqual( + [editedProfit, arpu], + ); + }); +}); diff --git a/web-common/src/features/dashboards/ephemeral-measures/library.ts b/web-common/src/features/dashboards/ephemeral-measures/library.ts new file mode 100644 index 000000000000..8ebe68d5b0e3 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/library.ts @@ -0,0 +1,142 @@ +import type { EphemeralMeasureDef } from "./types"; + +/** + * The ad-hoc measure library persists every definition a user creates for a + * metrics view in localStorage, keyed by the metrics view rather than the + * explore. Any explore on the same metrics view restores the library on load, + * so a definition survives without a bookmark and is not lost when it is + * hidden: the URL only carries definitions the explore state references (see + * `referencedEphemeralMeasures`), which keeps shared links short. + * + * Deletions must be explicit (`syncEphemeralMeasureLibrary` with the previous + * definitions); loading never removes entries, since an explore may expose a + * subset of the metrics view's measures and drop definitions it cannot use. + */ + +function getKeyForLibrary( + metricsViewName: string, + storageNamespacePrefix: string | undefined, +) { + return `rill:app:adhoc-measures:${storageNamespacePrefix ?? ""}${metricsViewName}`.toLowerCase(); +} + +export function loadEphemeralMeasureLibrary( + metricsViewName: string, + storageNamespacePrefix: string | undefined, +): EphemeralMeasureDef[] { + try { + const raw = localStorage.getItem( + getKeyForLibrary(metricsViewName, storageNamespacePrefix), + ); + if (!raw) return []; + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) return []; + return parsed.filter(isEphemeralMeasureDef); + } catch { + return []; + } +} + +export function saveEphemeralMeasureLibrary( + metricsViewName: string, + storageNamespacePrefix: string | undefined, + defs: EphemeralMeasureDef[], +) { + try { + const key = getKeyForLibrary(metricsViewName, storageNamespacePrefix); + if (!defs.length) { + localStorage.removeItem(key); + return; + } + localStorage.setItem(key, JSON.stringify(defs)); + } catch { + // no-op: storage may be unavailable (private mode, embeds, quota) + } +} + +/** + * Upserts the given definitions into the library without removing anything. + * Used on load, where a missing definition does not mean it was deleted. + */ +export function upsertIntoEphemeralMeasureLibrary( + metricsViewName: string, + storageNamespacePrefix: string | undefined, + defs: EphemeralMeasureDef[] | undefined, +) { + if (!defs?.length) return; + const library = loadEphemeralMeasureLibrary( + metricsViewName, + storageNamespacePrefix, + ); + saveEphemeralMeasureLibrary( + metricsViewName, + storageNamespacePrefix, + mergeEphemeralMeasureDefs(library, defs, defs), + ); +} + +/** + * Applies a state transition to the library: definitions in `next` are + * upserted, and definitions present in `previous` but absent from `next` + * were deleted by the user and are removed. + */ +export function syncEphemeralMeasureLibrary( + metricsViewName: string, + storageNamespacePrefix: string | undefined, + previous: EphemeralMeasureDef[] | undefined, + next: EphemeralMeasureDef[] | undefined, +) { + const nextNames = new Set(next?.map((def) => def.name) ?? []); + const removed = (previous ?? []).filter((def) => !nextNames.has(def.name)); + if (!removed.length && !next?.length) return; + + const library = loadEphemeralMeasureLibrary( + metricsViewName, + storageNamespacePrefix, + ); + const removedNames = new Set(removed.map((def) => def.name)); + saveEphemeralMeasureLibrary( + metricsViewName, + storageNamespacePrefix, + mergeEphemeralMeasureDefs( + library.filter((def) => !removedNames.has(def.name)), + next ?? [], + next ?? [], + ), + ); +} + +/** + * Unions two definition lists by name, keeping `base` order and appending + * unseen entries from `extra`. Entries from `override` replace same-named + * entries from `base` (used so an edit wins over the stored copy). + */ +export function mergeEphemeralMeasureDefs( + base: EphemeralMeasureDef[], + extra: EphemeralMeasureDef[], + override: EphemeralMeasureDef[] = [], +): EphemeralMeasureDef[] { + const overrides = new Map(override.map((def) => [def.name, def])); + const seen = new Set(); + const merged: EphemeralMeasureDef[] = []; + for (const def of [...base, ...extra]) { + if (seen.has(def.name)) continue; + seen.add(def.name); + merged.push(overrides.get(def.name) ?? def); + } + return merged; +} + +function isEphemeralMeasureDef(value: unknown): value is EphemeralMeasureDef { + if (!value || typeof value !== "object") return false; + const def = value as Record; + return ( + typeof def.name === "string" && + !!def.name && + typeof def.displayName === "string" && + !!def.displayName && + typeof def.expression === "string" && + !!def.expression && + (def.formatPreset === undefined || typeof def.formatPreset === "string") + ); +} diff --git a/web-common/src/features/dashboards/ephemeral-measures/measure-expression.js b/web-common/src/features/dashboards/ephemeral-measures/measure-expression.js new file mode 100644 index 000000000000..249ba048b5b8 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/measure-expression.js @@ -0,0 +1,233 @@ +// Generated automatically by nearley, version 2.20.1 +// http://github.com/Hardmath123/nearley +function id(x) { + return x[0]; +} +let Lexer = undefined; +let ParserRules = [ + { + name: "main", + symbols: ["_", "additive", "_"], + postprocess: ([, expr]) => expr, + }, + { + name: "additive", + symbols: ["additive", "_", /[+\-]/, "_", "multiplicative"], + postprocess: ([left, , op, , right], pos) => ({ + type: "binary", + op, + left, + right, + pos, + }), + }, + { name: "additive", symbols: ["multiplicative"], postprocess: id }, + { + name: "multiplicative", + symbols: ["multiplicative", "_", /[*/%]/, "_", "unary"], + postprocess: ([left, , op, , right], pos) => ({ + type: "binary", + op, + left, + right, + pos, + }), + }, + { name: "multiplicative", symbols: ["unary"], postprocess: id }, + { + name: "unary", + symbols: [{ literal: "-" }, "_", "unary"], + postprocess: ([, , expr], pos) => ({ type: "unary", expr, pos }), + }, + { name: "unary", symbols: ["primary"], postprocess: id }, + { + name: "primary", + symbols: ["number"], + postprocess: ([literal], pos) => ({ type: "literal", literal, pos }), + }, + { + name: "primary", + symbols: ["quoted_ident"], + postprocess: ([name], pos) => ({ type: "ref", name, pos }), + }, + { name: "primary$ebnf$1", symbols: ["args"], postprocess: id }, + { + name: "primary$ebnf$1", + symbols: [], + postprocess: function (d) { + return null; + }, + }, + { + name: "primary", + symbols: [ + "ident", + "_", + { literal: "(" }, + "_", + "primary$ebnf$1", + "_", + { literal: ")" }, + ], + postprocess: ([name, , , , args], pos) => ({ + type: "func", + name, + args: args ?? [], + pos, + }), + }, + { + name: "primary", + symbols: ["ident"], + postprocess: ([name], pos) => ({ type: "word", name, pos }), + }, + { + name: "primary", + symbols: [{ literal: "(" }, "_", "additive", "_", { literal: ")" }], + postprocess: ([, , expr], pos) => ({ type: "paren", expr, pos }), + }, + { name: "args$ebnf$1", symbols: [] }, + { + name: "args$ebnf$1$subexpression$1", + symbols: ["_", { literal: "," }, "_", "additive"], + }, + { + name: "args$ebnf$1", + symbols: ["args$ebnf$1", "args$ebnf$1$subexpression$1"], + postprocess: function arrpush(d) { + return d[0].concat([d[1]]); + }, + }, + { + name: "args", + symbols: ["additive", "args$ebnf$1"], + postprocess: ([first, rest]) => [first, ...rest.map(([, , , arg]) => arg)], + }, + { name: "ident$ebnf$1", symbols: [] }, + { + name: "ident$ebnf$1", + symbols: ["ident$ebnf$1", /[A-Za-z0-9_]/], + postprocess: function arrpush(d) { + return d[0].concat([d[1]]); + }, + }, + { + name: "ident", + symbols: [/[A-Za-z_]/, "ident$ebnf$1"], + postprocess: ([first, rest]) => first + rest.join(""), + }, + { name: "quoted_ident$ebnf$1", symbols: ["qchar"] }, + { + name: "quoted_ident$ebnf$1", + symbols: ["quoted_ident$ebnf$1", "qchar"], + postprocess: function arrpush(d) { + return d[0].concat([d[1]]); + }, + }, + { + name: "quoted_ident", + symbols: [{ literal: '"' }, "quoted_ident$ebnf$1", { literal: '"' }], + postprocess: ([, chars]) => chars.join(""), + }, + { name: "qchar", symbols: [/[^"]/], postprocess: id }, + { + name: "qchar$string$1", + symbols: [{ literal: '"' }, { literal: '"' }], + postprocess: function joiner(d) { + return d.join(""); + }, + }, + { name: "qchar", symbols: ["qchar$string$1"], postprocess: () => '"' }, + { + name: "number$ebnf$1$subexpression$1$ebnf$1", + symbols: ["digits"], + postprocess: id, + }, + { + name: "number$ebnf$1$subexpression$1$ebnf$1", + symbols: [], + postprocess: function (d) { + return null; + }, + }, + { + name: "number$ebnf$1$subexpression$1", + symbols: [{ literal: "." }, "number$ebnf$1$subexpression$1$ebnf$1"], + }, + { + name: "number$ebnf$1", + symbols: ["number$ebnf$1$subexpression$1"], + postprocess: id, + }, + { + name: "number$ebnf$1", + symbols: [], + postprocess: function (d) { + return null; + }, + }, + { name: "number$ebnf$2", symbols: ["exponent"], postprocess: id }, + { + name: "number$ebnf$2", + symbols: [], + postprocess: function (d) { + return null; + }, + }, + { + name: "number", + symbols: ["digits", "number$ebnf$1", "number$ebnf$2"], + postprocess: ([int, frac, exp]) => + int + (frac ? "." + (frac[1] ?? "") : "") + (exp ?? ""), + }, + { name: "number$ebnf$3", symbols: ["exponent"], postprocess: id }, + { + name: "number$ebnf$3", + symbols: [], + postprocess: function (d) { + return null; + }, + }, + { + name: "number", + symbols: [{ literal: "." }, "digits", "number$ebnf$3"], + postprocess: ([, frac, exp]) => "." + frac + (exp ?? ""), + }, + { name: "digits$ebnf$1", symbols: [/[0-9]/] }, + { + name: "digits$ebnf$1", + symbols: ["digits$ebnf$1", /[0-9]/], + postprocess: function arrpush(d) { + return d[0].concat([d[1]]); + }, + }, + { + name: "digits", + symbols: ["digits$ebnf$1"], + postprocess: ([d]) => d.join(""), + }, + { name: "exponent$ebnf$1", symbols: [/[+\-]/], postprocess: id }, + { + name: "exponent$ebnf$1", + symbols: [], + postprocess: function (d) { + return null; + }, + }, + { + name: "exponent", + symbols: [/[eE]/, "exponent$ebnf$1", "digits"], + postprocess: ([e, sign, d]) => e + (sign ?? "") + d, + }, + { name: "_$ebnf$1", symbols: [] }, + { + name: "_$ebnf$1", + symbols: ["_$ebnf$1", /[ \t\n\r]/], + postprocess: function arrpush(d) { + return d[0].concat([d[1]]); + }, + }, + { name: "_", symbols: ["_$ebnf$1"], postprocess: () => null }, +]; +let ParserStart = "main"; +export default { Lexer, ParserRules, ParserStart }; diff --git a/web-common/src/features/dashboards/ephemeral-measures/measure-expression.ne b/web-common/src/features/dashboards/ephemeral-measures/measure-expression.ne new file mode 100644 index 000000000000..eb0b4ad2a3d7 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/measure-expression.ne @@ -0,0 +1,45 @@ +# Grammar for ephemeral measure expressions. +# +# Mirrors the server-side grammar in runtime/metricsview/measure_expression.go, +# which is the authority: measure references (bare or double-quoted +# identifiers), numeric literals, NULL, unary minus, the binary operators +# + - * / %, parentheses and function calls. Function names, arities, reserved +# words and nesting limits are validated after parsing in expression-parser.ts. +# +# Regenerate the compiled grammar with: +# npm run build-measure-expression-grammar -w web-common + +@preprocessor esmodule + +main -> _ additive _ {% ([, expr]) => expr %} + +additive -> additive _ [+\-] _ multiplicative {% ([left, , op, , right], pos) => ({ type: "binary", op, left, right, pos }) %} + | multiplicative {% id %} + +multiplicative -> multiplicative _ [*/%] _ unary {% ([left, , op, , right], pos) => ({ type: "binary", op, left, right, pos }) %} + | unary {% id %} + +unary -> "-" _ unary {% ([, , expr], pos) => ({ type: "unary", expr, pos }) %} + | primary {% id %} + +primary -> number {% ([literal], pos) => ({ type: "literal", literal, pos }) %} + | quoted_ident {% ([name], pos) => ({ type: "ref", name, pos }) %} + | ident _ "(" _ args:? _ ")" {% ([name, , , , args], pos) => ({ type: "func", name, args: args ?? [], pos }) %} + | ident {% ([name], pos) => ({ type: "word", name, pos }) %} + | "(" _ additive _ ")" {% ([, , expr], pos) => ({ type: "paren", expr, pos }) %} + +args -> additive (_ "," _ additive):* {% ([first, rest]) => [first, ...rest.map(([, , , arg]) => arg)] %} + +ident -> [A-Za-z_] [A-Za-z0-9_]:* {% ([first, rest]) => first + rest.join("") %} + +# Double-quoted identifier; "" escapes a quote, matching ANSI SQL. +quoted_ident -> "\"" qchar:+ "\"" {% ([, chars]) => chars.join("") %} +qchar -> [^"] {% id %} + | "\"\"" {% () => '"' %} + +number -> digits ("." digits:?):? exponent:? {% ([int, frac, exp]) => int + (frac ? "." + (frac[1] ?? "") : "") + (exp ?? "") %} + | "." digits exponent:? {% ([, frac, exp]) => "." + frac + (exp ?? "") %} +digits -> [0-9]:+ {% ([d]) => d.join("") %} +exponent -> [eE] [+\-]:? digits {% ([e, sign, d]) => e + (sign ?? "") + d %} + +_ -> [ \t\n\r]:* {% () => null %} diff --git a/web-common/src/features/dashboards/ephemeral-measures/measure-mapping.ts b/web-common/src/features/dashboards/ephemeral-measures/measure-mapping.ts new file mode 100644 index 000000000000..88221a844589 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/measure-mapping.ts @@ -0,0 +1,137 @@ +import type { + MetricsViewSpecMeasure, + V1MetricsViewAggregationMeasure, +} from "@rilldata/web-common/runtime-client"; +import { FormatPreset } from "@rilldata/web-common/lib/number-formatting/humanizer-types"; +import type { EphemeralMeasureDef } from "./types"; + +/** + * Builds a synthetic metrics view spec measure for an ephemeral measure so UI + * code (labels, formatters, tooltips, selectors) can treat it like any other + * measure. + */ +export function ephemeralMeasureToSpecMeasure( + def: EphemeralMeasureDef, +): MetricsViewSpecMeasure { + return { + name: def.name, + displayName: def.displayName, + expression: def.expression, + // Surfaces the calculation in description-driven tooltips. + description: def.expression, + formatPreset: def.formatPreset ?? (FormatPreset.HUMANIZE as string), + }; +} + +/** + * Appends synthetic spec measures for the given ephemeral measure + * definitions to a list of spec measures. + */ +export function appendEphemeralSpecMeasures( + measures: MetricsViewSpecMeasure[], + defs: EphemeralMeasureDef[] | undefined, +): MetricsViewSpecMeasure[] { + if (!defs?.length) return measures; + return [...measures, ...defs.map(ephemeralMeasureToSpecMeasure)]; +} + +/** + * Attaches the `expression` compute to request measures whose name matches a + * ephemeral measure definition. Leaves other measures untouched. + */ +export function mapEphemeralMeasuresForRequest( + measures: V1MetricsViewAggregationMeasure[], + defs: EphemeralMeasureDef[] | undefined, +): V1MetricsViewAggregationMeasure[] { + if (!defs?.length) return measures; + const byName = new Map(defs.map((def) => [def.name, def])); + return measures.map((measure) => { + const def = measure.name ? byName.get(measure.name) : undefined; + // Never attach an expression to a measure that already carries a compute + // (e.g. a comparison accessor whose alias happens to match a definition). + const hasCompute = + measure.expression || + measure.count || + measure.countDistinct || + measure.comparisonValue || + measure.comparisonDelta || + measure.comparisonRatio || + measure.percentOfTotal || + measure.uri || + measure.comparisonTime; + if (!def || hasCompute) return measure; + return { + ...measure, + expression: { + expression: def.expression, + displayName: def.displayName, + }, + }; + }); +} + +/** + * Splits measure names for the time series API: names of spec measures go in + * `measureNames`, ephemeral measures go in `measures` with their expression. + */ +export function splitTimeSeriesMeasures( + names: string[], + defs: EphemeralMeasureDef[] | undefined, +): { + measureNames: string[]; + ephemeralMeasures: V1MetricsViewAggregationMeasure[] | undefined; +} { + if (!defs?.length) + return { measureNames: names, ephemeralMeasures: undefined }; + const byName = new Map(defs.map((def) => [def.name, def])); + const measureNames: string[] = []; + const measures: V1MetricsViewAggregationMeasure[] = []; + for (const name of names) { + const def = byName.get(name); + if (def) { + measures.push({ + name: def.name, + expression: { + expression: def.expression, + displayName: def.displayName, + }, + }); + } else { + measureNames.push(name); + } + } + return { + measureNames, + ephemeralMeasures: measures.length ? measures : undefined, + }; +} + +/** + * Recovers ephemeral measure definitions from request measures that carry an + * `expression` compute, e.g. when editing a saved report outside an explore. + */ +export function ephemeralDefsFromRequestMeasures( + measures: V1MetricsViewAggregationMeasure[] | undefined, +): EphemeralMeasureDef[] | undefined { + const defs = (measures ?? []).flatMap((measure) => + measure.name && measure.expression?.expression + ? [ + { + name: measure.name, + displayName: measure.expression.displayName || measure.name, + expression: measure.expression.expression, + }, + ] + : [], + ); + return defs.length ? defs : undefined; +} + +/** + * Returns the set of ephemeral measure names, for quick membership checks. + */ +export function ephemeralMeasureNameSet( + defs: EphemeralMeasureDef[] | undefined, +): Set { + return new Set(defs?.map((def) => def.name) ?? []); +} diff --git a/web-common/src/features/dashboards/ephemeral-measures/types.ts b/web-common/src/features/dashboards/ephemeral-measures/types.ts new file mode 100644 index 000000000000..34ee2dae76d2 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/types.ts @@ -0,0 +1,18 @@ +// An ad-hoc "ephemeral measure" defined by the user for an explore dashboard, +// derived from existing metrics view measures via an arithmetic expression +// (e.g. Profit = revenue - cost). It is computed server-side via the metrics +// APIs' `expression` measure compute, which restricts expressions to +// references to existing measures, numeric literals, basic arithmetic and a +// small allowlist of functions. Definitions live on the explore state, are +// shared by all views (leaderboards, charts, pivot, ...) and are encoded in +// the `ephemeral` URL param so shared links reproduce them. +export type EphemeralMeasureDef = { + // Query alias, e.g. "profit". Must not collide with metrics view field names. + name: string; + // Display name shown in the UI, e.g. "Profit". + displayName: string; + // Arithmetic expression over existing measure names, e.g. "revenue - cost". + expression: string; + // Optional FormatPreset for rendering values; defaults to humanize. + formatPreset?: string; +}; diff --git a/web-common/src/features/dashboards/ephemeral-measures/url-param.spec.ts b/web-common/src/features/dashboards/ephemeral-measures/url-param.spec.ts new file mode 100644 index 000000000000..f5d85228ae7b --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/url-param.spec.ts @@ -0,0 +1,251 @@ +import { PivotChipType } from "@rilldata/web-common/features/dashboards/pivot/types"; +import { metricsExplorerStore } from "@rilldata/web-common/features/dashboards/stores/dashboard-stores"; +import { + AD_BIDS_EXPLORE_INIT, + AD_BIDS_EXPLORE_NAME, + AD_BIDS_METRICS_3_MEASURES_DIMENSIONS, + AD_BIDS_METRICS_INIT, + AD_BIDS_METRICS_VIEW, + AD_BIDS_NAME, + AD_BIDS_TIME_RANGE_SUMMARY, +} from "@rilldata/web-common/features/dashboards/stores/test-data/data"; +import { getInitExploreStateForTest } from "@rilldata/web-common/features/dashboards/stores/test-data/helpers"; +import { getDefaultExplorePreset } from "@rilldata/web-common/features/dashboards/url-state/getDefaultExplorePreset"; +import { + applyURLToExploreState, + getCleanMetricsExploreForAssertion, + useTestFilterManager, +} from "@rilldata/web-common/features/dashboards/url-state/test/url-state-test-utils"; +import { DashboardState_ActivePage } from "@rilldata/web-common/proto/gen/rill/ui/v1/dashboard_pb"; +import { get } from "svelte/store"; +import { beforeEach, describe, expect, it } from "vitest"; +import { + fromEphemeralMeasuresParam, + toEphemeralMeasuresParam, +} from "./url-param"; + +describe("ephemeral url param", () => { + describe("round trips", () => { + const cases = [ + [{ name: "profit", displayName: "Profit", expression: "revenue - cost" }], + [ + { + name: "profit", + displayName: "Profit", + expression: "revenue - cost", + formatPreset: "currency_usd", + }, + { + name: "arpu", + displayName: "ARPU (µ)", + expression: "round(revenue / users, 2)", + }, + ], + // Hostile characters in display names and expressions. + [ + { + name: "weird", + displayName: 'a:b;c,d%e+f"g', + expression: '"total volume" % 100 + 0.5', + }, + ], + ]; + it.each(cases)("%j", (...defs) => { + const param = toEphemeralMeasuresParam(defs); + const { ephemeralMeasures, invalidEntries } = + fromEphemeralMeasuresParam(param); + expect(invalidEntries).toEqual([]); + expect(ephemeralMeasures).toEqual(defs); + }); + }); + + it("returns an empty param for no definitions", () => { + expect(toEphemeralMeasuresParam(undefined)).toBe(""); + expect(toEphemeralMeasuresParam([])).toBe(""); + }); + + it("drops malformed entries and keeps the rest", () => { + const { ephemeralMeasures, invalidEntries } = fromEphemeralMeasuresParam( + "profit:Profit:revenue;bad-entry;other:Other:cost", + ); + expect(ephemeralMeasures.map((d) => d.name)).toEqual(["profit", "other"]); + expect(invalidEntries).toEqual(["bad-entry"]); + }); + + it("drops duplicate names", () => { + const { ephemeralMeasures, invalidEntries } = fromEphemeralMeasuresParam( + "profit:Profit:revenue;profit:Other:cost", + ); + expect(ephemeralMeasures.map((d) => d.name)).toEqual(["profit"]); + expect(invalidEntries).toEqual(["profit:Other:cost"]); + }); +}); + +// Filters live in the ExpressionFilterManager, which needs the specs of the +// metrics view backing the explore. +const getFilterManager = useTestFilterManager({ + [AD_BIDS_NAME]: AD_BIDS_METRICS_VIEW, +}); + +describe("ephemeral measures URL state integration", () => { + beforeEach(() => { + metricsExplorerStore.remove(AD_BIDS_EXPLORE_NAME); + metricsExplorerStore.init( + AD_BIDS_EXPLORE_NAME, + getInitExploreStateForTest( + AD_BIDS_METRICS_3_MEASURES_DIMENSIONS, + AD_BIDS_EXPLORE_INIT, + AD_BIDS_TIME_RANGE_SUMMARY, + ), + ); + }); + + function applyUrl(url: string) { + const defaultExplorePreset = getDefaultExplorePreset( + AD_BIDS_EXPLORE_INIT, + AD_BIDS_METRICS_INIT, + AD_BIDS_TIME_RANGE_SUMMARY.timeRangeSummary, + ); + return applyURLToExploreState( + new URL(url), + AD_BIDS_EXPLORE_INIT, + defaultExplorePreset, + getFilterManager(), + ); + } + + it("restores ephemeral measures and their pivot columns from the URL", () => { + const errors = applyUrl( + "http://localhost/explore/AdBids_explore?view=pivot&rows=publisher&cols=impressions,profit&ephemeral=profit:Profit:impressions*2", + ); + expect(errors).toEqual([]); + + const state = getCleanMetricsExploreForAssertion(); + expect(state.ephemeralMeasures).toEqual([ + { name: "profit", displayName: "Profit", expression: "impressions*2" }, + ]); + expect(state.pivot?.columns).toEqual([ + { id: "impressions", title: "impressions", type: PivotChipType.Measure }, + { id: "profit", title: "Profit", type: PivotChipType.Measure }, + ]); + }); + + it("drops definitions referencing unknown measures, and their columns", () => { + const errors = applyUrl( + "http://localhost/explore/AdBids_explore?view=pivot&cols=impressions,profit&ephemeral=profit:Profit:unknown*2", + ); + expect(errors.map((e) => e.message)).toEqual([ + `Selected adhoc measure: "profit ("unknown" is not a measure in this dashboard)" is not valid.`, + `Selected pivot column: "profit" is not valid.`, + ]); + + const state = getCleanMetricsExploreForAssertion(); + expect(state.ephemeralMeasures).toBeUndefined(); + expect(state.pivot?.columns).toEqual([ + { id: "impressions", title: "impressions", type: PivotChipType.Measure }, + ]); + }); + + it("drops definitions named after a measure hidden from the explore", () => { + // publisher_count exists in the metrics view but not in the explore, and + // the server rejects computed fields that collide with any measure. + const errors = applyUrl( + "http://localhost/explore/AdBids_explore?view=pivot&cols=publisher_count&ephemeral=publisher_count:PC:impressions*2", + ); + expect(errors.length).toBe(2); + expect(errors[0].message).toContain("already used by another field"); + expect( + getCleanMetricsExploreForAssertion().ephemeralMeasures, + ).toBeUndefined(); + }); + + it("does not report all measures visible while a spec measure is hidden", () => { + const errors = applyUrl( + "http://localhost/explore/AdBids_explore?measures=impressions,profit&ephemeral=profit:Profit:impressions*2", + ); + expect(errors).toEqual([]); + const state = getCleanMetricsExploreForAssertion(); + expect(state.visibleMeasures).toEqual(["impressions", "profit"]); + expect(state.allMeasuresVisible).toBe(false); + }); + + it("drops definitions whose name collides with a metrics view field", () => { + const errors = applyUrl( + "http://localhost/explore/AdBids_explore?view=pivot&cols=bid_price&ephemeral=bid_price:Custom:impressions*2", + ); + expect(errors.length).toBe(1); + expect(errors[0].message).toContain("already used by another field"); + + const state = getCleanMetricsExploreForAssertion(); + expect(state.ephemeralMeasures).toBeUndefined(); + }); + + it("accepts conditional formatting on an ephemeral measure", () => { + const errors = applyUrl( + "http://localhost/explore/AdBids_explore?view=pivot&cols=profit&ephemeral=profit:Profit:impressions*2&format=profit:heatmap:greens", + ); + expect(errors).toEqual([]); + + const state = getCleanMetricsExploreForAssertion(); + expect(state.pivot?.measureFormatting).toEqual({ + profit: { mode: "heatmap", scheme: "greens" }, + }); + }); + + it("survives a state → URL → state round trip", async () => { + const { convertPartialExploreStateToUrlParams } = await import( + "@rilldata/web-common/features/dashboards/url-state/convert-partial-explore-state-to-url-params" + ); + const { convertURLSearchParamsToExploreState } = await import( + "@rilldata/web-common/features/dashboards/url-state/convertURLSearchParamsToExploreState" + ); + const { AD_BIDS_METRICS_VIEW } = await import( + "@rilldata/web-common/features/dashboards/stores/test-data/data" + ); + + // Switch to the pivot view first so the add also places a pivot column. + get(metricsExplorerStore).entities[AD_BIDS_EXPLORE_NAME].activePage = + DashboardState_ActivePage.PIVOT; + metricsExplorerStore.addEphemeralMeasure(AD_BIDS_EXPLORE_NAME, { + name: "profit", + displayName: "Profit (net)", + expression: 'impressions * 2 + "bid_price"', + }); + const exploreState = + get(metricsExplorerStore).entities[AD_BIDS_EXPLORE_NAME]; + + const urlParams = convertPartialExploreStateToUrlParams( + AD_BIDS_EXPLORE_INIT, + AD_BIDS_METRICS_VIEW, + exploreState, + undefined, + ); + // Serialize + reparse, as a browser would. + const reparsed = new URLSearchParams(urlParams.toString()); + // An empty `f=` param errors in the filter parser; in the app it is + // stripped by cleanUrlParams, which is not part of this round trip. + if (reparsed.get("f") === "") reparsed.delete("f"); + const { partialExploreState, errors } = + convertURLSearchParamsToExploreState( + reparsed, + AD_BIDS_METRICS_VIEW, + AD_BIDS_EXPLORE_INIT, + {}, + ); + expect(errors).toEqual([]); + expect(partialExploreState.ephemeralMeasures).toEqual([ + { + name: "profit", + displayName: "Profit (net)", + expression: 'impressions * 2 + "bid_price"', + }, + ]); + expect( + partialExploreState.pivot?.columns.find((c) => c.id === "profit"), + ).toEqual({ + id: "profit", + title: "Profit (net)", + type: PivotChipType.Measure, + }); + }); +}); diff --git a/web-common/src/features/dashboards/ephemeral-measures/url-param.ts b/web-common/src/features/dashboards/ephemeral-measures/url-param.ts new file mode 100644 index 000000000000..295f3a84a445 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/url-param.ts @@ -0,0 +1,78 @@ +import type { EphemeralMeasureDef } from "./types"; +import { MAX_EPHEMERAL_MEASURES } from "./validation"; + +/** + * Compact serialization of ephemeral measure definitions for the stateful URL + * (and the explore preset, which stores the same string). + * + * Grammar: + * param := entry (";" entry)* + * entry := name ":" enc(displayName) ":" enc(expression) [":" formatPreset] + * + * `enc` is encodeURIComponent applied per field: it encodes ":" and ";" so + * splitting is unambiguous. `name` is a validated slug and needs no encoding. + * + * Example: `profit:Profit:revenue%20-%20cost;arpu:ARPU:revenue%20%2F%20users:currency_usd` + */ + +export function toEphemeralMeasuresParam( + defs: EphemeralMeasureDef[] | undefined, +): string { + if (!defs?.length) return ""; + return defs + .map((def) => { + const parts = [ + def.name, + encodeURIComponent(def.displayName), + encodeURIComponent(def.expression), + ]; + if (def.formatPreset) parts.push(def.formatPreset); + return parts.join(":"); + }) + .join(";"); +} + +export function fromEphemeralMeasuresParam(param: string): { + ephemeralMeasures: EphemeralMeasureDef[]; + invalidEntries: string[]; +} { + const ephemeralMeasures: EphemeralMeasureDef[] = []; + const invalidEntries: string[] = []; + const seen = new Set(); + + for (const entry of param.split(";")) { + if (!entry) continue; + const parts = entry.split(":"); + if (parts.length < 3 || parts.length > 4) { + invalidEntries.push(entry); + continue; + } + const [name, encDisplayName, encExpression, formatPreset] = parts; + let displayName: string; + let expression: string; + try { + displayName = decodeURIComponent(encDisplayName); + expression = decodeURIComponent(encExpression); + } catch { + invalidEntries.push(entry); + continue; + } + if (!name || !displayName || !expression || seen.has(name)) { + invalidEntries.push(entry); + continue; + } + if (ephemeralMeasures.length >= MAX_EPHEMERAL_MEASURES) { + invalidEntries.push(entry); + continue; + } + seen.add(name); + ephemeralMeasures.push({ + name, + displayName, + expression, + ...(formatPreset ? { formatPreset } : {}), + }); + } + + return { ephemeralMeasures, invalidEntries }; +} diff --git a/web-common/src/features/dashboards/ephemeral-measures/url-state.spec.ts b/web-common/src/features/dashboards/ephemeral-measures/url-state.spec.ts new file mode 100644 index 000000000000..763ad0799f2c --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/url-state.spec.ts @@ -0,0 +1,102 @@ +import { PivotChipType } from "@rilldata/web-common/features/dashboards/pivot/types"; +import type { ExploreState } from "@rilldata/web-common/features/dashboards/stores/explore-state"; +import { describe, expect, it } from "vitest"; +import type { EphemeralMeasureDef } from "./types"; +import { referencedEphemeralMeasures } from "./url-state"; + +const profit: EphemeralMeasureDef = { + name: "profit", + displayName: "Profit", + expression: "revenue - cost", +}; +const arpu: EphemeralMeasureDef = { + name: "arpu", + displayName: "ARPU", + expression: "revenue / users", +}; +const margin: EphemeralMeasureDef = { + name: "margin", + displayName: "Margin", + expression: "profit / revenue", +}; + +describe("referencedEphemeralMeasures", () => { + it("returns nothing without definitions", () => { + expect(referencedEphemeralMeasures({})).toEqual([]); + expect( + referencedEphemeralMeasures({ + ephemeralMeasures: [], + visibleMeasures: [], + }), + ).toEqual([]); + }); + + it("keeps every definition when all measures are visible", () => { + expect( + referencedEphemeralMeasures({ + ephemeralMeasures: [profit, arpu], + allMeasuresVisible: true, + visibleMeasures: ["revenue"], + }), + ).toEqual([profit, arpu]); + }); + + it("drops definitions no view references", () => { + expect( + referencedEphemeralMeasures({ + ephemeralMeasures: [profit, arpu, margin], + allMeasuresVisible: false, + visibleMeasures: ["revenue", "profit"], + leaderboardMeasureNames: ["revenue"], + leaderboardSortByMeasureName: "revenue", + }), + ).toEqual([profit]); + }); + + it("treats leaderboard, sort, TDD and pivot usages as references", () => { + const state: Partial = { + ephemeralMeasures: [profit, arpu, margin], + allMeasuresVisible: false, + visibleMeasures: [], + leaderboardMeasureNames: ["arpu"], + leaderboardSortByMeasureName: "arpu", + }; + expect(referencedEphemeralMeasures(state)).toEqual([arpu]); + + expect( + referencedEphemeralMeasures({ + ...state, + leaderboardMeasureNames: [], + leaderboardSortByMeasureName: "margin", + }), + ).toEqual([margin]); + + expect( + referencedEphemeralMeasures({ + ...state, + leaderboardMeasureNames: [], + leaderboardSortByMeasureName: "revenue", + tdd: { + expandedMeasureName: "profit", + chartType: "" as never, + pinIndex: -1, + }, + }), + ).toEqual([profit]); + + expect( + referencedEphemeralMeasures({ + ...state, + leaderboardMeasureNames: [], + leaderboardSortByMeasureName: "revenue", + pivot: { + columns: [ + { id: "margin", title: "Margin", type: PivotChipType.Measure }, + ], + rows: [], + sorting: [{ id: "profit", desc: true }], + } as never, + }), + ).toEqual([profit, margin]); + }); +}); diff --git a/web-common/src/features/dashboards/ephemeral-measures/url-state.ts b/web-common/src/features/dashboards/ephemeral-measures/url-state.ts new file mode 100644 index 000000000000..db6c9d77f55f --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/url-state.ts @@ -0,0 +1,122 @@ +import type { + MetricsViewSpecMeasure, + V1MetricsViewSpec, +} from "@rilldata/web-common/runtime-client"; +import { ephemeralMeasureToSpecMeasure } from "./measure-mapping"; +import { fromEphemeralMeasuresParam } from "./url-param"; +import type { ExploreState } from "@rilldata/web-common/features/dashboards/stores/explore-state"; +import type { EphemeralMeasureDef } from "./types"; +import { + isReferenceableMeasure, + validateEphemeralMeasureDef, +} from "./validation"; + +/** + * Validates ephemeral measure definitions during URL state conversion. + * `measures` is the explore's measure map (what an expression may reference); + * names are reserved against every field of the metrics view, matching the + * server's collision check, so a definition can never shadow a hidden + * measure, a dimension or the time dimension. Returns the valid definitions + * and human-readable labels for the dropped ones. + */ +export function validateEphemeralDefsAgainstSpec( + defs: EphemeralMeasureDef[], + measures: Map, + metricsView: V1MetricsViewSpec, +): { valid: EphemeralMeasureDef[]; invalidEntries: string[] } { + const knownMeasureNames = new Set( + [...measures.values()] + .filter(isReferenceableMeasure) + .map((m) => m.name as string), + ); + const reservedNames = new Set([ + ...(metricsView.measures ?? []).map((m) => m.name as string), + ...(metricsView.dimensions ?? []).map( + (d) => (d.name || d.column) as string, + ), + ...(metricsView.timeDimension ? [metricsView.timeDimension] : []), + ]); + const valid: EphemeralMeasureDef[] = []; + const invalidEntries: string[] = []; + for (const def of defs) { + const error = validateEphemeralMeasureDef( + def, + knownMeasureNames, + reservedNames, + ); + if (error) { + invalidEntries.push(`${def.name} (${error})`); + } else { + valid.push(def); + // Later definitions may not reuse this name either. + reservedNames.add(def.name); + } + } + return { valid, invalidEntries }; +} + +/** + * Parses and validates a `ephemeral` URL param value. + */ +export function parseAndValidateEphemeralParam( + param: string, + measures: Map, + metricsView: V1MetricsViewSpec, +): { valid: EphemeralMeasureDef[]; invalidEntries: string[] } { + const { ephemeralMeasures, invalidEntries } = + fromEphemeralMeasuresParam(param); + const res = validateEphemeralDefsAgainstSpec( + ephemeralMeasures, + measures, + metricsView, + ); + return { + valid: res.valid, + invalidEntries: [...invalidEntries, ...res.invalidEntries], + }; +} + +/** + * Adds synthetic spec measures for the definitions into the measures map used + * during URL state conversion, so every existing name validation (visible + * measures, leaderboards, sort, pivot columns, formatting, ...) accepts + * ephemeral measure names without per-param special cases. + */ +export function injectEphemeralMeasuresIntoMap( + measures: Map, + defs: EphemeralMeasureDef[], +): void { + for (const def of defs) { + measures.set(def.name, ephemeralMeasureToSpecMeasure(def)); + } +} + +/** + * Returns the definitions the explore state actually uses: visible or + * leaderboard measures, the sort measure, the expanded TDD measure and pivot + * chips. Only these go into the URL; the rest stay in the per-metrics-view + * library (see `library.ts`), which keeps shared links from growing with + * every definition the user has ever created. + */ +export function referencedEphemeralMeasures( + exploreState: Partial, +): EphemeralMeasureDef[] { + const defs = exploreState.ephemeralMeasures ?? []; + if (!defs.length) return defs; + if (exploreState.allMeasuresVisible) return defs; + + const referenced = new Set([ + ...(exploreState.visibleMeasures ?? []), + ...(exploreState.leaderboardMeasureNames ?? []), + ...(exploreState.pivot?.rows ?? []).map((chip) => chip.id), + ...(exploreState.pivot?.columns ?? []).map((chip) => chip.id), + ...(exploreState.pivot?.sorting ?? []).map((sort) => sort.id), + ]); + if (exploreState.leaderboardSortByMeasureName) { + referenced.add(exploreState.leaderboardSortByMeasureName); + } + if (exploreState.tdd?.expandedMeasureName) { + referenced.add(exploreState.tdd.expandedMeasureName); + } + return defs.filter((def) => referenced.has(def.name)); +} diff --git a/web-common/src/features/dashboards/ephemeral-measures/validation.ts b/web-common/src/features/dashboards/ephemeral-measures/validation.ts new file mode 100644 index 000000000000..0447aeb71219 --- /dev/null +++ b/web-common/src/features/dashboards/ephemeral-measures/validation.ts @@ -0,0 +1,124 @@ +import { + MetricsViewSpecMeasureType, + type MetricsViewSpecMeasure, +} from "@rilldata/web-common/runtime-client"; +import { ComparisonModifierSuffixRegex } from "../pivot/types"; +import { parseMeasureExpression } from "./expression-parser"; +import type { EphemeralMeasureDef } from "./types"; + +// Maximum number of ephemeral measures per pivot. +export const MAX_EPHEMERAL_MEASURES = 10; + +export const EPHEMERAL_MEASURE_NAME_REGEX = /^[a-zA-Z][a-zA-Z0-9_]*$/; + +// Accessor suffixes appended to measure names when building requests: +// the pivot's `__delta_abs`-style suffixes (ComparisonModifierSuffixRegex) and +// the leaderboard/dimension-table `_prev`-style suffixes below. An ephemeral +// measure ending in one of these would collide with another measure's +// comparison accessor. +const RESERVED_NAME_SUFFIXES = [ + "_prev", + "_delta", + "_delta_perc", + "_percent_of_total", +]; + +/** + * Returns whether a measure may be referenced from an ephemeral measure's + * expression. Window measures, measures with required dimensions and + * time-comparison measures are excluded: the same views that filter those + * spec measures out of their requests (totals, big numbers, ...) cannot + * filter an ephemeral wrapper, so allowing the reference would produce + * permanent query errors. + */ +export function isReferenceableMeasure( + measure: MetricsViewSpecMeasure, +): boolean { + return ( + !measure.window && + !measure.requiredDimensions?.length && + measure.type !== MetricsViewSpecMeasureType.MEASURE_TYPE_TIME_COMPARISON + ); +} + +/** + * Validates an ephemeral measure's name (the query alias). + * `reservedNames` should contain all metrics view field names (measures, + * dimensions, the time dimension) plus the names of other ephemeral measures. + */ +export function validateEphemeralMeasureName( + name: string, + reservedNames: Set, +): string | undefined { + if (!EPHEMERAL_MEASURE_NAME_REGEX.test(name)) { + return "name must start with a letter and contain only letters, numbers and underscores"; + } + if ( + ComparisonModifierSuffixRegex.test(name) || + RESERVED_NAME_SUFFIXES.some((suffix) => name.endsWith(suffix)) + ) { + return "name must not end with a comparison suffix"; + } + if (name.includes("_rill_")) { + return 'name must not contain "_rill_"'; + } + if (reservedNames.has(name)) { + return `"${name}" is already used by another field`; + } + return undefined; +} + +/** + * Derives a query alias from a display name that passes + * `validateEphemeralMeasureName`: reserved suffixes and collisions are + * resolved with a numeric suffix, so a valid display name is always saveable. + */ +export function slugifyEphemeralMeasureName( + label: string, + reservedNames: Set, +): string { + let slug = label + .toLowerCase() + .replace(/[^a-z0-9_]+/g, "_") + // Collapse runs of underscores so the `__previous`-style pivot suffixes + // can never appear. + .replace(/_+/g, "_") + .replace(/^_+|_+$/g, ""); + if (!EPHEMERAL_MEASURE_NAME_REGEX.test(slug)) slug = `m_${slug}`; + while (slug.includes("_rill_")) slug = slug.replace("_rill_", "_rill"); + let candidate = slug; + for (let i = 2; validateEphemeralMeasureName(candidate, reservedNames); i++) { + candidate = `${slug}_${i}`; + } + return candidate; +} + +/** + * Validates a full ephemeral measure definition. + * `knownMeasureNames` are the measure names an expression may reference + * (the metrics view's measures available in this explore; ephemeral measures + * cannot reference other ephemeral measures). + */ +export function validateEphemeralMeasureDef( + def: EphemeralMeasureDef, + knownMeasureNames: Set, + reservedNames: Set, +): string | undefined { + const nameError = validateEphemeralMeasureName(def.name, reservedNames); + if (nameError) return nameError; + + if (def.displayName.trim() === "") { + return "display name is required"; + } + + const parsed = parseMeasureExpression(def.expression); + if (parsed.error) { + return parsed.error.message; + } + for (const ref of parsed.refs) { + if (!knownMeasureNames.has(ref)) { + return `"${ref}" is not a measure in this dashboard`; + } + } + return undefined; +} diff --git a/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte b/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte index 2554287f04db..e295823b780d 100644 --- a/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte +++ b/web-common/src/features/dashboards/leaderboard/Leaderboard.svelte @@ -1,6 +1,8 @@
- +
+ + +
{#if !collapsed}
{#if items.length} - + {:else}

{m.dashboard_no_available_fields()} diff --git a/web-common/src/features/dashboards/pivot/PivotHeader.svelte b/web-common/src/features/dashboards/pivot/PivotHeader.svelte index ddf5bbd52cdc..bfb84cee37dd 100644 --- a/web-common/src/features/dashboards/pivot/PivotHeader.svelte +++ b/web-common/src/features/dashboards/pivot/PivotHeader.svelte @@ -2,6 +2,7 @@ import Column from "@rilldata/web-common/components/icons/Column.svelte"; import Row from "@rilldata/web-common/components/icons/Row.svelte"; import { splitPivotChips } from "@rilldata/web-common/features/dashboards/pivot/pivot-utils.ts"; + import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; import { slide } from "svelte/transition"; import DragList from "./DragList.svelte"; import PivotAutoArrangeZone from "./PivotAutoArrangeZone.svelte"; @@ -21,8 +22,16 @@ | ((measureName: string, fmt: PivotMeasureFormatting | null) => void) | undefined = undefined; export let lowerIsBetterMap: Record = {}; + export let ephemeralMeasures: EphemeralMeasureDef[] | undefined = undefined; + export let onEditEphemeralMeasure: ((id: string) => void) | undefined = + undefined; + export let onDeleteEphemeralMeasure: ((id: string) => void) | undefined = + undefined; $: ({ rows, columns, tableMode, measureFormatting } = pivotState); + $: ephemeralMeasureNames = new Set( + ephemeralMeasures?.map((def) => def.name) ?? [], + ); $: splitColumns = splitPivotChips(columns); $: fullColumns = splitColumns.dimension.concat(splitColumns.measure); $: isFlat = tableMode === "flat"; @@ -78,6 +87,9 @@ {measureFormatting} {setMeasureFormatting} {lowerIsBetterMap} + {ephemeralMeasureNames} + {onEditEphemeralMeasure} + {onDeleteEphemeralMeasure} />

diff --git a/web-common/src/features/dashboards/pivot/PivotSidebar.svelte b/web-common/src/features/dashboards/pivot/PivotSidebar.svelte index 27192e53afe8..27b78e682cae 100644 --- a/web-common/src/features/dashboards/pivot/PivotSidebar.svelte +++ b/web-common/src/features/dashboards/pivot/PivotSidebar.svelte @@ -12,11 +12,15 @@ splitTagItems, } from "@rilldata/web-common/features/dashboards/pivot/pivot-utils.ts"; import { type TimeControlState } from "@rilldata/web-common/features/dashboards/time-controls/time-control-store"; + import { getStateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; + import { metricsExplorerStore } from "@rilldata/web-common/features/dashboards/stores/dashboard-stores"; import { onMount } from "svelte"; import type { PivotSidebarSection, PivotState, } from "web-common/src/features/dashboards/pivot/types.ts"; + import Add from "@rilldata/web-common/components/icons/Add.svelte"; + import { ephemeralMeasureDialog } from "../ephemeral-measures/dialog-store"; import PivotDrag from "./PivotDrag.svelte"; import PivotTagRow from "./PivotTagRow.svelte"; import { timePillActions, timePillSelectors } from "./time-pill-store"; @@ -37,8 +41,26 @@ "timeStart" | "timeEnd" | "minTimeGrain" >; + const { exploreName, dashboardStore, validSpecStore } = getStateManagers(); + $: ({ rows, columns, tableMode } = pivotState); $: splitColumns = splitPivotChips(columns); + $: ephemeralMeasureNames = new Set( + $dashboardStore.ephemeralMeasures?.map((def) => def.name) ?? [], + ); + + function editEphemeralMeasure(id: string) { + const def = $dashboardStore.ephemeralMeasures?.find((d) => d.name === id); + if (def) ephemeralMeasureDialog.set({ def }); + } + + function deleteEphemeralMeasure(id: string) { + metricsExplorerStore.removeEphemeralMeasure( + $exploreName, + id, + $validSpecStore.data?.explore, + ); + } let sidebarHeight = 0; let searchText = ""; @@ -213,7 +235,21 @@ title={MEASURES_ZONE} label={m.dashboard_measures()} items={filteredMeasures} - /> + {ephemeralMeasureNames} + onEditEphemeralMeasure={editEphemeralMeasure} + onDeleteEphemeralMeasure={deleteEphemeralMeasure} + > + + { const measureName = m.id; const group = [measureName]; @@ -138,10 +141,15 @@ export function getPivotConfig( measureNames, rowDimensionNames, colDimensionNames, - allMeasures: allMeasures({ - validMetricsView: metricsView, - validExplore: explore, - }), + // Ephemeral measures get a synthetic spec entry so column definitions + // (labels, formatters, tooltips) resolve them like any other measure. + allMeasures: appendEphemeralSpecMeasures( + allMeasures({ + validMetricsView: metricsView, + validExplore: explore, + }), + ephemeralMeasures, + ), allDimensions: allDimensions({ validMetricsView: metricsView, validExplore: explore, @@ -153,6 +161,7 @@ export function getPivotConfig( time, searchText, isFlat, + ephemeralMeasures, }; const currentKey = getPivotConfigKey(config); diff --git a/web-common/src/features/dashboards/pivot/pivot-export.ts b/web-common/src/features/dashboards/pivot/pivot-export.ts index 0591d4f39c79..4cf301d9a833 100644 --- a/web-common/src/features/dashboards/pivot/pivot-export.ts +++ b/web-common/src/features/dashboards/pivot/pivot-export.ts @@ -13,7 +13,7 @@ import { import { get } from "svelte/store"; import type { StateManagers } from "../state-managers/state-managers"; import { getPivotConfig } from "./pivot-data-config"; -import { prepareMeasureForComparison } from "./pivot-utils"; +import { prepareMeasuresForRequest } from "./pivot-utils"; import { COMPARISON_DELTA, COMPARISON_PERCENT, @@ -189,9 +189,10 @@ export function getPivotAggregationRequest({ metricsView: metricsViewName, timeRange, comparisonTimeRange: comparisonTime, - measures: enableComparison - ? prepareMeasureForComparison(measures) - : measures, + measures: prepareMeasuresForRequest( + measures, + exploreState.ephemeralMeasures, + ), dimensions: allDimensions, where: sanitiseExpression(exploreState.whereFilter, undefined), pivotOn, diff --git a/web-common/src/features/dashboards/pivot/pivot-queries.ts b/web-common/src/features/dashboards/pivot/pivot-queries.ts index 478341c3f6f1..165ca92c27c6 100644 --- a/web-common/src/features/dashboards/pivot/pivot-queries.ts +++ b/web-common/src/features/dashboards/pivot/pivot-queries.ts @@ -26,7 +26,7 @@ import { getTimeGrainFromDimension, getUriMeasuresForDimensions, isTimeDimension, - prepareMeasureForComparison, + prepareMeasuresForRequest, } from "./pivot-utils"; import { COMPARISON_DELTA, @@ -79,7 +79,10 @@ export function createPivotAggregationRowQuery( ctx.runtimeClient, { metricsView: metricsViewName, - measures: prepareMeasureForComparison(measures), + measures: prepareMeasuresForRequest( + measures, + config.ephemeralMeasures, + ), dimensions, where: sanitiseExpression(whereFilter, undefined), timeRange: { diff --git a/web-common/src/features/dashboards/pivot/pivot-utils.ts b/web-common/src/features/dashboards/pivot/pivot-utils.ts index 3c790e926cb4..7be4af5a9639 100644 --- a/web-common/src/features/dashboards/pivot/pivot-utils.ts +++ b/web-common/src/features/dashboards/pivot/pivot-utils.ts @@ -30,6 +30,8 @@ import { getURIRequestMeasure } from "@rilldata/web-common/features/dashboards/d import { SHOW_MORE_BUTTON } from "./pivot-constants"; import { getColumnFiltersForPage } from "./pivot-infinite-scroll"; import { mergeFilters } from "./pivot-merge-filters"; +import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; +import { mapEphemeralMeasuresForRequest } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; import { COMPARISON_DELTA, COMPARISON_PERCENT, @@ -95,8 +97,11 @@ export function getPivotConfigKey(config: PivotDataStoreConfig) { const dimsAndMeasures = rowDimensionNames .concat(measureNames, colDimensionNames) .join("_"); + // Ephemeral measure definitions are part of the key so editing an + // expression (without renaming) refetches instead of serving cached data. + const ephemeralMeasuresKey = JSON.stringify(config.ephemeralMeasures ?? []); - return `${dimsAndMeasures}_${timeKey}_${sortingKey}_${tableModeKey}_${filterKey}_${enableComparison}_${comparisonTimeKey}_${showTotalsColumn}_${showTotalsRow}_${rowLimit ?? "all"}_${outermostRowLimit ?? "none"}`; + return `${dimsAndMeasures}_${timeKey}_${sortingKey}_${tableModeKey}_${filterKey}_${enableComparison}_${comparisonTimeKey}_${showTotalsColumn}_${showTotalsRow}_${rowLimit ?? "all"}_${outermostRowLimit ?? "none"}_${ephemeralMeasuresKey}`; } /** @@ -540,6 +545,21 @@ export function prepareMeasureForComparison( }); } +/** + * Maps plain measure names to request measures: comparison-suffixed names get + * their comparison compute, and ephemeral measure names get the `expression` + * compute carrying their definition. + */ +export function prepareMeasuresForRequest( + measures: V1MetricsViewAggregationMeasure[], + ephemeralMeasures: EphemeralMeasureDef[] | undefined, +): V1MetricsViewAggregationMeasure[] { + return mapEphemeralMeasuresForRequest( + prepareMeasureForComparison(measures), + ephemeralMeasures, + ); +} + export function canEnablePivotComparison( pivotState: PivotState, comparisonStart: string | Date | undefined, diff --git a/web-common/src/features/dashboards/pivot/types.ts b/web-common/src/features/dashboards/pivot/types.ts index b76d9ab7b497..fb7029602220 100644 --- a/web-common/src/features/dashboards/pivot/types.ts +++ b/web-common/src/features/dashboards/pivot/types.ts @@ -1,3 +1,4 @@ +import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; import { type TimeRangeString } from "@rilldata/web-common/lib/time/types"; import type { MetricsViewSpecDimension, @@ -155,6 +156,10 @@ export interface PivotDataStoreConfig { comparisonTime: TimeRangeString | undefined; searchText: string | undefined; isFlat: boolean; + // ephemeral measure definitions available to this pivot. + // Sourced from the explore state in explore dashboards, and from the + // component spec in canvas table/pivot components. + ephemeralMeasures?: EphemeralMeasureDef[]; } export interface PivotAxesData { diff --git a/web-common/src/features/dashboards/proto-state/fromProto.ts b/web-common/src/features/dashboards/proto-state/fromProto.ts index 88267d948131..d2919f6b6164 100644 --- a/web-common/src/features/dashboards/proto-state/fromProto.ts +++ b/web-common/src/features/dashboards/proto-state/fromProto.ts @@ -226,6 +226,15 @@ export function getDashboardStateFromProto( entity.leaderboardMeasureNames = dashboard.leaderboardMeasures; } + if (dashboard.ephemeralMeasures?.length) { + entity.ephemeralMeasures = dashboard.ephemeralMeasures.map((def) => ({ + name: def.name, + displayName: def.displayName, + expression: def.expression, + ...(def.formatPreset ? { formatPreset: def.formatPreset } : {}), + })); + } + if (dashboard.activePage === DashboardState_ActivePage.PIVOT) { entity.pivot = fromPivotProto(dashboard, metricsView); } else if (dashboard.activePage !== DashboardState_ActivePage.UNSPECIFIED) { @@ -397,6 +406,10 @@ function fromPivotProto( } }; + const ephemeralMeasuresMap = new Map( + (dashboard.ephemeralMeasures ?? []).map((def) => [def.name, def]), + ); + const measuresMap = getMapFromArray( metricsView.measures ?? [], (m) => m.name, @@ -405,7 +418,11 @@ function fromPivotProto( const mes = measuresMap.get(name); return { id: name, - title: mes?.displayName || mes?.name || "Unknown", + title: + mes?.displayName || + mes?.name || + ephemeralMeasuresMap.get(name)?.displayName || + "Unknown", type: PivotChipType.Measure, }; }; diff --git a/web-common/src/features/dashboards/proto-state/toProto.ts b/web-common/src/features/dashboards/proto-state/toProto.ts index 7b78805b34b7..ad5c291d0f19 100644 --- a/web-common/src/features/dashboards/proto-state/toProto.ts +++ b/web-common/src/features/dashboards/proto-state/toProto.ts @@ -76,6 +76,14 @@ export function getProtoFromDashboardState( if (!exploreState) return ""; const state: PartialMessage = {}; + if (exploreState.ephemeralMeasures?.length) { + state.ephemeralMeasures = exploreState.ephemeralMeasures.map((def) => ({ + name: def.name, + displayName: def.displayName, + expression: def.expression, + formatPreset: def.formatPreset ?? "", + })); + } if (exploreState.whereFilter) { state.where = toExpressionProto(exploreState.whereFilter); } diff --git a/web-common/src/features/dashboards/state-managers/loaders/DashboardStateDataLoader.ts b/web-common/src/features/dashboards/state-managers/loaders/DashboardStateDataLoader.ts index 4f48473d10d4..c8c3dd51baa3 100644 --- a/web-common/src/features/dashboards/state-managers/loaders/DashboardStateDataLoader.ts +++ b/web-common/src/features/dashboards/state-managers/loaders/DashboardStateDataLoader.ts @@ -5,7 +5,12 @@ import { } from "@rilldata/web-common/features/compound-query-result"; import { cascadingExploreStateMerge } from "@rilldata/web-common/features/dashboards/state-managers/cascading-explore-state-merge"; import { getPartialExploreStateFromSessionStorage } from "@rilldata/web-common/features/dashboards/state-managers/loaders/explore-web-view-store"; +import { + loadEphemeralMeasureLibrary, + mergeEphemeralMeasureDefs, +} from "@rilldata/web-common/features/dashboards/ephemeral-measures/library"; import { getMostRecentPartialExploreState } from "@rilldata/web-common/features/dashboards/state-managers/loaders/most-recent-explore-state"; +import { validateAndCleanExploreState } from "@rilldata/web-common/features/dashboards/stores/validate-and-clean-explore-state"; import { getExploreStateFromYAMLConfig } from "@rilldata/web-common/features/dashboards/stores/get-explore-state-from-yaml-config"; import { getRillDefaultExploreState } from "@rilldata/web-common/features/dashboards/stores/get-rill-default-explore-state"; import type { ExploreState } from "@rilldata/web-common/features/dashboards/stores/explore-state"; @@ -373,8 +378,43 @@ export class DashboardStateDataLoader { const finalExploreState = cascadingExploreStateMerge( nonEmptyExploreStateOrder, ) as ExploreState; + this.mergeEphemeralMeasureLibrary( + metricsViewSpec, + exploreSpec, + finalExploreState, + ); correctExploreState(metricsViewSpec, finalExploreState); return finalExploreState; } + + /** + * Restores ad-hoc measure definitions from the per-metrics-view library. + * The URL only carries the definitions the state references, so this is what + * brings back hidden definitions, and definitions created in another explore + * on the same metrics view. Definitions from the state win over stored copies. + */ + private mergeEphemeralMeasureLibrary( + metricsViewSpec: V1MetricsViewSpec, + exploreSpec: V1ExploreSpec, + exploreState: ExploreState, + ) { + if (this.disableMostRecentDashboardState || !exploreSpec.metricsView) { + return; + } + const library = loadEphemeralMeasureLibrary( + exploreSpec.metricsView, + this.storageNamespacePrefix, + ); + if (!library.length) return; + // Validate stored definitions against the current spec, dropping any that + // reference measures this explore does not expose. + const libraryState: Partial = { ephemeralMeasures: library }; + validateAndCleanExploreState(metricsViewSpec, exploreSpec, libraryState); + if (!libraryState.ephemeralMeasures?.length) return; + exploreState.ephemeralMeasures = mergeEphemeralMeasureDefs( + exploreState.ephemeralMeasures ?? [], + libraryState.ephemeralMeasures, + ); + } } diff --git a/web-common/src/features/dashboards/state-managers/loaders/DashboardStateSync.ts b/web-common/src/features/dashboards/state-managers/loaders/DashboardStateSync.ts index 77a0f956f741..d499c92d1da9 100644 --- a/web-common/src/features/dashboards/state-managers/loaders/DashboardStateSync.ts +++ b/web-common/src/features/dashboards/state-managers/loaders/DashboardStateSync.ts @@ -1,6 +1,11 @@ import { goto } from "$app/navigation"; import { page } from "$app/stores"; import { DashboardStateDataLoader } from "@rilldata/web-common/features/dashboards/state-managers/loaders/DashboardStateDataLoader"; +import { + syncEphemeralMeasureLibrary, + upsertIntoEphemeralMeasureLibrary, +} from "@rilldata/web-common/features/dashboards/ephemeral-measures/library"; +import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; import { saveMostRecentPartialExploreState } from "@rilldata/web-common/features/dashboards/state-managers/loaders/most-recent-explore-state"; import { metricsExplorerStore, @@ -15,6 +20,7 @@ import { import { updateExploreSessionStore } from "@rilldata/web-common/features/dashboards/state-managers/loaders/explore-web-view-store"; import { getCleanedUrlParamsForGoto } from "@rilldata/web-common/features/dashboards/url-state/convert-partial-explore-state-to-url-params"; import { createRillDefaultExploreUrlParams } from "@rilldata/web-common/features/dashboards/url-state/get-rill-default-explore-url-params"; +import type { V1ExploreSpec } from "@rilldata/web-common/runtime-client"; import type { RuntimeClient } from "@rilldata/web-common/runtime-client/v2"; import type { AfterNavigate } from "@sveltejs/kit"; import { getContext, setContext } from "svelte"; @@ -41,6 +47,9 @@ export class DashboardStateSync { private readonly unsubExploreState: (() => void) | undefined; private initialized = false; + // Ad-hoc measure definitions as of the last persisted state. + // Used to detect deletions to mirror into the per-metrics-view library. + private lastEphemeralMeasures: EphemeralMeasureDef[] | undefined; // There can be cases when updating either the url or the state can impact the code handling the other part. // So we need a lock to make sure an update doesn't trigger the counterpart code. private updating = false; @@ -96,6 +105,26 @@ export class DashboardStateSync { this.unsubExploreState?.(); } + /** + * Mirrors the state's ad-hoc measure definitions into the per-metrics-view + * library: edits and additions are upserted, and a definition that was in + * the previously persisted state but is gone now was deleted by the user. + */ + private syncEphemeralMeasureLibrary( + exploreSpec: V1ExploreSpec, + next: EphemeralMeasureDef[] | undefined, + ) { + if (exploreSpec.metricsView) { + syncEphemeralMeasureLibrary( + exploreSpec.metricsView, + this.extraPrefix, + this.lastEphemeralMeasures, + next, + ); + } + this.lastEphemeralMeasures = next; + } + public getUrlForExploreState(exploreState: ExploreState) { const { data: validSpecData } = get(this.dataLoader.validSpecQuery); const exploreSpec = validSpecData?.explore ?? {}; @@ -187,7 +216,15 @@ export class DashboardStateSync { this.extraPrefix, initExploreState, ); + // Definitions from the URL or a bookmark join the library; a missing + // definition on init is not a deletion, so only upsert here. + upsertIntoEphemeralMeasureLibrary( + exploreSpec.metricsView ?? "", + this.extraPrefix, + initExploreState.ephemeralMeasures, + ); } + this.lastEphemeralMeasures = initExploreState.ephemeralMeasures; this.expressionFilterManager.setUrlParams(redirectUrl.searchParams); this.expressionFilterManager.updating = false; @@ -298,6 +335,10 @@ export class DashboardStateSync { this.extraPrefix, updatedExploreState, ); + this.syncEphemeralMeasureLibrary( + exploreSpec, + updatedExploreState.ephemeralMeasures, + ); } } finally { // Release before the goto below: state changes made while the navigation is in flight @@ -366,6 +407,10 @@ export class DashboardStateSync { this.extraPrefix, exploreState, ); + this.syncEphemeralMeasureLibrary( + exploreSpec, + exploreState.ephemeralMeasures, + ); } this.expressionFilterManager.setUrlParams(newUrl.searchParams); diff --git a/web-common/src/features/dashboards/state-managers/loaders/most-recent-explore-state.ts b/web-common/src/features/dashboards/state-managers/loaders/most-recent-explore-state.ts index 8eec7a7c9077..574ff84bbca7 100644 --- a/web-common/src/features/dashboards/state-managers/loaders/most-recent-explore-state.ts +++ b/web-common/src/features/dashboards/state-managers/loaders/most-recent-explore-state.ts @@ -67,6 +67,8 @@ export function saveMostRecentPartialExploreState( { selectedTimezone: exploreState.selectedTimezone, + // Ad-hoc measure definitions are stored per metrics view instead; see + // ephemeral-measures/library.ts. visibleMeasures: exploreState.visibleMeasures, allMeasuresVisible: exploreState.allMeasuresVisible, visibleDimensions: exploreState.visibleDimensions, diff --git a/web-common/src/features/dashboards/state-managers/selectors/active-measure.ts b/web-common/src/features/dashboards/state-managers/selectors/active-measure.ts index 82e4632b14a8..3f5b517fe0db 100644 --- a/web-common/src/features/dashboards/state-managers/selectors/active-measure.ts +++ b/web-common/src/features/dashboards/state-managers/selectors/active-measure.ts @@ -1,3 +1,4 @@ +import { appendEphemeralSpecMeasures } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; import type { MetricsViewSpecMeasure } from "@rilldata/web-common/runtime-client"; import type { DashboardDataSources } from "./types"; @@ -8,10 +9,13 @@ export const activeMeasure = ( return undefined; } - const activeMeasure = dashData.validMetricsView.measures.find( - (measure) => measure.name === activeMeasureName(dashData), - ); - return activeMeasure; + // Ephemeral measures can be the sort measure too; they have no spec entry, + // so synthesize one for formatters and tooltips. + const name = activeMeasureName(dashData); + return appendEphemeralSpecMeasures( + dashData.validMetricsView.measures, + dashData.dashboard.ephemeralMeasures, + ).find((measure) => measure.name === name); }; export const activeMeasureName = (dashData: DashboardDataSources): string => { diff --git a/web-common/src/features/dashboards/state-managers/selectors/measures.ts b/web-common/src/features/dashboards/state-managers/selectors/measures.ts index e67645f7c66c..5ccf11a0b076 100644 --- a/web-common/src/features/dashboards/state-managers/selectors/measures.ts +++ b/web-common/src/features/dashboards/state-managers/selectors/measures.ts @@ -1,3 +1,4 @@ +import { appendEphemeralSpecMeasures } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; import type { ExploreState } from "@rilldata/web-common/features/dashboards/stores/explore-state"; import { MetricsViewSpecMeasureType, @@ -32,21 +33,26 @@ type AggregationMeasureRef = Pick< export const allMeasures = ({ validMetricsView, validExplore, -}: Pick< - DashboardDataSources, - "validMetricsView" | "validExplore" ->): MetricsViewSpecMeasure[] => { + dashboard, +}: Pick & + Partial< + Pick + >): MetricsViewSpecMeasure[] => { if (!validMetricsView?.measures || !validExplore?.measures) return []; - return ( - validMetricsView.measures - .filter((m) => validExplore.measures!.includes(m.name!)) - // Sort the filtered measures based on their order in validExplore.measures - .sort( - (a, b) => - validExplore.measures!.indexOf(a.name!) - - validExplore.measures!.indexOf(b.name!), - ) + const specMeasures = validMetricsView.measures + .filter((m) => validExplore.measures!.includes(m.name!)) + // Sort the filtered measures based on their order in validExplore.measures + .sort( + (a, b) => + validExplore.measures!.indexOf(a.name!) - + validExplore.measures!.indexOf(b.name!), + ); + // ephemeral measures behave like regular measures across the + // explore; they get synthetic spec entries so labels/formatting resolve. + return appendEphemeralSpecMeasures( + specMeasures, + dashboard?.ephemeralMeasures, ); }; @@ -57,8 +63,9 @@ export const visibleMeasures = ({ }: DashboardDataSources): MetricsViewSpecMeasure[] => { if (!validMetricsView?.measures || !validExplore?.measures) return []; + const all = allMeasures({ validMetricsView, validExplore, dashboard }); return dashboard.visibleMeasures - .map((mes) => validMetricsView.measures?.find((m) => m.name === mes)) + .map((mes) => all.find((m) => m.name === mes)) .filter(Boolean) as MetricsViewSpecMeasure[]; }; @@ -70,13 +77,11 @@ export const getMeasureByName = ( }; }; -export const measureLabel = ({ - validMetricsView, -}: DashboardDataSources): ((m: string) => string) => { +export const measureLabel = ( + dashData: DashboardDataSources, +): ((m: string) => string) => { return (measureName) => { - const measure = validMetricsView?.measures?.find( - (d) => d.name === measureName, - ); + const measure = allMeasures(dashData).find((d) => d.name === measureName); return measure?.displayName || measureName; }; }; @@ -148,7 +153,17 @@ export const filterOutSomeAdvancedMeasures = ( ) => { const measuresSeen = new Set(); + const ephemeralMeasureNames = new Set( + exploreState.ephemeralMeasures?.map((def) => def.name) ?? [], + ); + return measureNames.filter((measureName) => { + // ephemeral measures are simple aggregations; always supported. + if (ephemeralMeasureNames.has(measureName)) { + if (measuresSeen.has(measureName)) return false; + measuresSeen.add(measureName); + return true; + } const measureSpec = metricsViewSpec.measures?.find( (m) => m.name === measureName, ); @@ -187,19 +202,25 @@ export const filterOutSomeAdvancedAggregationMeasures = < ): T[] => { const measuresSeen = new Set(); + const ephemeralMeasureNames = new Set( + exploreState.ephemeralMeasures?.map((def) => def.name) ?? [], + ); + return measures.filter((measure) => { + const sourceMeasureName = + measure.comparisonDelta?.measure ?? + measure.comparisonValue?.measure ?? + measure.comparisonRatio?.measure ?? + measure.percentOfTotal?.measure ?? + measure.name ?? + ""; // Ephemeral expression measures are derived from measures already in the spec and have no spec // entry of their own, so there is no source measure to resolve or check for support. - const isEphemeralExpression = - !!measure.expression && typeof measure.expression === "object"; - if (!isEphemeralExpression) { - const sourceMeasureName = - measure.comparisonDelta?.measure ?? - measure.comparisonValue?.measure ?? - measure.comparisonRatio?.measure ?? - measure.percentOfTotal?.measure ?? - measure.name ?? - ""; + // The same applies to comparison measures derived from an ephemeral measure. + const isEphemeral = + (!!measure.expression && typeof measure.expression === "object") || + ephemeralMeasureNames.has(sourceMeasureName); + if (!isEphemeral) { const measureSpec = metricsViewSpec.measures?.find( (m) => m.name === sourceMeasureName, ); diff --git a/web-common/src/features/dashboards/state-managers/selectors/pivot.ts b/web-common/src/features/dashboards/state-managers/selectors/pivot.ts index 9c2d9857c78f..2b9aaaba64fd 100644 --- a/web-common/src/features/dashboards/state-managers/selectors/pivot.ts +++ b/web-common/src/features/dashboards/state-managers/selectors/pivot.ts @@ -21,7 +21,7 @@ export const pivotSelectors = { dashData.dashboard.pivot.columns, ).measure; - return measures + const specChips = measures .filter((m) => !columnMeasures.find((c) => c.id === m.name)) .map((measure) => ({ id: measure.name || "Unknown", @@ -29,6 +29,18 @@ export const pivotSelectors = { type: PivotChipType.Measure, description: measure.description, })); + + // Unplaced ephemeral measures stay available for re-adding until deleted. + const ephemeralChips = (dashData.dashboard.ephemeralMeasures ?? []) + .filter((def) => !columnMeasures.find((c) => c.id === def.name)) + .map((def) => ({ + id: def.name, + title: def.displayName, + type: PivotChipType.Measure, + description: def.expression, + })); + + return [...specChips, ...ephemeralChips]; }, dimensions: ({ validMetricsView, diff --git a/web-common/src/features/dashboards/stores/AdvancedMeasureCorrector.ts b/web-common/src/features/dashboards/stores/AdvancedMeasureCorrector.ts index 09fff3e8766f..9d47098db3df 100644 --- a/web-common/src/features/dashboards/stores/AdvancedMeasureCorrector.ts +++ b/web-common/src/features/dashboards/stores/AdvancedMeasureCorrector.ts @@ -18,11 +18,15 @@ import { export class AdvancedMeasureCorrector { private measuresMap: Map; private measuresGrains: Map; + private ephemeralMeasureNames: Set; private constructor( private readonly exploreState: ExploreState, private readonly metricsViewSpec: V1MetricsViewSpec, ) { + this.ephemeralMeasureNames = new Set( + exploreState.ephemeralMeasures?.map((def) => def.name) ?? [], + ); this.measuresMap = getMapFromArray( metricsViewSpec.measures ?? [], (m) => m.name ?? "", @@ -124,7 +128,13 @@ export class AdvancedMeasureCorrector { measureName: string, supportsComparisonMeasure: boolean, supportsWindowedMeasure: boolean, + supportsEphemeralMeasure = true, ) { + // ephemeral measures are not in the metrics view spec; + // they are validated separately when parsing the URL state. + if (this.ephemeralMeasureNames.has(measureName)) { + return !supportsEphemeralMeasure; + } const measure = this.measuresMap.get(measureName); if (!measure) return true; const grain = diff --git a/web-common/src/features/dashboards/stores/dashboard-stores.ts b/web-common/src/features/dashboards/stores/dashboard-stores.ts index 88e1d7a1b430..694a0de4faf5 100644 --- a/web-common/src/features/dashboards/stores/dashboard-stores.ts +++ b/web-common/src/features/dashboards/stores/dashboard-stores.ts @@ -35,6 +35,8 @@ import { type PivotMeasureFormatting, type PivotTableMode, } from "../pivot/types"; +import { parseMeasureExpression } from "../ephemeral-measures/expression-parser"; +import type { EphemeralMeasureDef } from "../ephemeral-measures/types"; import type { ExpressionFilterManager } from "@rilldata/web-common/features/dashboards/filters/ExpressionFilterManager.svelte.ts"; export interface MetricsExplorerStoreType { @@ -75,13 +77,46 @@ export function includeExcludeModeFromFilters( return map; } +// syncEphemeralMeasures drops ephemeral measures that reference measures no +// longer in the explore, so a spec change never leaves views permanently +// erroring. Must run before syncMeasures/syncDimensions, which treat the +// remaining ephemeral measure names as valid. +function syncEphemeralMeasures( + explore: V1ExploreSpec, + exploreState: ExploreState, +) { + if (!exploreState.ephemeralMeasures?.length) return; + const measuresSet = new Set(explore.measures ?? []); + const dimensionsSet = new Set(explore.dimensions ?? []); + exploreState.ephemeralMeasures = exploreState.ephemeralMeasures.filter( + (def) => { + // A field later added to the spec with the same name must win; + // otherwise the stale definition would silently shadow it in requests. + if (measuresSet.has(def.name) || dimensionsSet.has(def.name)) { + return false; + } + const parsed = parseMeasureExpression(def.expression); + if (parsed.error) return false; + return parsed.refs.every((ref) => measuresSet.has(ref)); + }, + ); + if (!exploreState.ephemeralMeasures.length) { + exploreState.ephemeralMeasures = undefined; + } +} + function syncMeasures(explore: V1ExploreSpec, exploreState: ExploreState) { const measuresSet = new Set(explore.measures ?? []); + // Ephemeral measure names are valid anywhere a measure name is used. + const validNames = new Set([ + ...measuresSet, + ...(exploreState.ephemeralMeasures?.map((def) => def.name) ?? []), + ]); // sync measures with selected leaderboard measure and ensure default measure is set if (explore.measures?.length) { const defaultMeasure = explore.measures[0]; - if (!measuresSet.has(exploreState.leaderboardSortByMeasureName)) { + if (!validNames.has(exploreState.leaderboardSortByMeasureName)) { exploreState.leaderboardSortByMeasureName = defaultMeasure; } if (!exploreState.leaderboardMeasureNames?.length) { @@ -102,18 +137,22 @@ function syncMeasures(explore: V1ExploreSpec, exploreState: ExploreState) { if ( exploreState.tdd.expandedMeasureName && - !measuresSet.has(exploreState.tdd.expandedMeasureName) + !validNames.has(exploreState.tdd.expandedMeasureName) ) { exploreState.tdd.expandedMeasureName = undefined; } if (exploreState.allMeasuresVisible) { // this makes sure that the visible keys is in sync with list of measures - exploreState.visibleMeasures = [...measuresSet]; + // (ephemeral measures count as selectable measures too) + exploreState.visibleMeasures = [ + ...measuresSet, + ...(exploreState.ephemeralMeasures?.map((def) => def.name) ?? []), + ]; } else { // remove any visible measures that doesn't exist anymore exploreState.visibleMeasures = exploreState.visibleMeasures.filter((m) => - measuresSet.has(m), + validNames.has(m), ); // If there are no visible measures, make the first measure visible if (explore.measures?.length && exploreState.visibleMeasures.length === 0) { @@ -140,10 +179,15 @@ function syncDimensions(explore: V1ExploreSpec, exploreState: ExploreState) { dimensionsSet.has(dimension.id) || dimension.type === PivotChipType.Time, ); + const ephemeralMeasureNames = new Set( + exploreState.ephemeralMeasures?.map((def) => def.name) ?? [], + ); + exploreState.pivot.columns = exploreState.pivot.columns.filter( (col) => measuresSet.has(col.id) || dimensionsSet.has(col.id) || + ephemeralMeasureNames.has(col.id) || col.type === PivotChipType.Time, ); @@ -228,6 +272,9 @@ const metricsViewReducers = { sync(name: string, explore: V1ExploreSpec) { if (!name || !explore || !explore.measures) return; updateMetricsExplorerByName(name, (exploreState) => { + // remove ephemeral measures referencing non existent measures + syncEphemeralMeasures(explore, exploreState); + // remove references to non existent measures syncMeasures(explore, exploreState); @@ -641,6 +688,98 @@ const metricsViewReducers = { }); }, + addEphemeralMeasure(name: string, def: EphemeralMeasureDef) { + updateMetricsExplorerByName(name, (exploreState) => { + exploreState.ephemeralMeasures = [ + ...(exploreState.ephemeralMeasures ?? []), + def, + ]; + if (exploreState.activePage === DashboardState_ActivePage.PIVOT) { + exploreState.pivot.rowPage = 1; + exploreState.pivot.activeCell = null; + exploreState.pivot.columns.push({ + id: def.name, + title: def.displayName, + type: PivotChipType.Measure, + }); + } else { + // Make the new measure visible in the explore view. + exploreState.visibleMeasures = [ + ...exploreState.visibleMeasures, + def.name, + ]; + exploreState.allMeasuresVisible = false; + } + }); + }, + + updateEphemeralMeasure(name: string, def: EphemeralMeasureDef) { + updateMetricsExplorerByName(name, (exploreState) => { + exploreState.pivot.rowPage = 1; + exploreState.pivot.activeCell = null; + exploreState.ephemeralMeasures = ( + exploreState.ephemeralMeasures ?? [] + ).map((d) => (d.name === def.name ? def : d)); + // Chip titles are denormalized; keep any placed pivot chip in sync on rename. + exploreState.pivot.columns = exploreState.pivot.columns.map((col) => + col.id === def.name ? { ...col, title: def.displayName } : col, + ); + }); + }, + + removeEphemeralMeasure( + name: string, + measureName: string, + explore: V1ExploreSpec | undefined, + ) { + updateMetricsExplorerByName(name, (exploreState) => { + exploreState.ephemeralMeasures = ( + exploreState.ephemeralMeasures ?? [] + ).filter((d) => d.name !== measureName); + + // Remove all usages across views. + exploreState.pivot.rowPage = 1; + exploreState.pivot.activeCell = null; + exploreState.pivot.columns = exploreState.pivot.columns.filter( + (col) => col.id !== measureName, + ); + exploreState.pivot.sorting = exploreState.pivot.sorting.filter( + (s) => s.id !== measureName, + ); + if (exploreState.pivot.measureFormatting?.[measureName]) { + const measureFormatting = { ...exploreState.pivot.measureFormatting }; + delete measureFormatting[measureName]; + exploreState.pivot.measureFormatting = measureFormatting; + } + + exploreState.visibleMeasures = exploreState.visibleMeasures.filter( + (m) => m !== measureName, + ); + exploreState.leaderboardMeasureNames = ( + exploreState.leaderboardMeasureNames ?? [] + ).filter((m) => m !== measureName); + if (exploreState.leaderboardSortByMeasureName === measureName) { + exploreState.leaderboardSortByMeasureName = + exploreState.leaderboardMeasureNames[0] ?? + exploreState.visibleMeasures[0] ?? + ""; + } + if (exploreState.tdd.expandedMeasureName === measureName) { + exploreState.tdd.expandedMeasureName = undefined; + if ( + exploreState.activePage === + DashboardState_ActivePage.TIME_DIMENSIONAL_DETAIL + ) { + exploreState.activePage = DashboardState_ActivePage.DEFAULT; + } + } + + // The removed measure may have been the only visible or leaderboard + // measure; restore the spec defaults so views never end up measure-less. + if (explore) syncMeasures(explore, exploreState); + }); + }, + setPivotRowLimitForExpandedRow( name: string, expandIndex: string, diff --git a/web-common/src/features/dashboards/stores/explore-state.ts b/web-common/src/features/dashboards/stores/explore-state.ts index f37434e3a8ff..44e4fb60598f 100644 --- a/web-common/src/features/dashboards/stores/explore-state.ts +++ b/web-common/src/features/dashboards/stores/explore-state.ts @@ -1,3 +1,4 @@ +import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; import type { MeasureFilterEntry } from "@rilldata/web-common/features/dashboards/filters/measure-filters/measure-filter-entry"; import { LeaderboardContextColumn, @@ -32,6 +33,15 @@ export interface ExploreState { */ visibleMeasures: string[]; + /** + * Ad-hoc measures derived from existing metrics view measures via an + * arithmetic expression (e.g. Profit = revenue - cost). Shared by all + * views (leaderboards, charts, pivot, ...) and encoded in the `ephemeral` URL + * param. Definitions are independent of usage: an unused definition stays + * available in the measure menus until explicitly deleted. + */ + ephemeralMeasures?: EphemeralMeasureDef[]; + /** * While the `visibleMeasureKeys` has the list of visible measures, * this is explicitly needed to fill the state. diff --git a/web-common/src/features/dashboards/stores/validate-and-clean-explore-state.ts b/web-common/src/features/dashboards/stores/validate-and-clean-explore-state.ts index 4ce33ee46075..7a4c6d7a203c 100644 --- a/web-common/src/features/dashboards/stores/validate-and-clean-explore-state.ts +++ b/web-common/src/features/dashboards/stores/validate-and-clean-explore-state.ts @@ -1,3 +1,7 @@ +import { + injectEphemeralMeasuresIntoMap, + validateEphemeralDefsAgainstSpec, +} from "@rilldata/web-common/features/dashboards/ephemeral-measures/url-state"; import type { ExploreState } from "@rilldata/web-common/features/dashboards/stores/explore-state"; import { getMultiFieldError, @@ -46,6 +50,21 @@ export function validateAndCleanExploreState( (d) => d.name!, ); + // Validate restored ephemeral measures and treat the valid ones as known + // measure names for the validations below. + if (exploreState.ephemeralMeasures !== undefined) { + const { valid, invalidEntries } = validateEphemeralDefsAgainstSpec( + exploreState.ephemeralMeasures, + measures, + metricsViewSpec, + ); + exploreState.ephemeralMeasures = valid.length ? valid : undefined; + injectEphemeralMeasuresIntoMap(measures, valid); + if (invalidEntries.length) { + errors.push(getMultiFieldError("adhoc measure", invalidEntries)); + } + } + const errorsFromExploreView = validateAndCleanExploreViewState( measures, dimensions, @@ -144,8 +163,9 @@ function validateAndCleanMeasureRelatedExploreState( if (selectedMeasures.length > 0) { // If there are any remaining valid measures then set it. - exploreState.allMeasuresVisible = - selectedMeasures.length === exploreSpec.measures?.length; + // `measures` includes ephemeral measures, so a hidden spec measure can + // never be masked by a visible ephemeral one. + exploreState.allMeasuresVisible = selectedMeasures.length === measures.size; exploreState.visibleMeasures = selectedMeasures; } else { // Else remove the relevant fields so that cascading merge can set fields from other sources. diff --git a/web-common/src/features/dashboards/time-dimension-details/TDDHeader.svelte b/web-common/src/features/dashboards/time-dimension-details/TDDHeader.svelte index 2800d0be5c12..71531e532ef2 100644 --- a/web-common/src/features/dashboards/time-dimension-details/TDDHeader.svelte +++ b/web-common/src/features/dashboards/time-dimension-details/TDDHeader.svelte @@ -3,6 +3,7 @@ import Column from "@rilldata/web-common/components/icons/Column.svelte"; import Row from "@rilldata/web-common/components/icons/Row.svelte"; import SearchableFilterChip from "@rilldata/web-common/components/searchable-filter-menu/SearchableFilterChip.svelte"; + import { ephemeralMeasureDialog } from "@rilldata/web-common/features/dashboards/ephemeral-measures/dialog-store"; import { splitPivotChips } from "@rilldata/web-common/features/dashboards/pivot/pivot-utils"; import ReplacePivotDialog from "@rilldata/web-common/features/dashboards/pivot/ReplacePivotDialog.svelte"; import { getStateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; @@ -67,17 +68,25 @@ expressionFilterManager, } = stateManagers; + const ephemeralDefsByName = $derived( + new Map( + ($dashboardStore?.ephemeralMeasures ?? []).map((def) => [def.name, def]), + ), + ); + const selectableMeasures = $derived( $allMeasures .filter((m) => m.name !== undefined || m.displayName !== undefined) - .map((m) => + .map((m) => { + const def = ephemeralDefsByName.get(m.name || ""); // Note: undefined values are filtered out above, so the // empty string fallback is unreachable. - ({ + return { name: m.name || "", label: m.displayName || "", - }), - ), + ...(def ? { description: def.expression, ephemeral: true } : {}), + }; + }), ); const selectedMeasureLabel = $derived( @@ -233,10 +242,20 @@ { + const def = ephemeralDefsByName.get(name); + if (def) ephemeralMeasureDialog.set({ def }); + }} selectableItems={selectableMeasures} selectedItems={[expandedMeasureName]} tooltipText="Choose a measure to display" - /> + > + + {#if ephemeralDefsByName.has(expandedMeasureName)} + ƒx + {/if} + +
diff --git a/web-common/src/features/dashboards/time-dimension-details/charts/TDDChart.svelte b/web-common/src/features/dashboards/time-dimension-details/charts/TDDChart.svelte index 60b659087167..38a76962ed4a 100644 --- a/web-common/src/features/dashboards/time-dimension-details/charts/TDDChart.svelte +++ b/web-common/src/features/dashboards/time-dimension-details/charts/TDDChart.svelte @@ -8,6 +8,7 @@ } from "@rilldata/web-common/features/components/charts/highlight-controller"; import type { ChartProvider } from "@rilldata/web-common/features/components/charts/types"; import { THEME_STORE_CONTEXT_KEY } from "@rilldata/web-common/features/themes/theme-boundary"; + import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; import type { TimeAndFilterStore } from "@rilldata/web-common/features/dashboards/time-controls/time-control-store"; import { chartBrushStore, @@ -37,6 +38,7 @@ export let metricsViewName: string; export let measure: MetricsViewSpecMeasure; + export let ephemeralMeasures: EphemeralMeasureDef[] | undefined = undefined; export let timeDimension: string | undefined = undefined; export let interval: Interval | undefined = undefined; export let comparisonInterval: Interval | undefined = undefined; @@ -80,6 +82,7 @@ dimensionData, showTimeDimensionDetail, dynamicYAxis, + ephemeralMeasures, ); $: componentChartType = TDD_TO_COMPONENT_CHART_TYPE[chartType]; diff --git a/web-common/src/features/dashboards/time-dimension-details/charts/tdd-chart-config.spec.ts b/web-common/src/features/dashboards/time-dimension-details/charts/tdd-chart-config.spec.ts index 22245b0f27ed..e447f1ed7111 100644 --- a/web-common/src/features/dashboards/time-dimension-details/charts/tdd-chart-config.spec.ts +++ b/web-common/src/features/dashboards/time-dimension-details/charts/tdd-chart-config.spec.ts @@ -35,6 +35,61 @@ describe("createTDDCartesianSpec", () => { }); }); + // Comparing by dimension switches the default chart to a Vega stacked bar, + // whose query builds its own measures. Without the definition the runtime + // rejects the bare name with `measure "..." not found`. + it("carries the ephemeral measure definition into the chart spec", () => { + const ephemeralMeasures = [ + { + name: "sales_per_order", + displayName: "Sales per order", + expression: "total_sales / orders", + }, + ]; + + const spec = createTDDCartesianSpec( + "my_metrics_view", + "sales_per_order", + "timestamp", + "country", + ["US"], + undefined, + true, + false, + ephemeralMeasures, + ); + + expect(spec.adhoc_measures).toEqual([ + { + name: "sales_per_order", + display_name: "Sales per order", + expression: "total_sales / orders", + }, + ]); + }); + + it("omits adhoc_measures for a spec measure", () => { + const spec = createTDDCartesianSpec( + "my_metrics_view", + "total_sales", + "timestamp", + "country", + ["US"], + undefined, + true, + false, + [ + { + name: "sales_per_order", + displayName: "Sales per order", + expression: "total_sales / orders", + }, + ], + ); + + expect(spec.adhoc_measures).toBeUndefined(); + }); + it("includes colorMapping from dimensionData when available", () => { const dimensionData = [ { dimensionValue: "US", color: "#ff0000", isFetching: false, data: [] }, diff --git a/web-common/src/features/dashboards/time-dimension-details/charts/tdd-chart-config.ts b/web-common/src/features/dashboards/time-dimension-details/charts/tdd-chart-config.ts index c17a9cccb39d..e435591466bc 100644 --- a/web-common/src/features/dashboards/time-dimension-details/charts/tdd-chart-config.ts +++ b/web-common/src/features/dashboards/time-dimension-details/charts/tdd-chart-config.ts @@ -4,6 +4,8 @@ import type { ChartType, ColorMapping, } from "@rilldata/web-common/features/components/charts/types"; +import { ephemeralDefsToSpecs } from "@rilldata/web-common/features/dashboards/ephemeral-measures/canvas"; +import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; import type { DimensionSeriesData } from "@rilldata/web-common/features/dashboards/time-series/measure-chart/types"; import { TDDChart } from "../types"; @@ -39,6 +41,7 @@ export function createTDDCartesianSpec( dimensionData?: DimensionSeriesData[], showTimeDimensionDetail = true, dynamicYAxis = false, + ephemeralMeasures?: EphemeralMeasureDef[], ): CartesianChartSpec & Pick { const spec: CartesianChartSpec & Pick = { metrics_view: metricsViewName, @@ -54,6 +57,11 @@ export function createTDDCartesianSpec( // The "Dynamic Y-axis scale" toggle: off means the axis is anchored at zero. zeroBasedOrigin: !dynamicYAxis, }, + // An ephemeral measure has no spec entry, so the chart's query has to carry + // its definition; without this the runtime rejects the bare measure name. + ...(ephemeralMeasures?.some((def) => def.name === measureName) + ? { adhoc_measures: ephemeralDefsToSpecs(ephemeralMeasures) } + : {}), isInteractive: true, // Fix vertical alignment across stacked TDD charts: force a fixed axis // width so every measure's plot area is identical regardless of label width. diff --git a/web-common/src/features/dashboards/time-dimension-details/tdd-export.ts b/web-common/src/features/dashboards/time-dimension-details/tdd-export.ts index bbc379d353f0..cd3101cf79b3 100644 --- a/web-common/src/features/dashboards/time-dimension-details/tdd-export.ts +++ b/web-common/src/features/dashboards/time-dimension-details/tdd-export.ts @@ -1,3 +1,4 @@ +import { mapEphemeralMeasuresForRequest } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; import { SortDirection } from "@rilldata/web-common/features/dashboards/proto-state/derived-types"; import type { StateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; import type { ExploreState } from "@rilldata/web-common/features/dashboards/stores/explore-state"; @@ -86,9 +87,11 @@ export function getTDDAggregationRequest({ } if (!timeRange) return undefined; - const measures: V1MetricsViewAggregationMeasure[] = [ - { name: exploreState.tdd.expandedMeasureName }, - ]; + const measures: V1MetricsViewAggregationMeasure[] = + mapEphemeralMeasuresForRequest( + [{ name: exploreState.tdd.expandedMeasureName }], + exploreState.ephemeralMeasures, + ); // CAST SAFETY: exports are only available in TDD when a comparison dimension is selected const dimensionName = exploreState.selectedComparisonDimension as string; diff --git a/web-common/src/features/dashboards/time-dimension-details/time-dimension-data-store.ts b/web-common/src/features/dashboards/time-dimension-details/time-dimension-data-store.ts index cfce38c95031..6fc81c3598c3 100644 --- a/web-common/src/features/dashboards/time-dimension-details/time-dimension-data-store.ts +++ b/web-common/src/features/dashboards/time-dimension-details/time-dimension-data-store.ts @@ -1,3 +1,4 @@ +import { ephemeralMeasureToSpecMeasure } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; import { createSparkline } from "@rilldata/web-common/components/data-graphic/marks/sparkline"; import { useSelectedValuesForCompareDimension } from "@rilldata/web-common/features/dashboards/state-managers/selectors/dimension-filters"; import type { StateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; @@ -407,9 +408,14 @@ export function createTimeDimensionDataStore( const isAllTime = timeControls?.selectedTimeRange?.name === TimeRangePreset.ALL_TIME; - const measure = validSpec.data?.metricsView?.measures?.find( - (m) => m.name === measureName, - ); + const measure = + validSpec.data?.metricsView?.measures?.find( + (m) => m.name === measureName, + ) ?? + // ephemeral measures have no spec entry; synthesize one. + dashboardStore?.ephemeralMeasures + ?.filter((def) => def.name === measureName) + .map(ephemeralMeasureToSpecMeasure)[0]; let comparing; let data: TableData | undefined = undefined; diff --git a/web-common/src/features/dashboards/time-series/MetricsTimeSeriesCharts.svelte b/web-common/src/features/dashboards/time-series/MetricsTimeSeriesCharts.svelte index 39cb6c66727a..1acae1746889 100644 --- a/web-common/src/features/dashboards/time-series/MetricsTimeSeriesCharts.svelte +++ b/web-common/src/features/dashboards/time-series/MetricsTimeSeriesCharts.svelte @@ -3,6 +3,9 @@ import CaretDownIcon from "@rilldata/web-common/components/icons/CaretDownIcon.svelte"; import { m } from "@rilldata/web-common/lib/i18n/gen/messages"; import DashboardMetricsDraggableList from "@rilldata/web-common/components/menu/DashboardMetricsDraggableList.svelte"; + import CreateEphemeralMeasureButton from "@rilldata/web-common/features/dashboards/ephemeral-measures/CreateEphemeralMeasureButton.svelte"; + import { ephemeralMeasureDialog } from "@rilldata/web-common/features/dashboards/ephemeral-measures/dialog-store"; + import { ephemeralMeasureNameSet } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; import ReplacePivotDialog from "@rilldata/web-common/features/dashboards/pivot/ReplacePivotDialog.svelte"; import { splitPivotChips } from "@rilldata/web-common/features/dashboards/pivot/pivot-utils"; import { @@ -198,6 +201,13 @@ let allMeasureNames = $derived( $allMeasures.map(({ name }) => name).filter(isDefined), ); + + function openEphemeralMeasureEditor(name: string) { + const def = $dashboardStore?.ephemeralMeasures?.find( + (d) => d.name === name, + ); + if (def) ephemeralMeasureDialog.set({ def }); + } function isDefined(value: string | undefined): value is string { return value !== undefined; } @@ -328,7 +338,15 @@ allItems={$allMeasures} tagIndex={$measureTagIndex} selectedItems={visibleMeasureNames} - /> + ephemeralNames={ephemeralMeasureNameSet( + $dashboardStore?.ephemeralMeasures, + )} + onEditEphemeral={openEphemeralMeasureEditor} + > +
+ +
+ {#if $rillTime && activeTimeGrain} @@ -426,6 +444,7 @@ {#each renderedMeasures as measure (measure.name)} + import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; import { Button } from "@rilldata/web-common/components/button"; import * as Dialog from "@rilldata/web-common/components/dialog"; import { TDDChart } from "@rilldata/web-common/features/dashboards/time-dimension-details/types"; @@ -21,6 +22,7 @@ export let open = false; export let measure: MetricsViewSpecMeasure; + export let ephemeralMeasures: EphemeralMeasureDef[] | undefined = undefined; export let metricsViewName: string; export let where: V1Expression | undefined = undefined; export let expressionFilterManager: ExpressionFilterManager; @@ -137,6 +139,7 @@ + import { splitTimeSeriesMeasures } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; + import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; import InlineErrorIndicator from "@rilldata/web-common/features/dashboards/errors/InlineErrorIndicator.svelte"; import TDDMeasureChart from "@rilldata/web-common/features/dashboards/time-dimension-details/charts/TDDChart.svelte"; import { TDDChart } from "@rilldata/web-common/features/dashboards/time-dimension-details/types"; @@ -33,6 +35,7 @@ const VISIBILITY_ROOT_MARGIN = "120px"; export let measure: MetricsViewSpecMeasure; + export let ephemeralMeasures: EphemeralMeasureDef[] | undefined = undefined; export let metricsViewName: string; export let where: V1Expression | undefined = undefined; export let timeDimension: string | undefined = undefined; @@ -83,6 +86,8 @@ }); $: measureName = measure.name ?? ""; + $: ({ measureNames: tsMeasureNames, ephemeralMeasures: tsEphemeralMeasures } = + splitTimeSeriesMeasures([measureName], ephemeralMeasures)); $: height = showTimeDimensionDetail ? tddChartHeight : 145; $: effectiveChartType = resolveEffectiveChartType( @@ -124,7 +129,8 @@ client, { metricsViewName, - measureNames: [measureName], + measureNames: tsMeasureNames, + ephemeralMeasures: tsEphemeralMeasures, where, timeDimension, timeStart, @@ -145,7 +151,8 @@ client, { metricsViewName, - measureNames: [measureName], + measureNames: tsMeasureNames, + ephemeralMeasures: tsEphemeralMeasures, where, timeDimension, timeStart: comparisonTimeStart, @@ -190,6 +197,7 @@ client, metricsViewName, measureName, + ephemeralMeasures, comparisonDimension!, dimensionValues, dimensionWhere, @@ -208,6 +216,7 @@ client, metricsViewName, measureName, + ephemeralMeasures, comparisonDimension!, dimensionValues, dimensionWhere, @@ -332,6 +341,7 @@ chartType={effectiveChartType} {metricsViewName} {measure} + {ephemeralMeasures} {timeDimension} {interval} comparisonInterval={showComparison ? comparisonInterval : undefined} diff --git a/web-common/src/features/dashboards/time-series/measure-chart/use-dimension-data.ts b/web-common/src/features/dashboards/time-series/measure-chart/use-dimension-data.ts index 9383b5afb4ca..914f6cc9d4d6 100644 --- a/web-common/src/features/dashboards/time-series/measure-chart/use-dimension-data.ts +++ b/web-common/src/features/dashboards/time-series/measure-chart/use-dimension-data.ts @@ -1,3 +1,5 @@ +import { mapEphemeralMeasuresForRequest } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; +import type { EphemeralMeasureDef } from "@rilldata/web-common/features/dashboards/ephemeral-measures/types"; import { createAndExpression, createInExpression, @@ -33,6 +35,7 @@ export function createDimensionAggregationQuery( client: RuntimeClient, metricsViewName: string, measureName: string, + ephemeralMeasures: EphemeralMeasureDef[] | undefined, dimensionName: string, dimensionValues: (string | null)[], where: V1Expression | undefined, @@ -55,7 +58,10 @@ export function createDimensionAggregationQuery( client, { metricsView: metricsViewName, - measures: [{ name: measureName }], + measures: mapEphemeralMeasuresForRequest( + [{ name: measureName }], + ephemeralMeasures, + ), dimensions: [ { name: dimensionName }, { name: timeDimension, timeGrain: timeGranularity as any, timeZone }, diff --git a/web-common/src/features/dashboards/time-series/multiple-dimension-queries.ts b/web-common/src/features/dashboards/time-series/multiple-dimension-queries.ts index 705e01b4b2ff..8b4ef6aea750 100644 --- a/web-common/src/features/dashboards/time-series/multiple-dimension-queries.ts +++ b/web-common/src/features/dashboards/time-series/multiple-dimension-queries.ts @@ -1,3 +1,4 @@ +import { mapEphemeralMeasuresForRequest } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; import { getURIRequestMeasure, URI_DIMENSION_SUFFIX, @@ -136,9 +137,11 @@ export function getDimensionValuesForComparison( const hasUri = !!validSpec?.data?.metricsView?.dimensions?.find( (d) => d.name === dimensionName, )?.uri; - const tddMeasures: V1MetricsViewAggregationMeasure[] = measures.map( - (measure) => ({ name: measure }), - ); + const tddMeasures: V1MetricsViewAggregationMeasure[] = + mapEphemeralMeasuresForRequest( + measures.map((measure) => ({ name: measure })), + dashboardStore.ephemeralMeasures, + ); if (hasUri) { tddMeasures.push(getURIRequestMeasure(dimensionName)); } @@ -288,7 +291,10 @@ function getAggregationQueryForTopList( ctx.runtimeClient, { metricsView: metricsViewName, - measures: measures.map((measure) => ({ name: measure })), + measures: mapEphemeralMeasuresForRequest( + measures.map((measure) => ({ name: measure })), + dashboardStore.ephemeralMeasures, + ), dimensions: [ { name: dimensionName }, { name: timeDimension, timeGrain, timeZone }, diff --git a/web-common/src/features/dashboards/time-series/timeseries-data-store.ts b/web-common/src/features/dashboards/time-series/timeseries-data-store.ts index 48a8dd21b26c..2bc7e8837f99 100644 --- a/web-common/src/features/dashboards/time-series/timeseries-data-store.ts +++ b/web-common/src/features/dashboards/time-series/timeseries-data-store.ts @@ -1,3 +1,7 @@ +import { + ephemeralMeasureNameSet, + splitTimeSeriesMeasures, +} from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; import { filterOutSomeAdvancedMeasures } from "@rilldata/web-common/features/dashboards/state-managers/selectors/measures"; import type { StateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; import { sanitiseExpression } from "@rilldata/web-common/features/dashboards/stores/filter-utils"; @@ -59,11 +63,17 @@ export function createMetricsViewTimeSeries( ([metricsViewName, dashboardStore, timeControls], set) => { const timeGrain = timeControls.selectedTimeRange?.interval; + // Ephemeral measures travel in `measures` with their expression; + // regular measures stay in `measureNames`. + const { measureNames, ephemeralMeasures: ephemeralRequestMeasures } = + splitTimeSeriesMeasures(measures, dashboardStore.ephemeralMeasures); + return createQueryServiceMetricsViewTimeSeries( ctx.runtimeClient, { metricsViewName, - measureNames: measures, + measureNames, + ephemeralMeasures: ephemeralRequestMeasures, where: sanitiseExpression(dashboardStore.whereFilter, undefined), timeStart: isComparison ? timeControls.comparisonAdjustedStart @@ -122,7 +132,10 @@ export function createTimeSeriesDataStore( ); const expandedMeasuerName = dashboardStore?.tdd?.expandedMeasureName; if (showTimeDimensionDetail && expandedMeasuerName) { - measures = allMeasures.filter( + const ephemeralMeasureNames = ephemeralMeasureNameSet( + dashboardStore?.ephemeralMeasures, + ); + measures = [...allMeasures, ...ephemeralMeasureNames].filter( (measure) => measure === expandedMeasuerName, ); } else { diff --git a/web-common/src/features/dashboards/time-series/totals-data-store.ts b/web-common/src/features/dashboards/time-series/totals-data-store.ts index bc7269f04647..0b677d741908 100644 --- a/web-common/src/features/dashboards/time-series/totals-data-store.ts +++ b/web-common/src/features/dashboards/time-series/totals-data-store.ts @@ -1,3 +1,4 @@ +import { mapEphemeralMeasuresForRequest } from "@rilldata/web-common/features/dashboards/ephemeral-measures/measure-mapping"; import type { StateManagers } from "@rilldata/web-common/features/dashboards/state-managers/state-managers"; import { sanitiseExpression } from "@rilldata/web-common/features/dashboards/stores/filter-utils"; import { useTimeControlStore } from "@rilldata/web-common/features/dashboards/time-controls/time-control-store"; @@ -19,7 +20,10 @@ export function createTotalsForMeasure( ctx.runtimeClient, { metricsView: metricsViewName, - measures: measures.map((measure) => ({ name: measure })), + measures: mapEphemeralMeasuresForRequest( + measures.map((measure) => ({ name: measure })), + dashboard.ephemeralMeasures, + ), where: sanitiseExpression(dashboard.whereFilter, undefined), timeRange: { start: isComparison @@ -58,7 +62,10 @@ export function createUnfilteredTotalsForMeasure( ctx.runtimeClient, { metricsView: metricsViewName, - measures: measures.map((measure) => ({ name: measure })), + measures: mapEphemeralMeasuresForRequest( + measures.map((measure) => ({ name: measure })), + dashboard.ephemeralMeasures, + ), where: updatedFilter, timeRange: { start: timeControls.timeStart, diff --git a/web-common/src/features/dashboards/url-state/convert-partial-explore-state-to-url-params.ts b/web-common/src/features/dashboards/url-state/convert-partial-explore-state-to-url-params.ts index a6f4e5f1d221..26e8c970d37a 100644 --- a/web-common/src/features/dashboards/url-state/convert-partial-explore-state-to-url-params.ts +++ b/web-common/src/features/dashboards/url-state/convert-partial-explore-state-to-url-params.ts @@ -1,3 +1,5 @@ +import { toEphemeralMeasuresParam } from "@rilldata/web-common/features/dashboards/ephemeral-measures/url-param"; +import { referencedEphemeralMeasures } from "@rilldata/web-common/features/dashboards/ephemeral-measures/url-state"; import { toPivotFormattingParam } from "@rilldata/web-common/features/dashboards/pivot/pivot-formatting-param"; import { type PivotChipData, @@ -119,6 +121,18 @@ export function convertPartialExploreStateToUrlParams( searchParams.set(ExploreStateURLParams.Filters, filterParam); } + if ("ephemeralMeasures" in partialExploreState) { + // Only definitions the state references go into the URL; unused ones live + // in the per-metrics-view library. Always set so deleting or hiding the + // last one removes it from the URL; cleanUrlParams strips the empty value. + searchParams.set( + ExploreStateURLParams.EphemeralMeasures, + toEphemeralMeasuresParam( + referencedEphemeralMeasures(partialExploreState), + ), + ); + } + switch (partialExploreState.activePage) { case DashboardState_ActivePage.UNSPECIFIED: case DashboardState_ActivePage.DEFAULT: diff --git a/web-common/src/features/dashboards/url-state/convertLegacyStateToExplorePreset.ts b/web-common/src/features/dashboards/url-state/convertLegacyStateToExplorePreset.ts index 908e10ff19ef..eecad38263c2 100644 --- a/web-common/src/features/dashboards/url-state/convertLegacyStateToExplorePreset.ts +++ b/web-common/src/features/dashboards/url-state/convertLegacyStateToExplorePreset.ts @@ -1,3 +1,8 @@ +import { toEphemeralMeasuresParam } from "@rilldata/web-common/features/dashboards/ephemeral-measures/url-param"; +import { + injectEphemeralMeasuresIntoMap, + validateEphemeralDefsAgainstSpec, +} from "@rilldata/web-common/features/dashboards/ephemeral-measures/url-state"; import { toPivotFormattingParam } from "@rilldata/web-common/features/dashboards/pivot/pivot-formatting-param"; import { FromProtoTimeGrainMap } from "@rilldata/web-common/features/dashboards/proto-state/enum-maps"; import { convertFilterToExpression } from "@rilldata/web-common/features/dashboards/proto-state/filter-converter"; @@ -71,6 +76,24 @@ export function convertLegacyStateToExplorePreset( (d) => d.name!, ); + if (legacyState.ephemeralMeasures?.length) { + const { valid, invalidEntries } = validateEphemeralDefsAgainstSpec( + legacyState.ephemeralMeasures.map((def) => ({ + name: def.name, + displayName: def.displayName, + expression: def.expression, + ...(def.formatPreset ? { formatPreset: def.formatPreset } : {}), + })), + measures, + metricsView, + ); + preset.ephemeralMeasures = toEphemeralMeasuresParam(valid); + injectEphemeralMeasuresIntoMap(measures, valid); + if (invalidEntries.length) { + errors.push(getMultiFieldError("adhoc measure", invalidEntries)); + } + } + if (legacyState.activePage !== DashboardState_ActivePage.UNSPECIFIED) { preset.view = FromActivePageMap[legacyState.activePage]; } diff --git a/web-common/src/features/dashboards/url-state/convertPresetToExploreState.ts b/web-common/src/features/dashboards/url-state/convertPresetToExploreState.ts index bce116eeeb7e..a5153835c815 100644 --- a/web-common/src/features/dashboards/url-state/convertPresetToExploreState.ts +++ b/web-common/src/features/dashboards/url-state/convertPresetToExploreState.ts @@ -1,3 +1,5 @@ +import { fromEphemeralMeasuresParam } from "@rilldata/web-common/features/dashboards/ephemeral-measures/url-param"; +import { injectEphemeralMeasuresIntoMap } from "@rilldata/web-common/features/dashboards/ephemeral-measures/url-state"; import { fromPivotFormattingParam } from "@rilldata/web-common/features/dashboards/pivot/pivot-formatting-param"; import { type PivotChipData, @@ -67,6 +69,19 @@ export function convertPresetToExploreState( (d) => d.name!, ); + // The preset's ephemeral measures were already validated when the preset + // was built (convertURLToExplorePreset); resolve them and make their names + // valid everywhere a measure name is used below. + if (preset.ephemeralMeasures !== undefined) { + const { ephemeralMeasures } = fromEphemeralMeasuresParam( + preset.ephemeralMeasures, + ); + partialExploreState.ephemeralMeasures = ephemeralMeasures.length + ? ephemeralMeasures + : undefined; + injectEphemeralMeasuresIntoMap(measures, ephemeralMeasures); + } + if (preset.view) { partialExploreState.activePage = Number( ToActivePageViewMap[preset.view] ?? "0", @@ -246,8 +261,10 @@ function fromExploreUrlParams( errors.push(getMultiFieldError("measure", missingMeasures)); } + // `measures` includes ephemeral measures, so a hidden spec measure can + // never be masked by a visible ephemeral one. partialExploreState.allMeasuresVisible = - selectedMeasures.length === explore.measures?.length; + selectedMeasures.length === measures.size; partialExploreState.visibleMeasures = [...selectedMeasures]; } diff --git a/web-common/src/features/dashboards/url-state/convertURLToExplorePreset.ts b/web-common/src/features/dashboards/url-state/convertURLToExplorePreset.ts index d4eb26a641ab..baecc8961a26 100644 --- a/web-common/src/features/dashboards/url-state/convertURLToExplorePreset.ts +++ b/web-common/src/features/dashboards/url-state/convertURLToExplorePreset.ts @@ -1,4 +1,9 @@ import { stripMeasureSuffix } from "@rilldata/web-common/features/dashboards/filters/measure-filters/measure-filter-entry"; +import { toEphemeralMeasuresParam } from "@rilldata/web-common/features/dashboards/ephemeral-measures/url-param"; +import { + injectEphemeralMeasuresIntoMap, + parseAndValidateEphemeralParam, +} from "@rilldata/web-common/features/dashboards/ephemeral-measures/url-state"; import { PIVOT_ROW_LIMIT_OPTIONS } from "@rilldata/web-common/features/dashboards/pivot/pivot-constants"; import { fromPivotFormattingParam, @@ -103,6 +108,26 @@ export function convertURLToExplorePreset( errors.push(...errorsFromLegacyState); } + // Parse ephemeral measures before any other param: their names are valid + // measure names everywhere (visible measures, leaderboards, sort, pivot + // columns, formatting, ...), which is achieved by injecting synthetic spec + // measures into the `measures` map used by all validations below. + if (searchParams.has(ExploreStateURLParams.EphemeralMeasures)) { + const ephemeralParam = searchParams.get( + ExploreStateURLParams.EphemeralMeasures, + ) as string; + const { valid, invalidEntries } = parseAndValidateEphemeralParam( + ephemeralParam, + measures, + metricsView, + ); + preset.ephemeralMeasures = toEphemeralMeasuresParam(valid); + injectEphemeralMeasuresIntoMap(measures, valid); + if (invalidEntries.length) { + errors.push(getMultiFieldError("adhoc measure", invalidEntries)); + } + } + if (searchParams.has(ExploreStateURLParams.WebView)) { const view = searchParams.get(ExploreStateURLParams.WebView) as string; if (view in FromURLParamViewMap) { diff --git a/web-common/src/features/dashboards/url-state/url-params.ts b/web-common/src/features/dashboards/url-state/url-params.ts index 4971206d9e88..878c748f1234 100644 --- a/web-common/src/features/dashboards/url-state/url-params.ts +++ b/web-common/src/features/dashboards/url-state/url-params.ts @@ -34,6 +34,7 @@ export enum ExploreStateURLParams { PivotShowTotalsColumn = "show_totals_column", PivotShowTotalsRow = "show_totals_row", PivotFormatting = "format", + EphemeralMeasures = "ephemeral", DynamicYAxisScale = "dyn_y", diff --git a/web-common/src/features/dashboards/workspace/Dashboard.svelte b/web-common/src/features/dashboards/workspace/Dashboard.svelte index 1c047daca0bd..de679c7dd53a 100644 --- a/web-common/src/features/dashboards/workspace/Dashboard.svelte +++ b/web-common/src/features/dashboards/workspace/Dashboard.svelte @@ -5,6 +5,8 @@ extractErrorStatusCode, isNotFoundError, } from "@rilldata/web-common/lib/errors"; + import EphemeralMeasureDialog from "@rilldata/web-common/features/dashboards/ephemeral-measures/EphemeralMeasureDialog.svelte"; + import { ephemeralMeasureDialog } from "@rilldata/web-common/features/dashboards/ephemeral-measures/dialog-store"; import PivotDisplay from "@rilldata/web-common/features/dashboards/pivot/PivotDisplay.svelte"; import TabBar from "@rilldata/web-common/features/dashboards/tab-bar/TabBar.svelte"; import { useExploreValidSpec } from "@rilldata/web-common/features/explores/selectors"; @@ -297,6 +299,10 @@ {/if} + + {#if $ephemeralMeasureDialog} + + {/if}