Preserve integer precision and object key order in fromjson

builtins.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"math"
 	"sort"
 	"strconv"
@@ -488,12 +489,16 @@ func filterFromjson(args []Value, kwargs map[string]Value) (Value, error) {
 		return Undefined(), nil
 	}
 
-	var result any
-	if err := json.Unmarshal([]byte(args[0].AsString()), &result); err != nil {
+	dec := json.NewDecoder(strings.NewReader(args[0].AsString()))
+	dec.UseNumber()
+	result, err := decodeJSONValue(dec)
+	if err != nil {
 		return Undefined(), fmt.Errorf("fromjson: %w", err)
 	}
-
-	return FromGoValue(result), nil
+	if _, err := dec.Token(); err != io.EOF {
+		return Undefined(), fmt.Errorf("fromjson: trailing data")
+	}
+	return result, nil
 }
 
 func filterItems(args []Value, kwargs map[string]Value) (Value, error) {

jinjafix_regression_test.go

@@ -0,0 +1,141 @@
+package jinja
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"runtime/debug"
+	"testing"
+	"time"
+)
+
+type jinjaFixResult struct {
+	ID     string `json:"id"`
+	Stage  string `json:"stage"`
+	Output string `json:"output,omitempty"`
+	Error  string `json:"error,omitempty"`
+}
+
+func jinjaFixEvaluate(source string) (r jinjaFixResult) {
+	defer func() {
+		if p := recover(); p != nil {
+			r = jinjaFixResult{Stage: "panic", Error: fmt.Sprint(p)}
+		}
+	}()
+	template, err := Compile(source)
+	if err != nil {
+		return jinjaFixResult{Stage: "compile_error", Error: err.Error()}
+	}
+	output, err := template.Render(nil)
+	if err != nil {
+		return jinjaFixResult{Stage: "render_error", Error: err.Error()}
+	}
+	return jinjaFixResult{Stage: "render", Output: output}
+}
+
+// Child processes bound nontermination and stack exhaustion in broken revisions.
+func TestJinjaFixWorker(t *testing.T) {
+	if os.Getenv("JINJAFIX_WORKER") != "1" {
+		return
+	}
+	debug.SetMaxStack(2 << 20)
+	source, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		os.Exit(2)
+	}
+	if json.NewEncoder(os.Stdout).Encode(jinjaFixEvaluate(string(source))) != nil {
+		os.Exit(2)
+	}
+	os.Exit(0)
+}
+
+func jinjaFixReferenceMissing(found bool) bool {
+	return !found
+}
+
+func TestJinjaFixRegression(t *testing.T) {
+	var cases []struct {
+		ID      string `json:"id"`
+		Source  string `json:"source"`
+		GoStage string `json:"go_stage,omitempty"`
+		Reason  string `json:"reason,omitempty"`
+	}
+	var python struct {
+		Cases []jinjaFixResult `json:"cases"`
+	}
+	read := func(path string, v any) {
+		b, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err = json.Unmarshal(b, v); err != nil {
+			t.Fatal(err)
+		}
+	}
+	read("testdata/jinjafix/cases.json", &cases)
+	read("testdata/jinjafix/python-result.json", &python)
+	expected := map[string]jinjaFixResult{}
+	for _, r := range python.Cases {
+		expected[r.ID] = r
+	}
+	var results []jinjaFixResult
+	for _, c := range cases {
+		t.Run(c.ID, func(t *testing.T) {
+			want, ok := expected[c.ID]
+			if jinjaFixReferenceMissing(ok) {
+				t.Fatal("missing Python result")
+			}
+			if c.GoStage != "" {
+				if c.Reason == "" {
+					t.Fatal("undocumented parity exception")
+				}
+				want.Stage = c.GoStage
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestJinjaFixWorker$")
+			cmd.Env = append(os.Environ(), "JINJAFIX_WORKER=1", "GOMEMLIMIT=64MiB", "GOTRACEBACK=none")
+			cmd.Stdin = bytes.NewBufferString(c.Source)
+			var stdout, stderr bytes.Buffer
+			cmd.Stdout = &stdout
+			cmd.Stderr = &stderr
+			err := cmd.Run()
+			var got jinjaFixResult
+			switch {
+			case ctx.Err() != nil:
+				got = jinjaFixResult{Stage: "timeout", Error: ctx.Err().Error()}
+			case err != nil:
+				got = jinjaFixResult{Stage: "process_error", Error: err.Error()}
+			default:
+				if err = json.Unmarshal(stdout.Bytes(), &got); err != nil {
+					t.Fatal(err)
+				}
+			}
+			got.ID = c.ID
+			results = append(results, got)
+			if jinjaFixResultsDiffer(got, want) {
+				t.Errorf("Go: stage=%s output=%q; expected: stage=%s output=%q; error=%s", got.Stage, got.Output, want.Stage, want.Output, got.Error)
+			}
+		})
+	}
+	if path := os.Getenv("JINJAFIX_RESULT_PATH"); path != "" {
+		b, err := json.MarshalIndent(results, "", "  ")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err = os.WriteFile(path, append(b, '\n'), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func jinjaFixResultsDiffer(got, want jinjaFixResult) bool {
+	if got.Stage != want.Stage {
+		return true
+	}
+	return want.Stage == "render" && got.Output != want.Output
+}

json_decode.go

@@ -0,0 +1,57 @@
+package jinja
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+func decodeJSONValue(dec *json.Decoder) (Value, error) {
+	token, err := dec.Token()
+	if err != nil {
+		return Undefined(), fmt.Errorf("decode JSON value: %w", err)
+	}
+	if isJSONScalar(token) {
+		return FromGoValue(token), nil
+	}
+	delim := token.(json.Delim)
+	switch delim {
+	case '{':
+		result := NewDict()
+		for dec.More() {
+			key, err := dec.Token()
+			if err != nil {
+				return Undefined(), fmt.Errorf("decode JSON value: %w", err)
+			}
+			item, err := decodeJSONValue(dec)
+			if err != nil {
+				return Undefined(), fmt.Errorf("decode JSON value: %w", err)
+			}
+			result.AsDict().Set(key.(string), item)
+		}
+		_, err := dec.Token()
+		if err != nil {
+			return Undefined(), fmt.Errorf("close JSON object: %w", err)
+		}
+		return result, nil
+	case '[':
+		var items []Value
+		for dec.More() {
+			item, err := decodeJSONValue(dec)
+			if err != nil {
+				return Undefined(), fmt.Errorf("decode JSON value: %w", err)
+			}
+			items = append(items, item)
+		}
+		_, err := dec.Token()
+		if err != nil {
+			return Undefined(), fmt.Errorf("close JSON array: %w", err)
+		}
+		return NewList(items), nil
+	}
+	return Undefined(), fmt.Errorf("unexpected delimiter %q", delim)
+}
+
+func isJSONScalar(token any) bool {
+	_, delimiter := token.(json.Delim)
+	return !delimiter
+}

testdata/jinjafix/README.txt

@@ -0,0 +1,9 @@
+Reproduce the Python reference:
+  python3 testdata/jinjafix/reference.py
+
+Run the shared-input Go regression:
+  go test -count=1 -v -run ^TestJinjaFixRegression$ .
+
+The test commit records Python results and failing Go results from the same
+inputs. The fix commit records passing Go results and the full Go test suite.
+Worker processes bound panics, stack exhaustion and nontermination.

testdata/jinjafix/cases.json

@@ -0,0 +1,14 @@
+[
+  {
+    "id": "B077-01",
+    "source": "{{ ('{\"counter\":9007199254740993}'|fromjson).counter }}",
+    "origin": "testdata/provisioning/5g-network.jinja",
+    "source_line": 1159
+  },
+  {
+    "id": "B077-02",
+    "source": "{% for k in '{\"upf\":1,\"amf\":2}'|fromjson %}{{ k }};{% endfor %}",
+    "origin": "testdata/provisioning/5g-network.jinja",
+    "source_line": 1163
+  }
+]

testdata/jinjafix/go-after.json

@@ -0,0 +1,12 @@
+[
+  {
+    "id": "B077-01",
+    "stage": "render",
+    "output": "9007199254740993"
+  },
+  {
+    "id": "B077-02",
+    "stage": "render",
+    "output": "upf;amf;"
+  }
+]

testdata/jinjafix/go-after.txt

@@ -0,0 +1,8 @@
+=== RUN   TestJinjaFixRegression
+=== RUN   TestJinjaFixRegression/B077-01
+=== RUN   TestJinjaFixRegression/B077-02
+--- PASS: TestJinjaFixRegression (0.01s)
+    --- PASS: TestJinjaFixRegression/B077-01 (0.00s)
+    --- PASS: TestJinjaFixRegression/B077-02 (0.00s)
+PASS
+ok  	github.com/ardanlabs/jinja	0.012s

testdata/jinjafix/go-before.json

@@ -0,0 +1,12 @@
+[
+  {
+    "id": "B077-01",
+    "stage": "render",
+    "output": "9.007199254740992e+15"
+  },
+  {
+    "id": "B077-02",
+    "stage": "render",
+    "output": "amf;upf;"
+  }
+]

testdata/jinjafix/go-before.txt

@@ -0,0 +1,11 @@
+=== RUN   TestJinjaFixRegression
+=== RUN   TestJinjaFixRegression/B077-01
+    jinjafix_regression_test.go:121: Go: stage=render output="9.007199254740992e+15"; expected: stage=render output="9007199254740993"; error=
+=== RUN   TestJinjaFixRegression/B077-02
+    jinjafix_regression_test.go:121: Go: stage=render output="amf;upf;"; expected: stage=render output="upf;amf;"; error=
+--- FAIL: TestJinjaFixRegression (0.01s)
+    --- FAIL: TestJinjaFixRegression/B077-01 (0.00s)
+    --- FAIL: TestJinjaFixRegression/B077-02 (0.00s)
+FAIL
+FAIL	github.com/ardanlabs/jinja	0.013s
+FAIL

testdata/jinjafix/go-suite-after.txt

@@ -0,0 +1,4 @@
+ok  	github.com/ardanlabs/jinja	9.369s
+?   	github.com/ardanlabs/jinja/examples/basic	[no test files]
+?   	github.com/ardanlabs/jinja/examples/chat	[no test files]
+?   	github.com/ardanlabs/jinja/examples/toolcall	[no test files]

testdata/jinjafix/provenance.json

@@ -0,0 +1,9 @@
+{
+  "base_commit": "b1bf3a06adcade12b59a1dfaa2edef6f4a7d6e09",
+  "selected_fix_commit": "7c15f58971cfbd74af6b7ab1d3f49d69e5ae5103",
+  "original_branch": "cd-fix-json-decoding",
+  "corpus_sha256": "e844754b9724b93fd2fc162d3d756cca40a57d6ed07188b02d02b2bcbe834bb9",
+  "comparison": "Exact successful output; compile/render error stage agreement. Error strings are recorded, not equated.",
+  "previous_test_commit": "774e2c1d97f351ef8e909dc3612c1c137261b085",
+  "previous_fix_commit": "820f77de317489a87678a16526c26b305049a4d1"
+}

testdata/jinjafix/python-result.json

@@ -0,0 +1,24 @@
+{
+  "python": "3.14.4",
+  "jinja2": "3.1.6",
+  "settings": {
+    "trim_blocks": true,
+    "lstrip_blocks": true,
+    "keep_trailing_newline": true
+  },
+  "extensions": [
+    "fromjson=json.loads"
+  ],
+  "cases": [
+    {
+      "id": "B077-01",
+      "stage": "render",
+      "output": "9007199254740993"
+    },
+    {
+      "id": "B077-02",
+      "stage": "render",
+      "output": "upf;amf;"
+    }
+  ]
+}

testdata/jinjafix/reference.py

@@ -0,0 +1,42 @@
+"""Recompute Python results for the exact inputs used by the Go unit test."""
+import json
+import pathlib
+import platform
+import sys
+
+import jinja2
+
+
+def evaluate(source):
+    env = jinja2.Environment(trim_blocks=True, lstrip_blocks=True,
+                             keep_trailing_newline=True)
+    # fromjson is a repository extension; tojson remains stock Jinja2.
+    env.filters['fromjson'] = json.loads
+    try:
+        template = env.from_string(source)
+    except Exception as error:
+        return dict(stage='compile_error', error=str(error), exception=type(error).__name__)
+    try:
+        return dict(stage='render', output=template.render())
+    except Exception as error:
+        return dict(stage='render_error', error=str(error), exception=type(error).__name__)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) > 1 and sys.argv[1] == '--worker':
+        print(json.dumps(evaluate(sys.stdin.read()), ensure_ascii=False))
+    else:
+        import subprocess
+        here = pathlib.Path(__file__).resolve().parent
+        cases = json.loads((here / 'cases.json').read_text())
+        results = []
+        for case in cases:
+            child = subprocess.run([sys.executable, __file__, '--worker'],
+                                   input=case['source'], capture_output=True,
+                                   text=True, timeout=5, check=True)
+            results.append(dict(id=case['id'], **json.loads(child.stdout)))
+        print(json.dumps(dict(python=platform.python_version(), jinja2=jinja2.__version__,
+                              settings=dict(trim_blocks=True, lstrip_blocks=True,
+                                            keep_trailing_newline=True),
+                              extensions=['fromjson=json.loads'], cases=results),
+                         ensure_ascii=False, indent=2))

testdata/jinjafix/requirements.txt

@@ -0,0 +1 @@
+Jinja2==3.1.6