Skip to content

Commit 494b969

Browse files
He-PinHe-Pin
committed
fix: detect recursive deep array traversals
Motivation: Recursive arrays in std.deepJoin and std.flattenDeepArray could hang, and the optimized nested deepJoin path left an unused private helper that failed Scala 2.12 compilation. Modification: Add cross-platform active-array tracking for deep array traversals, route flattenDeepArray/deepJoin through the shared traversal helper, preserve flat fast paths, and remove the stale unused nested deepJoin helper. Result: Recursive deep array traversal now fails fast, shared acyclic arrays still evaluate, and the PR CI matrix compiles/tests locally except Graal, which the user explicitly skipped. References: PR #1041
1 parent bf7a9e5 commit 494b969

13 files changed

Lines changed: 426 additions & 29 deletions

‎sjsonnet/src-js/sjsonnet/stdlib/DeepArrayActiveSet.scala‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package sjsonnet.stdlib
2+
3+
import sjsonnet.Val
4+
5+
private[stdlib] final class DeepArrayActiveSet(initialCapacity: Int) {
6+
private var keys = new Array[Val.Arr](tableSizeFor(initialCapacity))
7+
private var size = 0
8+
private var growAt = keys.length >> 1
9+
10+
@inline def add(arr: Val.Arr): Boolean = {
11+
val current = keys
12+
val mask = current.length - 1
13+
var slot = slotOf(arr, mask)
14+
while (true) {
15+
val key = current(slot)
16+
if (key eq null) {
17+
current(slot) = arr
18+
size += 1
19+
if (size >= growAt) grow()
20+
return true
21+
}
22+
if (key eq arr) return false
23+
slot = (slot + 1) & mask
24+
}
25+
false
26+
}
27+
28+
@inline def remove(arr: Val.Arr): Unit = {
29+
val current = keys
30+
val mask = current.length - 1
31+
var slot = slotOf(arr, mask)
32+
while (true) {
33+
val key = current(slot)
34+
if (key eq null) return
35+
if (key eq arr) {
36+
removeAt(slot)
37+
return
38+
}
39+
slot = (slot + 1) & mask
40+
}
41+
}
42+
43+
private def grow(): Unit = {
44+
val old = keys
45+
val next = new Array[Val.Arr](old.length << 1)
46+
keys = next
47+
growAt = next.length >> 1
48+
var i = 0
49+
while (i < old.length) {
50+
val key = old(i)
51+
if (key ne null) insertExisting(next, key)
52+
i += 1
53+
}
54+
}
55+
56+
private def removeAt(slot0: Int): Unit = {
57+
val current = keys
58+
val mask = current.length - 1
59+
var slot = slot0
60+
var next = (slot + 1) & mask
61+
while (current(next) ne null) {
62+
val key = current(next)
63+
val home = slotOf(key, mask)
64+
if (((next - home) & mask) >= ((slot - home) & mask)) {
65+
current(slot) = key
66+
slot = next
67+
}
68+
next = (next + 1) & mask
69+
}
70+
current(slot) = null
71+
size -= 1
72+
}
73+
74+
private def insertExisting(table: Array[Val.Arr], arr: Val.Arr): Unit = {
75+
val mask = table.length - 1
76+
var slot = slotOf(arr, mask)
77+
while (table(slot) ne null) {
78+
slot = (slot + 1) & mask
79+
}
80+
table(slot) = arr
81+
}
82+
83+
@inline private def slotOf(arr: Val.Arr, mask: Int): Int = {
84+
var h = System.identityHashCode(arr)
85+
h ^= h >>> 16
86+
h & mask
87+
}
88+
89+
private def tableSizeFor(initialCapacity: Int): Int = {
90+
var n = 8
91+
val target = if (initialCapacity <= 0) 8 else initialCapacity << 1
92+
while (n < target) n <<= 1
93+
n
94+
}
95+
}

‎sjsonnet/src-jvm-native/sjsonnet/stdlib/DeepArrayActiveSet.scala‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package sjsonnet.stdlib
2+
3+
import sjsonnet.Val
4+
5+
private[stdlib] final class DeepArrayActiveSet(initialCapacity: Int) {
6+
private val active =
7+
new java.util.IdentityHashMap[Val.Arr, java.lang.Boolean](initialCapacity)
8+
9+
@inline def add(arr: Val.Arr): Boolean =
10+
active.put(arr, java.lang.Boolean.TRUE) eq null
11+
12+
@inline def remove(arr: Val.Arr): Unit =
13+
active.remove(arr)
14+
}

‎sjsonnet/src/sjsonnet/stdlib/ArrayModule.scala‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -531,20 +531,7 @@ object ArrayModule extends AbstractFunctionModule {
531531
val arr = value0.asInstanceOf[Val.Arr]
532532
val out = new mutable.ArrayBuilder.ofRef[Eval]
533533
out.sizeHint(arr.length)
534-
val q = new java.util.ArrayDeque[Eval](arr.length)
535-
var initialIdx = 0
536-
while (initialIdx < arr.length) {
537-
q.add(arr.eval(initialIdx))
538-
initialIdx += 1
539-
}
540-
while (!q.isEmpty) {
541-
q.removeFirst().value match {
542-
case v: Val.Arr =>
543-
var i = v.length - 1
544-
while (i >= 0) { q.push(v.eval(i)); i -= 1 }
545-
case x => out += x
546-
}
547-
}
534+
DeepArrayTraversal.appendFlattened(arr, out, ev.settings.maxMaterializeDepth)
548535
Val.Arr(pos, out.result())
549536
}
550537
}

‎sjsonnet/src/sjsonnet/stdlib/DeepArrayTraversal.scala‎

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
package sjsonnet.stdlib
2+
3+
import sjsonnet._
4+
5+
import scala.collection.mutable
6+
7+
private[stdlib] object DeepArrayTraversal {
8+
private final class ArrFrame(
9+
val arr: Val.Arr,
10+
var index: Int,
11+
val len: Int,
12+
val direct: Array[Eval])
13+
14+
private final class ArrFrameStack(initialCapacity: Int) {
15+
private var frames = new Array[ArrFrame](initialCapacity)
16+
private var depth = 0
17+
18+
@inline def isEmpty: Boolean = depth == 0
19+
20+
@inline def size: Int = depth
21+
22+
@inline def containsArr(arr: Val.Arr): Boolean = {
23+
var i = depth - 1
24+
while (i >= 0) {
25+
if (frames(i).arr eq arr) return true
26+
i -= 1
27+
}
28+
false
29+
}
30+
31+
@inline def peek(): ArrFrame =
32+
frames(depth - 1)
33+
34+
@inline def push(frame: ArrFrame): Unit = {
35+
if (depth == frames.length) grow()
36+
frames(depth) = frame
37+
depth += 1
38+
}
39+
40+
@inline def pop(): ArrFrame = {
41+
depth -= 1
42+
val frame = frames(depth)
43+
frames(depth) = null
44+
frame
45+
}
46+
47+
private def grow(): Unit = {
48+
val old = frames
49+
val next = new Array[ArrFrame](old.length << 1)
50+
System.arraycopy(old, 0, next, 0, old.length)
51+
frames = next
52+
}
53+
}
54+
55+
private final class TraversalState(maxDepth: Int) {
56+
private val stack = new ArrFrameStack(InitialDepthCapacity)
57+
private val active = new DeepArrayActiveSet(InitialDepthCapacity)
58+
59+
@inline def isEmpty: Boolean = stack.isEmpty
60+
61+
@inline def peek(): ArrFrame = stack.peek()
62+
63+
@inline def pushArray(arr: Val.Arr): Unit =
64+
pushArray(arr, 0, arr.length, arr.directBackingArray)
65+
66+
@inline def pushArray(arr: Val.Arr, index: Int, len: Int, direct: Array[Eval]): Unit = {
67+
if (stack.size >= maxDepth) Error.fail(MaxStackFramesExceededMessage)
68+
if (!active.add(arr))
69+
Error.fail(MaxStackFramesExceededMessage)
70+
stack.push(new ArrFrame(arr, index, len, direct))
71+
}
72+
73+
@inline def checkCanEnterArray(arr: Val.Arr): Unit = {
74+
if (stack.size >= maxDepth) Error.fail(MaxStackFramesExceededMessage)
75+
if (stack.containsArr(arr)) Error.fail(MaxStackFramesExceededMessage)
76+
}
77+
78+
@inline def popFrame(frame: ArrFrame): Unit = {
79+
active.remove(frame.arr)
80+
stack.pop()
81+
}
82+
83+
@inline def clearActive(): Unit = {
84+
while (!stack.isEmpty) {
85+
active.remove(stack.pop().arr)
86+
}
87+
}
88+
}
89+
90+
private val InitialDepthCapacity = 4
91+
private val MaxStackFramesExceededMessage = "max stack frames exceeded"
92+
93+
def appendFlattened(root: Val.Arr, out: mutable.ArrayBuilder.ofRef[Eval], maxDepth: Int): Unit = {
94+
if (maxDepth <= 0) Error.fail(MaxStackFramesExceededMessage)
95+
val rootDirect = root.directBackingArray
96+
val rootLen = root.length
97+
var rootIndex = 0
98+
while (rootIndex < rootLen) {
99+
val child =
100+
if (rootDirect ne null) rootDirect(rootIndex).value else root.value(rootIndex)
101+
rootIndex += 1
102+
child match {
103+
case arr: Val.Arr =>
104+
appendFlattenedFromNested(root, rootIndex, rootLen, rootDirect, arr, out, maxDepth)
105+
return
106+
case leaf => out += leaf
107+
}
108+
}
109+
}
110+
111+
private def appendFlattenedFromNested(
112+
root: Val.Arr,
113+
rootIndex: Int,
114+
rootLen: Int,
115+
rootDirect: Array[Eval],
116+
nested: Val.Arr,
117+
out: mutable.ArrayBuilder.ofRef[Eval],
118+
maxDepth: Int): Unit = {
119+
val state = new TraversalState(maxDepth)
120+
try {
121+
state.pushArray(root, rootIndex, rootLen, rootDirect)
122+
state.pushArray(nested)
123+
while (!state.isEmpty) {
124+
val frame = state.peek()
125+
if (frame.index < frame.len) {
126+
val child = {
127+
val d = frame.direct
128+
if (d ne null) d(frame.index).value else frame.arr.value(frame.index)
129+
}
130+
frame.index += 1
131+
child match {
132+
case arr: Val.Arr => state.pushArray(arr)
133+
case leaf => out += leaf
134+
}
135+
} else {
136+
state.popFrame(frame)
137+
}
138+
}
139+
} finally {
140+
state.clearActive()
141+
}
142+
}
143+
144+
def appendDeepJoined(root: Val.Arr, out: StringBuilderWriter, maxDepth: Int): Unit = {
145+
if (maxDepth <= 0) Error.fail(MaxStackFramesExceededMessage)
146+
val rootDirect = root.directBackingArray
147+
val rootLen = root.length
148+
var rootIndex = 0
149+
while (rootIndex < rootLen) {
150+
val child =
151+
if (rootDirect ne null) rootDirect(rootIndex).value else root.value(rootIndex)
152+
rootIndex += 1
153+
child match {
154+
case arr: Val.Arr =>
155+
if (appendDeepJoinedRootNested(root, rootIndex, rootLen, rootDirect, arr, out, maxDepth))
156+
return
157+
case s: Val.Str => out.write(s.str)
158+
case v =>
159+
Error.fail("std.deepJoin: expected string or array, got " + v.prettyName)
160+
}
161+
}
162+
}
163+
164+
private def appendDeepJoinedRootNested(
165+
root: Val.Arr,
166+
rootIndex: Int,
167+
rootLen: Int,
168+
rootDirect: Array[Eval],
169+
initial: Val.Arr,
170+
out: StringBuilderWriter,
171+
maxDepth: Int): Boolean = {
172+
if (maxDepth <= 1) Error.fail(MaxStackFramesExceededMessage)
173+
if (initial eq root) Error.fail(MaxStackFramesExceededMessage)
174+
175+
val direct = initial.directBackingArray
176+
val len = initial.length
177+
var index = 0
178+
var nested: Val.Arr = null
179+
while (index < len && (nested eq null)) {
180+
val child =
181+
if (direct ne null) direct(index).value else initial.value(index)
182+
index += 1
183+
child match {
184+
case arr: Val.Arr => nested = arr
185+
case s: Val.Str => out.write(s.str)
186+
case v =>
187+
Error.fail("std.deepJoin: expected string or array, got " + v.prettyName)
188+
}
189+
}
190+
if (nested eq null) return false
191+
192+
val state = new TraversalState(maxDepth)
193+
try {
194+
state.pushArray(root, rootIndex, rootLen, rootDirect)
195+
state.pushArray(initial, index, len, direct)
196+
appendDeepJoinedNested(state, nested, out)
197+
appendDeepJoinedRemaining(state, out)
198+
} finally {
199+
state.clearActive()
200+
}
201+
true
202+
}
203+
204+
private def appendDeepJoinedRemaining(state: TraversalState, out: StringBuilderWriter): Unit = {
205+
while (!state.isEmpty) {
206+
val frame = state.peek()
207+
if (frame.index < frame.len) {
208+
val child = {
209+
val d = frame.direct
210+
if (d ne null) d(frame.index).value else frame.arr.value(frame.index)
211+
}
212+
frame.index += 1
213+
child match {
214+
case arr: Val.Arr => appendDeepJoinedNested(state, arr, out)
215+
case s: Val.Str => out.write(s.str)
216+
case v =>
217+
Error.fail("std.deepJoin: expected string or array, got " + v.prettyName)
218+
}
219+
} else {
220+
state.popFrame(frame)
221+
}
222+
}
223+
}
224+
225+
private def appendDeepJoinedNested(
226+
state: TraversalState,
227+
initial: Val.Arr,
228+
out: StringBuilderWriter): Unit = {
229+
var current = initial
230+
while (true) {
231+
state.checkCanEnterArray(current)
232+
val direct = current.directBackingArray
233+
val len = current.length
234+
var index = 0
235+
var nested: Val.Arr = null
236+
while (index < len && (nested eq null)) {
237+
val child =
238+
if (direct ne null) direct(index).value else current.value(index)
239+
index += 1
240+
child match {
241+
case arr: Val.Arr => nested = arr
242+
case s: Val.Str => out.write(s.str)
243+
case v =>
244+
Error.fail("std.deepJoin: expected string or array, got " + v.prettyName)
245+
}
246+
}
247+
if (nested eq null) return
248+
state.pushArray(current, index, len, direct)
249+
current = nested
250+
}
251+
}
252+
}

0 commit comments

Comments
 (0)