fix recursive deepJoin and flattenDeepArray traversal hangs

bench/resources/bug_suite/deep_array_traversal.jsonnet

@@ -0,0 +1,13 @@
+local nest(n, x) = if n == 0 then x else [nest(n - 1, x)];
+local sharedStrings = std.makeArray(200, function(i) "x");
+local sharedValues = std.makeArray(200, function(i) if i % 2 == 0 then "x" else i);
+local sharedStringView = [sharedStrings + [], [] + sharedStrings, sharedStrings];
+local sharedValueView = [sharedValues + [], [] + sharedValues, sharedValues];
+{
+  deepJoinFlat: std.length(std.deepJoin(std.makeArray(20000, function(i) "x"))),
+  deepJoinNested: std.length(std.deepJoin(std.makeArray(1000, function(i) nest(20, "x")))),
+  flattenFlat: std.length(std.flattenDeepArray(std.makeArray(20000, function(i) i))),
+  flattenNested: std.length(std.flattenDeepArray(std.makeArray(1000, function(i) nest(20, i)))),
+  sharedViewDeepJoin: std.length(std.deepJoin(sharedStringView)),
+  sharedViewFlatten: std.length(std.flattenDeepArray(sharedValueView)),
+}

bench/resources/bug_suite/deep_array_traversal.jsonnet.golden

@@ -0,0 +1,8 @@
+{
+   "deepJoinFlat": 20000,
+   "deepJoinNested": 1000,
+   "flattenFlat": 20000,
+   "flattenNested": 1000,
+   "sharedViewDeepJoin": 600,
+   "sharedViewFlatten": 600
+}

sjsonnet/src-js/sjsonnet/stdlib/DeepArrayActiveSet.scala

@@ -0,0 +1,114 @@
+package sjsonnet.stdlib
+
+import sjsonnet.Val
+
+private[stdlib] final class DeepArrayActiveSet(initialCapacity: Int) {
+  private var keys = new Array[Val.Arr](tableSizeFor(initialCapacity))
+  private var size = 0
+  private var growAt = keys.length >> 1
+
+  @inline def contains(arr: Val.Arr): Boolean = {
+    val current = keys
+    val mask = current.length - 1
+    var slot = slotOf(arr, mask)
+    while (true) {
+      val key = current(slot)
+      if (key eq null) return false
+      if (key eq arr) return true
+      slot = (slot + 1) & mask
+    }
+    false
+  }
+
+  @inline def add(arr: Val.Arr): Boolean = {
+    val current = keys
+    val mask = current.length - 1
+    var slot = slotOf(arr, mask)
+    while (true) {
+      val key = current(slot)
+      if (key eq null) {
+        current(slot) = arr
+        size += 1
+        if (size >= growAt) grow()
+        return true
+      }
+      if (key eq arr) return false
+      slot = (slot + 1) & mask
+    }
+    false
+  }
+
+  @inline def remove(arr: Val.Arr): Unit = {
+    val current = keys
+    val mask = current.length - 1
+    var slot = slotOf(arr, mask)
+    while (true) {
+      val key = current(slot)
+      if (key eq null) return
+      if (key eq arr) {
+        removeAt(slot)
+        return
+      }
+      slot = (slot + 1) & mask
+    }
+  }
+
+  private def grow(): Unit = {
+    val old = keys
+    val next = new Array[Val.Arr](old.length << 1)
+    keys = next
+    growAt = next.length >> 1
+    var i = 0
+    while (i < old.length) {
+      val key = old(i)
+      if (key ne null) insertExisting(next, key)
+      i += 1
+    }
+  }
+
+  private def removeAt(slot0: Int): Unit = {
+    val current = keys
+    val mask = current.length - 1
+    var slot = slot0
+    var next = (slot + 1) & mask
+    while (current(next) ne null) {
+      val key = current(next)
+      val home = slotOf(key, mask)
+      if (((next - home) & mask) >= ((slot - home) & mask)) {
+        current(slot) = key
+        slot = next
+      }
+      next = (next + 1) & mask
+    }
+    current(slot) = null
+    size -= 1
+  }
+
+  private def insertExisting(table: Array[Val.Arr], arr: Val.Arr): Unit = {
+    val mask = table.length - 1
+    var slot = slotOf(arr, mask)
+    while (table(slot) ne null) {
+      slot = (slot + 1) & mask
+    }
+    table(slot) = arr
+  }
+
+  @inline private def slotOf(arr: Val.Arr, mask: Int): Int = {
+    var h = System.identityHashCode(arr)
+    // Scala.js identityHashCode uses stable object ids; avalanche before
+    // masking so regular id sequences do not cluster in power-of-two tables.
+    h ^= h >>> 16
+    h *= -2048144789
+    h ^= h >>> 13
+    h *= -1028477387
+    h ^= h >>> 16
+    h & mask
+  }
+
+  private def tableSizeFor(initialCapacity: Int): Int = {
+    var n = 8
+    val target = if (initialCapacity <= 0) 8 else initialCapacity << 1
+    while (n < target) n <<= 1
+    n
+  }
+}

sjsonnet/src-jvm-native/sjsonnet/stdlib/DeepArrayActiveSet.scala

@@ -0,0 +1,17 @@
+package sjsonnet.stdlib
+
+import sjsonnet.Val
+
+private[stdlib] final class DeepArrayActiveSet(initialCapacity: Int) {
+  private val active =
+    new java.util.IdentityHashMap[Val.Arr, java.lang.Boolean](initialCapacity)
+
+  @inline def contains(arr: Val.Arr): Boolean =
+    active.containsKey(arr)
+
+  @inline def add(arr: Val.Arr): Boolean =
+    active.put(arr, java.lang.Boolean.TRUE) eq null
+
+  @inline def remove(arr: Val.Arr): Unit =
+    active.remove(arr)
+}

sjsonnet/src/sjsonnet/stdlib/ArrayModule.scala

@@ -531,20 +531,11 @@ object ArrayModule extends AbstractFunctionModule {
       val arr = value0.asInstanceOf[Val.Arr]
       val out = new mutable.ArrayBuilder.ofRef[Eval]
       out.sizeHint(arr.length)
-      val q = new java.util.ArrayDeque[Eval](arr.length)
-      var initialIdx = 0
-      while (initialIdx < arr.length) {
-        q.add(arr.eval(initialIdx))
-        initialIdx += 1
-      }
-      while (!q.isEmpty) {
-        q.removeFirst().value match {
-          case v: Val.Arr =>
-            var i = v.length - 1
-            while (i >= 0) { q.push(v.eval(i)); i -= 1 }
-          case x => out += x
-        }
-      }
+      DeepArrayTraversal.appendFlattened(
+        arr,
+        out,
+        math.max(ev.settings.maxMaterializeDepth, ev.settings.maxStack)
+      )
       Val.Arr(pos, out.result())
     }
   }