Fix more tests for graalvm (#484)

- Enable most missing tests for graalvm.
- Fix golden file containing exceptions for consistency (some had the
full relative path to jsonnet, some did not)
- Add code handle OutOfMemory in the Materializer. The GraalVM binary
will basically run much longer before throwing a StackOverflow and
instead hits limits with heap allocations during Materialization

Author: stephenamar-db

sjsonnet/src/sjsonnet/Materializer.scala

@@ -70,6 +70,8 @@ abstract class Materializer {
   } catch {
     case _: StackOverflowError =>
       Error.fail("Stackoverflow while materializing, possibly due to recursive value", v.pos)
+    case _: OutOfMemoryError =>
+      Error.fail("Stackoverflow while materializing, possibly due to recursive value", v.pos)
   }
 
   def reverse(pos: Position, v: ujson.Value): Val = v match {

sjsonnet/test/graalvm/run_test_suites.py

@@ -15,10 +15,25 @@ def strip_trailing_empty_lines(text: str) -> str:
     lines.pop()
   return '\n'.join(lines)
 
+root_dir = os.getcwd()
 
-def run_binary(binary_path: str, test_dir: str, jsonnet_file: str) -> Tuple[str, int]:
+def run_binary(binary_path: str, jsonnet_file: str) -> Tuple[str, int]:
   """Run the binary on a jsonnet file and return output and exit code."""
-  cmd = [binary_path, '-Xss100m', '-J', test_dir, jsonnet_file]
+  cmd = [
+    binary_path,
+    '--ext-str', 'var1=test',
+    '--ext-code', 'var2=local f(a, b) = {[a]: b, \"y\": 2}; f(\"x\", 1)',
+    '--ext-str', 'stringVar=2 + 2',
+    '--ext-code', 'codeVar=3 + 3',
+    '--ext-code', 'errorVar=error \'xxx\'',
+    '--ext-code', 'staticErrorVar=)',
+    '--ext-code', 'UndeclaredX=x',
+    '--ext-code', 'selfRecursiveVar=[42, std.extVar(\"selfRecursiveVar\")[0] + 1]',
+    '--ext-code', 'mutuallyRecursiveVar1=[42, std.extVar(\"mutuallyRecursiveVar2\")[0] + 1]',
+    '--ext-code', 'mutuallyRecursiveVar2=[42, std.extVar(\"mutuallyRecursiveVar1\")[0] + 1]',
+    '--tla-str', 'var1=test',
+    '--tla-code', 'var2={\"x\": 1, \"y\": 2}',
+    jsonnet_file]
   try:
     result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
     return result.stdout + result.stderr, result.returncode
@@ -35,26 +50,31 @@ def setUpClass(cls):
     """Set up the test class by building the GraalVM native binary."""
     if not hasattr(cls, '_binary_built'):      
       subprocess.run(["./mill", "sjsonnet.graal.nativeImage"], check=True, cwd=".")
-      BaseGraalVMTestSuite._binary_built = True  
-    cls.binary_path = "out/sjsonnet/graal/nativeImage.dest/native-executable"
+      BaseGraalVMTestSuite._binary_built = True
+    cls.binary_path = os.path.join(root_dir, "out/sjsonnet/graal/nativeImage.dest/native-executable")
 
-  def run_individual_test(self, test_dir: str, suite_name: str, jsonnet_file: str, base_name: str):
+  @classmethod
+  def tearDownClass(cls):
+    """Clean up the test class by removing the GraalVM native binary."""
+    os.chdir(root_dir)
+
+  def run_individual_test(self, jsonnet_file: str, base_name: str):
     """Run a single jsonnet test and assert it passes."""
     # Run the binary on the jsonnet file and capture output
-    output, exit_code = run_binary(self.binary_path, test_dir, jsonnet_file)
-    with open(os.path.join(test_dir, f"{base_name}.jsonnet.golden"), 'r', encoding='utf-8') as f:
+    output, exit_code = run_binary(self.binary_path, jsonnet_file)
+    with open(f"{base_name}.jsonnet.golden", 'r', encoding='utf-8') as f:
       golden_content = f.read()
 
-    # Strip test directory path from output only for go_test_suite
-    if suite_name == "go_test_suite":
-      normalized_output = strip_trailing_empty_lines(output.replace(f"{test_dir}/", ""))
-      normalized_golden_content = strip_trailing_empty_lines(golden_content.replace(f"{test_dir}/", ""))
-    else:
-      normalized_output = strip_trailing_empty_lines(output)
-      normalized_golden_content = strip_trailing_empty_lines(golden_content)
+    normalized_output = strip_trailing_empty_lines(output.replace('Exception in thread "main" ', ''))
+    normalized_golden_content = strip_trailing_empty_lines(golden_content.replace('Exception in thread "main" ', ''))
+    if jsonnet_file.endswith("trace.jsonnet"):
+      normalized_output = normalized_output.replace("true", "").strip()
+      normalized_golden_content = normalized_golden_content.replace("true", "").strip()
 
     # Compare with golden file, ignoring trailing empty lines
-    if len(normalized_golden_content) > 10000 and normalized_golden_content != normalized_output:
+    if normalized_golden_content.startswith("java.lang.StackOverflowError") and normalized_output.startswith("java.lang.StackOverflowError"):
+      return
+    elif len(normalized_golden_content) > 10000 and normalized_golden_content != normalized_output:
       print(f"normalized_golden_content: {normalized_golden_content[:100]}")
       print(f"normalized_output: {normalized_output[:100]}")
       self.fail("Mismatch - but content very very large")
@@ -63,43 +83,25 @@ def run_individual_test(self, test_dir: str, suite_name: str, jsonnet_file: str,
 
 class MainTestSuite(BaseGraalVMTestSuite):
   """Test suite for the main jsonnet test files."""
+
+  @classmethod
+  def setUpClass(cls):
+    super(MainTestSuite, cls).setUpClass()
+    os.chdir(os.path.join(root_dir, "sjsonnet/test/resources/test_suite"))
 
   def test_all_files(self):
     """Test all files in the main test suite using subTest for each file."""
-    test_dir = "sjsonnet/test/resources/test_suite"
-    suite_name = "test_suite"
-    
-    if not os.path.exists(test_dir):
-      self.fail(f"Test directory {test_dir} not found")
-    
     # Skip list for main test suite
     skip_list = [
-      "error.obj_recursive_manifest",
-      "error.recursive_object_non_term",
-      "error.recursive_import",
-      "error.recursive_function_nonterm",
-      "error.function_infinite_default",
-      "error.obj_recursive",
-      "error.array_recursive_manifest",
-      "error.function_no_default_arg",
-      "error.invariant.option",
-      "error.negative_shfit",
+      # Some slight differences with how graalvm throws exceptions
       "error.overflow",
       "error.overflow2",
       "error.overflow3",
-      "error.parse_json",
-      "error.parse.string.invalid_escape",
-      "error.top_level_func",
-      "stdlib",
-      "tla.simple",
-      "trace"
     ]
 
     # Find all .jsonnet files in the test directory
-    jsonnet_pattern = os.path.join(test_dir, "*.jsonnet")
-    jsonnet_files = glob.glob(jsonnet_pattern)
-    
-    self.assertTrue(jsonnet_files, f"No .jsonnet files found in {test_dir}")
+    jsonnet_files = glob.glob("*.jsonnet")
+    self.assertTrue(jsonnet_files, f"No .jsonnet files found in {os.getcwd()}")
 
     for jsonnet_file in sorted(jsonnet_files):
       base_name = Path(jsonnet_file).stem
@@ -109,32 +111,24 @@ def test_all_files(self):
         continue
 
       with self.subTest(file=base_name):
-        self.run_individual_test(test_dir, suite_name, jsonnet_file, base_name)
+        self.run_individual_test(jsonnet_file, base_name)
 
 class GoTestSuite(BaseGraalVMTestSuite):
   """Test suite for the Go jsonnet test files."""
+
+  @classmethod
+  def setUpClass(cls):
+    super(GoTestSuite, cls).setUpClass()
+    os.chdir(os.path.join(root_dir, "sjsonnet/test/resources/go_test_suite"))
 
   def test_all_files(self):
     """Test all files in the go test suite using subTest for each file."""
-    test_dir = "sjsonnet/test/resources/go_test_suite"
-    suite_name = "go_test_suite"
-    
-    if not os.path.exists(test_dir):
-      self.fail(f"Test directory {test_dir} not found")
-    
-    # Skip list for go_test_suite
+    # Skip list for go_test_suite - almost all of them due to floating point precision differences
     skip_list = [
       "builtin_cos",
       "builtin_exp4", 
       "builtin_log3",
       "div3",
-      "extvar_code",
-      "extvar_error",
-      "extvar_hermetic",
-      "extvar_mutually_recursive",
-      "extvar_self_recursive",
-      "extvar_static_error",
-      "extvar_string",
       "function_too_many_params",
       "native1",
       "native2",
@@ -150,10 +144,8 @@ def test_all_files(self):
     ]
 
     # Find all .jsonnet files in the test directory
-    jsonnet_pattern = os.path.join(test_dir, "*.jsonnet")
-    jsonnet_files = glob.glob(jsonnet_pattern)
-    
-    self.assertTrue(jsonnet_files, f"No .jsonnet files found in {test_dir}")
+    jsonnet_files = glob.glob("*.jsonnet")
+    self.assertTrue(jsonnet_files, f"No .jsonnet files found in {os.getcwd()}")
 
     for jsonnet_file in sorted(jsonnet_files):
       base_name = Path(jsonnet_file).stem
@@ -163,7 +155,7 @@ def test_all_files(self):
         continue
 
       with self.subTest(file=base_name):
-        self.run_individual_test(test_dir, suite_name, jsonnet_file, base_name)
+        self.run_individual_test(jsonnet_file, base_name)
 
 
 if __name__ == "__main__":

sjsonnet/test/resources/go_test_suite/arrcomp5.jsonnet.golden

@@ -1,3 +1,3 @@
 sjsonnet.StaticError: Unknown variable: x
-    at [Id x].(sjsonnet/test/resources/go_test_suite/arrcomp5.jsonnet:1:14)
+    at [Id x].(arrcomp5.jsonnet:1:14)
 

sjsonnet/test/resources/go_test_suite/arrcomp_if4.jsonnet.golden

@@ -1,3 +1,3 @@
 sjsonnet.StaticError: Unknown variable: y
-    at [Id y].(sjsonnet/test/resources/go_test_suite/arrcomp_if4.jsonnet:1:33)
+    at [Id y].(arrcomp_if4.jsonnet:1:33)
 

sjsonnet/test/resources/go_test_suite/builtinSubStr_second_parameter_not_number.jsonnet.golden

@@ -1,2 +1,2 @@
 sjsonnet.Error: Expected a number for offset in substr, got string
-    at [std.substr].(sjsonnet/test/resources/go_test_suite/builtinSubStr_second_parameter_not_number.jsonnet:1:11)
+    at [std.substr].(builtinSubStr_second_parameter_not_number.jsonnet:1:11)

sjsonnet/test/resources/go_test_suite/builtinSubStr_third_parameter_not_number.jsonnet.golden

@@ -1,2 +1,2 @@
 sjsonnet.Error: Expected a number for len in substr, got string
-    at [std.substr].(sjsonnet/test/resources/go_test_suite/builtinSubStr_third_parameter_not_number.jsonnet:1:11)
+    at [std.substr].(builtinSubStr_third_parameter_not_number.jsonnet:1:11)

sjsonnet/test/resources/go_test_suite/dollar_bad.jsonnet.golden

@@ -1,3 +1,3 @@
 sjsonnet.StaticError: Can't use $ outside of an object
-    at [$].(sjsonnet/test/resources/go_test_suite/dollar_bad.jsonnet:1:1)
+    at [$].(dollar_bad.jsonnet:1:1)
 

sjsonnet/test/resources/go_test_suite/error_from_array.jsonnet.golden

@@ -1,3 +1,3 @@
 sjsonnet.Error: xxx
-    at [Error].(sjsonnet/test/resources/go_test_suite/error_from_array.jsonnet:1:2)
-    at [Lookup].(sjsonnet/test/resources/go_test_suite/error_from_array.jsonnet:1:14)
+    at [Error].(error_from_array.jsonnet:1:2)
+    at [Lookup].(error_from_array.jsonnet:1:14)

sjsonnet/test/resources/go_test_suite/error_from_func.jsonnet.golden

@@ -1,4 +1,4 @@
 sjsonnet.Error: xxx
-    at [Error].(sjsonnet/test/resources/go_test_suite/error_from_func.jsonnet:1:25)
-    at [Apply1].(sjsonnet/test/resources/go_test_suite/error_from_func.jsonnet:1:37)
+    at [Error].(error_from_func.jsonnet:1:25)
+    at [Apply1].(error_from_func.jsonnet:1:37)
 

sjsonnet/test/resources/go_test_suite/error_function_fail.jsonnet.golden

@@ -1,4 +1,4 @@
 sjsonnet.Error: Couldn't manifest function with params [x]
-    at .(sjsonnet/test/resources/go_test_suite/error_function_fail.jsonnet:1:8)
-    at [Error].(sjsonnet/test/resources/go_test_suite/error_function_fail.jsonnet:1:1)
+    at .(error_function_fail.jsonnet:1:8)
+    at [Error].(error_function_fail.jsonnet:1:1)