Fix issues found in our testing framework (#486)

- Some tests were not executed
- Exceptions in interpreter where thrown instead of being wrapper in
Either()
- Parsing of infinite number caused unhandled exceptions.
- Add new test suite to graalvm

Author: stephenamar-db

build.mill

@@ -202,8 +202,6 @@ object sjsonnet extends VersionFileModule {
         mvn"org.virtuslab::scala-yaml::0.3.0"
       )
 
-    val osName = System.getProperty("os.name").toLowerCase
-
     def releaseMode = ReleaseMode.ReleaseFull
     def nativeLTO = LTO.Full
     def nativeMultithreading = None

sjsonnet/src/sjsonnet/Interpreter.scala

@@ -155,11 +155,8 @@ class Interpreter(
   def evaluate(txt: String, path: Path): Either[Error, Val] = {
     val resolvedImport = StaticResolvedFile(txt)
     resolver.cache(path) = resolvedImport
-    for {
-      res <- resolver.parse(path, resolvedImport)(evaluator)
-      (parsed, _) = res
-      res0 <- handleException(evaluator.visitExpr(parsed)(ValScope.empty))
-      res = res0 match {
+    resolver.parse(path, resolvedImport)(evaluator) flatMap { case (expr, _) =>
+      handleException(evaluator.visitExpr(expr)(ValScope.empty)) flatMap {
         case f: Val.Func =>
           val defaults2 = f.params.defaultExprs.clone()
           val tlaExpressions = collection.mutable.Set.empty[Expr]
@@ -173,18 +170,20 @@ class Interpreter(
             }
             i += 1
           }
-          new Val.Func(f.pos, f.defSiteValScope, Params(f.params.names, defaults2)) {
+          val out = new Val.Func(f.pos, f.defSiteValScope, Params(f.params.names, defaults2)) {
             def evalRhs(vs: ValScope, es: EvalScope, fs: FileScope, pos: Position): Val =
               f.evalRhs(vs, es, fs, pos)
+
             override def evalDefault(expr: Expr, vs: ValScope, es: EvalScope): Val = {
               evaluator.visitExpr(expr)(
                 if (tlaExpressions.exists(_ eq expr)) ValScope.empty else vs
               )
             }
-          }.apply0(f.pos)(evaluator, TailstrictModeDisabled)
-        case x => x
+          }
+          handleException(out.apply0(f.pos)(evaluator, TailstrictModeDisabled))
+        case x => Right(x)
       }
-    } yield res
+    }
   }
 
   def materialize[T](res: Val, visitor: upickle.core.Visitor[T, T]): Either[Error, T] = {

sjsonnet/src/sjsonnet/Parser.scala

@@ -98,7 +98,12 @@ class Parser(
     if (s._2.length > 1 && Character.isDigit(s._2.charAt(1)) && s._2.charAt(0) == '0') {
       Fail.opaque("numbers cannot start with a 0 digit")
     } else {
-      Pass(Val.Num(s._1, s._2.toDouble))
+      val v = s._2.toDouble
+      if (v.isInfinite) {
+        Fail.opaque("finite number required")
+      } else {
+        Pass(Val.Num(s._1, v))
+      }
     }
   })
 

sjsonnet/test/graalvm/run_test_suites.py

@@ -65,14 +65,14 @@ def run_individual_test(self, jsonnet_file: str, base_name: str):
     with open(f"{base_name}.jsonnet.golden", 'r', encoding='utf-8') as f:
       golden_content = f.read()
 
-    normalized_output = strip_trailing_empty_lines(output.replace('Exception in thread "main" ', ''))
-    normalized_golden_content = strip_trailing_empty_lines(golden_content.replace('Exception in thread "main" ', ''))
+    normalized_output = output.strip()
+    normalized_golden_content = golden_content.strip()
     if jsonnet_file.endswith("trace.jsonnet"):
       normalized_output = normalized_output.replace("true", "").strip()
       normalized_golden_content = normalized_golden_content.replace("true", "").strip()
 
     # Compare with golden file, ignoring trailing empty lines
-    if normalized_golden_content.startswith("java.lang.StackOverflowError") and normalized_output.startswith("java.lang.StackOverflowError"):
+    if normalized_golden_content.startswith("""Exception in thread "main" java.lang.StackOverflowError""") and normalized_output.startswith("""Exception in thread "main" java.lang.StackOverflowError"""):
       return
     elif len(normalized_golden_content) > 10000 and normalized_golden_content != normalized_output:
       print(f"normalized_golden_content: {normalized_golden_content[:100]}")
@@ -92,12 +92,7 @@ def setUpClass(cls):
   def test_all_files(self):
     """Test all files in the main test suite using subTest for each file."""
     # Skip list for main test suite
-    skip_list = [
-      # Some slight differences with how graalvm throws exceptions
-      "error.overflow",
-      "error.overflow2",
-      "error.overflow3",
-    ]
+    skip_list = []
 
     # Find all .jsonnet files in the test directory
     jsonnet_files = glob.glob("*.jsonnet")
@@ -123,13 +118,16 @@ def setUpClass(cls):
 
   def test_all_files(self):
     """Test all files in the go test suite using subTest for each file."""
-    # Skip list for go_test_suite - almost all of them due to floating point precision differences
     skip_list = [
+      # GraalVM has slight difference with how it manages high precision floats
       "builtin_cos",
-      "builtin_exp4", 
+      "builtin_exp4",
       "builtin_log3",
       "div3",
-      "function_too_many_params",
+      "std.mantissa3",
+      "stdlib_smoke_test",
+
+      # These tests rely on custom native functions that are not implemented in the binary we use
       "native1",
       "native2",
       "native3",
@@ -139,8 +137,6 @@ def test_all_files(self):
       "native7",
       "native_error",
       "native_panic",
-      "std.mantissa3",
-      "stdlib_smoke_test"
     ]
 
     # Find all .jsonnet files in the test directory
@@ -157,6 +153,32 @@ def test_all_files(self):
       with self.subTest(file=base_name):
         self.run_individual_test(jsonnet_file, base_name)
 
+class NewTestSuite(BaseGraalVMTestSuite):
+  """Test suite for the new jsonnet test files."""
+
+  @classmethod
+  def setUpClass(cls):
+    super(NewTestSuite, cls).setUpClass()
+    os.chdir(os.path.join(root_dir, "sjsonnet/test/resources/new_test_suite"))
+
+  def test_all_files(self):
+    """Test all files in the main test suite using subTest for each file."""
+    # Skip list for main test suite
+    skip_list = []
+
+    # Find all .jsonnet files in the test directory
+    jsonnet_files = glob.glob("*jvm-native.jsonnet")
+    self.assertTrue(jsonnet_files, f"No .jsonnet files found in {os.getcwd()}")
+
+    for jsonnet_file in sorted(jsonnet_files):
+      base_name = Path(jsonnet_file).stem
+
+      # Check if this test should be skipped
+      if base_name in skip_list:
+        continue
+
+      with self.subTest(file=base_name):
+        self.run_individual_test(jsonnet_file, base_name)
 
 if __name__ == "__main__":
   unittest.main(verbosity=2)

sjsonnet/test/resources/go_test_suite/function_too_many_params.jsonnet.golden

@@ -1 +1,2 @@
 sjsonnet.Error: Function parameter x not bound in call
+    at .(function_too_many_params.jsonnet:1:1)

sjsonnet/test/resources/go_test_suite/native_panic.jsonnet.golden

@@ -1,3 +1,3 @@
 sjsonnet.Error: Internal Error
     at [std.nativePanic].(native_panic.jsonnet:1:26)
-
+Caused by: java.lang.RuntimeException: native function panic

sjsonnet/test/resources/test_suite/error.function_no_default_arg.jsonnet.golden

@@ -1,2 +1,2 @@
 sjsonnet.Error: Function parameter b not bound in call
-	at .(error.function_no_default_arg.jsonnet:17:1)
+    at .(error.function_no_default_arg.jsonnet:17:1)

sjsonnet/test/resources/test_suite/error.overflow.jsonnet.golden

@@ -1,3 +1,2 @@
-sjsonnet.Error: overflow
-    at [BinaryOp *].(error.overflow.jsonnet:17:7)
-
+sjsonnet.ParseError: Expected finite number required:17:6, found "\n"
+    at .(error.overflow.jsonnet:17:6)

sjsonnet/test/resources/test_suite/error.overflow3.jsonnet.golden

@@ -1,3 +1,3 @@
-sjsonnet.Error: numeric value is not finite
-    at [BinaryOp &].(error.overflow3.jsonnet:17:7)
+sjsonnet.ParseError: Expected finite number required:17:6, found " & 0\n"
+    at .(error.overflow3.jsonnet:17:6)
 

sjsonnet/test/resources/test_suite/error.top_level_func.jsonnet.golden

@@ -1,2 +1,2 @@
 sjsonnet.Error: Function parameter name not bound in call
-	at .(error.top_level_func.jsonnet:17:1)
+    at .(error.top_level_func.jsonnet:17:1)