Re-enable handling of unicode strings in std.base64 (#492) (#493)

This is an artificial limitation in go-jsonnet and we are not bound by
it.
We are perfectly capable of encoding and decoding unicode strings.

Author: stephenamar-db

sjsonnet/src/sjsonnet/Std.scala

@@ -1661,24 +1661,27 @@ class Std(
       rec(Materializer(value)(ev)).render
     },
     builtin("base64", "input") { (_, _, input: Val) =>
-      val b: Array[Int] = input match {
-        case Val.Str(_, value) => value.indices.map(value.codePointAt).toArray
-        case arr: Val.Arr      =>
-          if (!arr.forall(_.isInstanceOf[Val.Num])) {
-            Error.fail("Expected an array of numbers or a string")
+      input match {
+        case Val.Str(_, value) =>
+          Base64.getEncoder.encodeToString(value.getBytes("UTF-8"))
+        case arr: Val.Arr =>
+          val byteArr = new Array[Byte](arr.length)
+          for (i <- 0 until arr.length) {
+            val v = arr.force(i)
+            if (!v.isInstanceOf[Val.Num]) {
+              Error.fail(f"Expected an array of numbers, got a ${v.prettyName} at position $i")
+            }
+            val vInt = v.asInt
+            if (vInt < 0 || vInt > 255) {
+              Error.fail(
+                f"Found an invalid codepoint value at position $i (must be 0 <= X <= 255), got $vInt"
+              )
+            }
+            byteArr(i) = vInt.toByte
           }
-          arr.iterator.map(_.cast[Val.Num].asInt).toArray
+          Base64.getEncoder.encodeToString(byteArr)
         case x => Error.fail("Cannot base64 encode " + x.prettyName)
       }
-
-      for (i <- b) {
-        if (i < 0 || i > 255) {
-          Error.fail(
-            f"Found an invalid codepoint value in the ${input.prettyName} (must be 0 <= X <= 255), got $i"
-          )
-        }
-      }
-      Base64.getEncoder.encodeToString(b.map(_.toByte))
     },
     builtin("base64Decode", "str") { (_, _, str: String) =>
       try {

sjsonnet/test/graalvm/run_test_suites.py

@@ -137,6 +137,9 @@ def test_all_files(self):
       "native7",
       "native_error",
       "native_panic",
+
+      # We support base64 of unicode strings
+      "builtinBase64_string_high_codepoint",
     ]
 
     # Find all .jsonnet files in the test directory

sjsonnet/test/resources/go_test_suite/builtinBase64_invalid_byte_array.jsonnet.golden

@@ -1,3 +1,3 @@
-sjsonnet.Error: Expected an array of numbers or a string
+sjsonnet.Error: Expected an array of numbers, got a string at position 1
     at [std.base64].(builtinBase64_invalid_byte_array.jsonnet:1:11)
 

sjsonnet/test/resources/go_test_suite/builtinBase64_invalid_byte_array1.jsonnet.golden

@@ -1,3 +1,3 @@
-sjsonnet.Error: Found an invalid codepoint value in the array (must be 0 <= X <= 255), got -1
+sjsonnet.Error: Found an invalid codepoint value at position 1 (must be 0 <= X <= 255), got -1
     at [std.base64].(builtinBase64_invalid_byte_array1.jsonnet:1:11)
 

sjsonnet/test/resources/go_test_suite/builtinBase64_invalid_byte_array2.jsonnet.golden

@@ -1,3 +1,3 @@
-sjsonnet.Error: Found an invalid codepoint value in the array (must be 0 <= X <= 255), got 256
+sjsonnet.Error: Found an invalid codepoint value at position 1 (must be 0 <= X <= 255), got 256
     at [std.base64].(builtinBase64_invalid_byte_array2.jsonnet:1:11)
 

sjsonnet/test/src-js/sjsonnet/FileTests.scala

@@ -16,6 +16,8 @@ object FileTests extends BaseFileTests {
   )
 
   val goTestDataSkippedTests: Set[String] = Set(
+    // We support base64 of unicode strings
+    "builtinBase64_string_high_codepoint.jsonnet",
     "builtinSha1.jsonnet",
     "builtinSha256.jsonnet",
     "builtinSha3.jsonnet",

sjsonnet/test/src-jvm-native/sjsonnet/FileTests.scala

@@ -16,7 +16,10 @@ object FileTests extends BaseFileTests {
       )
     else Set.empty[String]
 
-  val goTestDataSkippedTests: Set[String] = Set.empty
+  val goTestDataSkippedTests: Set[String] = Set(
+    // We support base64 of unicode strings
+    "builtinBase64_string_high_codepoint.jsonnet"
+  )
 
   val tests: Tests = Tests {
     test("test_suite") - {