Skip to content

Commit da1dd8d

Browse files
He-PinHe-Pin
authored
chore: sync tests from upstream. (#651)
Fix Summary Root Cause: The JS/Native/WASM platforms utilize the `scala-yaml` library; specifically, its `parseYaml` function does not support two features found in YAML multi-document streams: 1. `--- <content>` — Content immediately following the document marker (e.g., `--- >` for a folded scalar, or `--- 3` for a scalar value). 2. Empty documents (where no content appears after the `---` marker until the next `---`). Fix Strategy (3 Changes): 1. `src-native/Platform.scala` + `src-js/Platform.scala`: Replaced the manual `splitYamlDocuments` implementation with `parseManyYamls` to correctly handle inline content scenarios such as `--- >` and `--- 3`. Added a `addExplicitNullsForEmptyDocs` pre-processing step to insert explicit `null` text for empty documents, thereby bypassing `scala-yaml`'s limitation regarding the handling of empty documents. 2. `src-js/Platform.scala` (`nodeToJson` function): Added a match case for `None` (specifically `case null | None => ujson.Null`) to ensure consistency with the Native platform. `scala-yaml` returns `None` when decoding `null` scalars, a scenario that was previously unhandled on the JS platform. 3. `test/src/ParseYamlTests.scala`: Added regression tests to cover scenarios involving `--- 3`, `--- >`, empty documents, bare document markers, and similar cases.
1 parent 3a39b43 commit da1dd8d

6 files changed

Lines changed: 161 additions & 100 deletions

File tree

‎sjsonnet/src-js/sjsonnet/Platform.scala‎

Lines changed: 46 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ object Platform {
1010
private def nodeToJson(node: Node): ujson.Value = node match {
1111
case _: Node.ScalarNode =>
1212
YamlDecoder.forAny.construct(node).getOrElse("") match {
13-
case null => ujson.Null
13+
case null | None => ujson.Null
1414
case v: String => ujson.read(s"\"${v.replace("\"", "\\\"").replace("\n", "\\n")}\"", false)
1515
case v: Boolean => ujson.Bool(v)
1616
case v: Int => ujson.Num(v.toDouble)
@@ -43,66 +43,66 @@ object Platform {
4343
}
4444

4545
def yamlToJson(s: String): ujson.Value = {
46-
// Split YAML multi-document stream manually, similar to SnakeYAML's loadAll
47-
// since parseManyYamls doesn't handle all cases correctly
48-
val documents = splitYamlDocuments(s)
49-
50-
documents.size match {
51-
case 0 => ujson.Null
52-
case 1 => parseSingleDocument(documents.head)
53-
case _ =>
54-
val buf = new mutable.ArrayBuffer[ujson.Value](documents.size)
55-
for (doc <- documents) {
56-
buf += parseSingleDocument(doc)
46+
if (s.trim.isEmpty) return ujson.Null
47+
48+
// Preprocess to add explicit nulls for empty documents,
49+
// since scala-yaml's parseManyYamls can't handle empty documents
50+
// (DocumentStart immediately followed by DocumentEnd).
51+
val preprocessed = addExplicitNullsForEmptyDocs(s)
52+
53+
parseManyYamls(preprocessed) match {
54+
case Right(documents) =>
55+
documents.size match {
56+
case 0 => ujson.Null
57+
case 1 => nodeToJson(documents.head)
58+
case _ =>
59+
val buf = new mutable.ArrayBuffer[ujson.Value](documents.size)
60+
for (doc <- documents) {
61+
buf += nodeToJson(doc)
62+
}
63+
ujson.Arr(buf)
5764
}
58-
ujson.Arr(buf)
65+
case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage)
5966
}
6067
}
6168

62-
private def splitYamlDocuments(s: String): List[String] = {
63-
if (s.trim.isEmpty) return Nil
64-
65-
// Split on document separator "---" at line start
66-
// But only if it's followed by whitespace or end of line
69+
/**
70+
* Inserts explicit "null" content for empty YAML documents. An empty document is one where a
71+
* "---" marker has no content before the next "---" marker or end of input.
72+
*/
73+
private def addExplicitNullsForEmptyDocs(s: String): String = {
6774
val lines = s.split("\n", -1).toList
68-
val documents = mutable.ArrayBuffer[String]()
69-
val currentDoc = mutable.ArrayBuffer[String]()
70-
var isFirstDoc = true
75+
val result = new mutable.ArrayBuffer[String](lines.size + 4)
76+
var pendingEmptySep = false
7177

7278
for (line <- lines) {
7379
val trimmed = line.trim
74-
// Check if this line starts with "---" and is followed by whitespace or end
75-
if (trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace)) {
76-
// Save previous document if not empty
77-
if (currentDoc.nonEmpty || !isFirstDoc) {
78-
documents += currentDoc.mkString("\n")
80+
val isSep =
81+
trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace)
82+
83+
if (isSep) {
84+
if (pendingEmptySep) {
85+
// Previous "---" had no content after it; insert explicit null
86+
result += "null"
7987
}
80-
currentDoc.clear()
81-
isFirstDoc = false
88+
result += line
89+
// Check if this separator has inline content (e.g. "--- 3", "--- >")
90+
val afterMarker = trimmed.substring(3).trim
91+
pendingEmptySep = afterMarker.isEmpty
8292
} else {
83-
currentDoc += line
93+
if (pendingEmptySep && trimmed.nonEmpty) {
94+
pendingEmptySep = false
95+
}
96+
result += line
8497
}
8598
}
8699

87-
// Add last document
88-
if (currentDoc.nonEmpty || documents.nonEmpty) {
89-
documents += currentDoc.mkString("\n")
100+
// Handle trailing "---" with no content
101+
if (pendingEmptySep) {
102+
result += "null"
90103
}
91104

92-
documents.toList
93-
}
94-
95-
private def parseSingleDocument(doc: String): ujson.Value = {
96-
val trimmed = doc.trim
97-
if (trimmed.isEmpty) {
98-
ujson.Null
99-
} else {
100-
// Use parseYaml for single document
101-
parseYaml(trimmed) match {
102-
case Right(node) => nodeToJson(node)
103-
case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage)
104-
}
105-
}
105+
result.mkString("\n")
106106
}
107107

108108
def md5(s: String): String = {

‎sjsonnet/src-native/sjsonnet/Platform.scala‎

Lines changed: 45 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -78,66 +78,66 @@ object Platform {
7878
}
7979

8080
def yamlToJson(s: String): ujson.Value = {
81-
// Split YAML multi-document stream manually, similar to SnakeYAML's loadAll
82-
// since parseManyYamls doesn't handle all cases correctly
83-
val documents = splitYamlDocuments(s)
84-
85-
documents.size match {
86-
case 0 => ujson.Null
87-
case 1 => parseSingleDocument(documents.head)
88-
case _ =>
89-
val buf = new mutable.ArrayBuffer[ujson.Value](documents.size)
90-
for (doc <- documents) {
91-
buf += parseSingleDocument(doc)
81+
if (s.trim.isEmpty) return ujson.Null
82+
83+
// Preprocess to add explicit nulls for empty documents,
84+
// since scala-yaml's parseManyYamls can't handle empty documents
85+
// (DocumentStart immediately followed by DocumentEnd).
86+
val preprocessed = addExplicitNullsForEmptyDocs(s)
87+
88+
parseManyYamls(preprocessed) match {
89+
case Right(documents) =>
90+
documents.size match {
91+
case 0 => ujson.Null
92+
case 1 => nodeToJson(documents.head)
93+
case _ =>
94+
val buf = new mutable.ArrayBuffer[ujson.Value](documents.size)
95+
for (doc <- documents) {
96+
buf += nodeToJson(doc)
97+
}
98+
ujson.Arr(buf)
9299
}
93-
ujson.Arr(buf)
100+
case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage)
94101
}
95102
}
96103

97-
private def splitYamlDocuments(s: String): List[String] = {
98-
if (s.trim.isEmpty) return Nil
99-
100-
// Split on document separator "---" at line start
101-
// But only if it's followed by whitespace or end of line
104+
/**
105+
* Inserts explicit "null" content for empty YAML documents. An empty document is one where a
106+
* "---" marker has no content before the next "---" marker or end of input.
107+
*/
108+
private def addExplicitNullsForEmptyDocs(s: String): String = {
102109
val lines = s.split("\n", -1).toList
103-
val documents = mutable.ArrayBuffer[String]()
104-
val currentDoc = mutable.ArrayBuffer[String]()
105-
var isFirstDoc = true
110+
val result = new mutable.ArrayBuffer[String](lines.size + 4)
111+
var pendingEmptySep = false
106112

107113
for (line <- lines) {
108114
val trimmed = line.trim
109-
// Check if this line starts with "---" and is followed by whitespace or end
110-
if (trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace)) {
111-
// Save previous document if not empty
112-
if (currentDoc.nonEmpty || !isFirstDoc) {
113-
documents += currentDoc.mkString("\n")
115+
val isSep =
116+
trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace)
117+
118+
if (isSep) {
119+
if (pendingEmptySep) {
120+
// Previous "---" had no content after it; insert explicit null
121+
result += "null"
114122
}
115-
currentDoc.clear()
116-
isFirstDoc = false
123+
result += line
124+
// Check if this separator has inline content (e.g. "--- 3", "--- >")
125+
val afterMarker = trimmed.substring(3).trim
126+
pendingEmptySep = afterMarker.isEmpty
117127
} else {
118-
currentDoc += line
128+
if (pendingEmptySep && trimmed.nonEmpty) {
129+
pendingEmptySep = false
130+
}
131+
result += line
119132
}
120133
}
121134

122-
// Add last document
123-
if (currentDoc.nonEmpty || documents.nonEmpty) {
124-
documents += currentDoc.mkString("\n")
135+
// Handle trailing "---" with no content
136+
if (pendingEmptySep) {
137+
result += "null"
125138
}
126139

127-
documents.toList
128-
}
129-
130-
private def parseSingleDocument(doc: String): ujson.Value = {
131-
val trimmed = doc.trim
132-
if (trimmed.isEmpty) {
133-
ujson.Null
134-
} else {
135-
// Use parseYaml for single document
136-
parseYaml(trimmed) match {
137-
case Right(node) => nodeToJson(node)
138-
case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage)
139-
}
140-
}
140+
result.mkString("\n")
141141
}
142142

143143
private def computeHash(algorithm: String, s: String) = {

‎sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
|||
3333
---
3434
a: 1
35-
---
35+
---
3636
a: 2
3737
|||,
3838

@@ -42,5 +42,23 @@
4242
---a: 2
4343
a---: 3
4444
|||,
45+
46+
// Scalar documents can start on the same line as the document-start marker
47+
|||
48+
a: 1
49+
--- >
50+
hello
51+
world
52+
--- 3
53+
|||,
54+
55+
// Documents can be empty; this is interpreted as null
56+
|||
57+
a: 1
58+
---
59+
--- 2
60+
|||,
61+
62+
"---",
4563
]
4664
]

‎sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet.golden‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,20 @@
3737
"---a": 2,
3838
"a": 1,
3939
"a---": 3
40-
}
40+
},
41+
[
42+
{
43+
"a": 1
44+
},
45+
"hello world\n",
46+
3
47+
],
48+
[
49+
{
50+
"a": 1
51+
},
52+
null,
53+
2
54+
],
55+
null
4156
]

‎sjsonnet/test/resources/go_test_suite/stdlib_smoke_test.jsonnet‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
// Functions without optional arguments need only one line
55
// Functions with optional arguments need two lines - one with none of the optional arguments
66
// and the other with all of them.
7-
87
local assertClose(a, b) =
98
// Using 1e-12 as tolerance. Jsonnet uses double-precision floats with machine epsilon of 2**-53 ≈ 1.11e-16.
109
// This tolerance is ~9000x the machine epsilon, which is quite lenient and should work across
@@ -79,13 +78,13 @@ local assertClose(a, b) =
7978
mantissa: std.mantissa(x=5),
8079
floor: std.floor(x=5),
8180
ceil: std.ceil(x=5),
82-
sqrt: std.assertEqual(std.sqrt(x=5), 2.23606797749979),
81+
sqrt: assertClose(std.sqrt(x=5), 2.2360679774997898),
8382
sin: assertClose(std.sin(x=5), -0.9589242746631385),
84-
cos: assertClose(std.cos(x=5), 0.28366218546322625),
85-
tan: assertClose(std.tan(x=5), -3.380515006246586),
86-
asin: assertClose(std.asin(x=0.5), 0.5235987755982989),
87-
acos: assertClose(std.acos(x=0.5), 1.0471975511965979),
88-
atan: assertClose(std.atan(x=5), 1.373400766945016),
83+
cos: assertClose(std.cos(x=5), 0.2836621854632263),
84+
tan: assertClose(std.tan(x=5), -3.3805150062465854),
85+
asin: assertClose(std.asin(x=0.5), 0.52359877559829893),
86+
acos: assertClose(std.acos(x=0.5), 1.0471975511965976),
87+
atan: assertClose(std.atan(x=5), 1.3734007669450157),
8988

9089
// Assertions and debugging
9190
assertEqual: std.assertEqual(a="a", b="a"),

‎sjsonnet/test/src/sjsonnet/ParseYamlTests.scala‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,5 +44,34 @@ object ParseYamlTests extends TestSuite {
4444
// Test that trailing empty document with whitespace is handled
4545
eval("std.parseYaml('1\\n---\\n')") ==> ujson.Value("""[1,null]""")
4646
}
47+
test {
48+
// Scalar documents can start on the same line as the document-start marker
49+
// "--- 3" as standalone
50+
eval("std.parseYaml('--- 3\\n')") ==> ujson.Value("""3""")
51+
}
52+
test {
53+
// Folded scalar as document
54+
eval("std.parseYaml('--- >\\n hello\\n world\\n')") ==> ujson.Value(""""hello world\n"""")
55+
}
56+
test {
57+
// Combined: scalar docs on same line as marker
58+
eval("std.parseYaml('a: 1\\n--- >\\n hello\\n world\\n--- 3\\n')") ==> ujson.Value(
59+
"""[{"a": 1}, "hello world\n", 3]"""
60+
)
61+
}
62+
test {
63+
// empty doc then inline scalar
64+
eval("std.parseYaml('a: 1\\n---\\n--- 2\\n')") ==> ujson.Value(
65+
"""[{"a": 1}, null, 2]"""
66+
)
67+
}
68+
test {
69+
// Bare document separator
70+
eval("""std.parseYaml("---")""") ==> ujson.Value("""null""")
71+
}
72+
test {
73+
// Folded scalar without document marker (directly)
74+
eval("std.parseYaml('>\\n hello\\n world\\n')") ==> ujson.Value(""""hello world\n"""")
75+
}
4776
}
4877
}

0 commit comments

Comments
 (0)