fix(ui): make tmTheme JSON/YAML/PLIST selectors match TextMate scopes (#984)

Author: sebthom

docs/adopter-guide.md

@@ -134,17 +134,22 @@ TM4E also defines commands for toggling line comments and adding or removing blo
 
 ## Contributing Themes
 
-TM4E ships with built-in Light and Dark themes that are linked to the Eclipse appearance themes, but plugins can contribute additional CSS-based themes through the `org.eclipse.tm4e.ui.themes` extension point.
+TM4E ships with built-in Light and Dark themes that are linked to the Eclipse appearance themes, but plugins can contribute additional themes through the `org.eclipse.tm4e.ui.themes` extension point.
 
 ```xml
 <extension point="org.eclipse.tm4e.ui.themes">
-  <theme
+   <theme
       id="com.example.MyTheme"
       name="MyTheme"
       path="themes/MyTheme.css"/>
 </extension>
 ```
 
+The `path` attribute can point to:
+
+- a CSS theme file (`*.css`), or
+- a TextMate theme file (for example `*.tmTheme`, `*.plist`, `*.json`, `*.yaml`, `*.yml`).
+
 Themes can be flagged as more suitable for light or dark appearances and can be associated with specific grammar scopes so that, for example, a dedicated theme applies whenever a particular language is active.
 You declare one or more `<theme>` elements and then add `themeAssociation` elements that link themes to one or more scopes and optional dark/light variants.
 The exact attributes and options are described in the `themes` extension point schema.

org.eclipse.tm4e.ui.tests/build.properties

@@ -6,3 +6,6 @@ bin.includes = META-INF/,\
                grammars/,\
                plugin.xml,\
                about.html
+
+# JDT Null Analysis for Eclipse
+additional.bundles = org.eclipse.jdt.annotation,assertj-core

org.eclipse.tm4e.ui.tests/src/main/java/org/eclipse/tm4e/ui/tests/internal/themes/CSSThemeTokenProviderTest.java

@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) 2025 Vegard IT GmbH and others.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * Sebastian Thomschke (Vegard IT GmbH) - initial implementation
+ */
+package org.eclipse.tm4e.ui.tests.internal.themes;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.eclipse.jface.text.TextAttribute;
+import org.eclipse.swt.SWT;
+import org.eclipse.tm4e.core.theme.RGB;
+import org.eclipse.tm4e.ui.themes.ColorManager;
+import org.eclipse.tm4e.ui.themes.css.CSSTokenProvider;
+import org.junit.jupiter.api.Test;
+
+class CSSThemeTokenProviderTest {
+
+	private final ColorManager colors = ColorManager.getInstance();
+
+	private TextAttribute getTextAttribute(final CSSTokenProvider provider, final String tokenType) {
+		final var tokenData = provider.getToken(tokenType).getData();
+		assertThat(tokenData).isInstanceOf(TextAttribute.class);
+		return (TextAttribute) tokenData;
+	}
+
+	@Test
+	void testBuiltInDarkCssTheme() throws Exception {
+		try (var in = Files.newInputStream(Path.of("../org.eclipse.tm4e.ui/themes/Dark.css"))) {
+			final var provider = new CSSTokenProvider(in);
+
+			assertThat(provider.getEditorForeground()).isEqualTo(colors.getColor(new RGB(212, 212, 212)));
+			assertThat(provider.getEditorBackground()).isEqualTo(colors.getColor(new RGB(30, 30, 30)));
+			assertThat(provider.getEditorCurrentLineHighlight()).isEqualTo(colors.getColor(new RGB(40, 40, 40)));
+
+			assertThat(getTextAttribute(provider, "entity.other.attribute-name").getForeground())
+					.isEqualTo(colors.getColor(new RGB(156, 220, 254)));
+
+			assertThat(getTextAttribute(provider, "storage.type.java").getForeground())
+					.isEqualTo(colors.getColor(new RGB(78, 201, 176)));
+
+			assertThat(provider.getToken("this.selector.does.not.exist").getData()).isNull();
+		}
+	}
+
+	@Test
+	void testBuiltInMonokaiCssTheme() throws Exception {
+		try (var in = Files.newInputStream(Path.of("../org.eclipse.tm4e.ui/themes/Monokai.css"))) {
+			final var provider = new CSSTokenProvider(in);
+
+			assertThat(provider.getEditorForeground()).isEqualTo(colors.getColor(new RGB(248, 248, 242)));
+			assertThat(provider.getEditorBackground()).isEqualTo(colors.getColor(new RGB(39, 40, 34)));
+
+			assertThat(getTextAttribute(provider, "keyword").getForeground())
+					.isEqualTo(colors.getColor(new RGB(249, 38, 114)));
+
+			final var storageTypeAttrs = getTextAttribute(provider, "storage.type");
+			assertThat(storageTypeAttrs.getForeground()).isEqualTo(colors.getColor(new RGB(102, 217, 239)));
+			assertThat(storageTypeAttrs.getStyle() & SWT.ITALIC).isEqualTo(SWT.ITALIC);
+
+			final var inheritedClassAttrs = getTextAttribute(provider, "entity.other.inherited-class.java");
+			assertThat(inheritedClassAttrs.getForeground()).isEqualTo(colors.getColor(new RGB(166, 226, 46)));
+			assertThat(inheritedClassAttrs.getStyle() & SWT.ITALIC).isEqualTo(SWT.ITALIC);
+			assertThat(inheritedClassAttrs.getStyle() & TextAttribute.UNDERLINE).isEqualTo(TextAttribute.UNDERLINE);
+		}
+	}
+}

org.eclipse.tm4e.ui.tests/src/main/java/org/eclipse/tm4e/ui/tests/internal/themes/TMTokenProviderTest.java

@@ -16,9 +16,11 @@
 import java.io.ByteArrayInputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
 
 import org.eclipse.jface.text.TextAttribute;
 import org.eclipse.swt.SWT;
+import org.eclipse.tm4e.core.model.TMToken;
 import org.eclipse.tm4e.core.registry.IThemeSource.ContentType;
 import org.eclipse.tm4e.core.theme.RGB;
 import org.eclipse.tm4e.ui.internal.themes.TMThemeTokenProvider;
@@ -157,4 +159,43 @@ void testVSCodeJsonTheme() throws Exception {
 			assertThat(attrs.getForeground()).isEqualTo(colors.getColor(RGB.fromHex("#00FF00")));
 		}
 	}
+
+	@Test
+	void testTMThemeMatchesAgainstScopesStack() throws Exception {
+		try (var in = new ByteArrayInputStream("""
+			{
+			  "name": "Scopes test",
+			  "tokenColors": [
+			    {
+			      "settings": {
+			        "foreground": "#000000"
+			      }
+			    },
+			    {
+			      "scope": "entity.other",
+			      "settings": {
+			        "foreground": "#D197D9"
+			      }
+			    }
+			  ]
+			}
+			""".getBytes())) {
+			final var theme = new TMThemeTokenProvider(ContentType.JSON, in);
+
+			final var token = new TMToken(
+					0,
+					"meta.java.other.definition.class.entity.inherited.classes.inherited-class",
+					List.of(
+							"source.java@org.eclipse.tm4e.language_pack",
+							"meta.class.java",
+							"meta.definition.class.inherited.classes.java",
+							"entity.other.inherited-class.java"),
+					null);
+
+			final var data = theme.getToken(token).getData();
+			assertThat(data).isInstanceOf(TextAttribute.class);
+			final var attrs = (TextAttribute) data;
+			assertThat(attrs.getForeground()).isEqualTo(colors.getColor(RGB.fromHex("#D197D9")));
+		}
+	}
 }

org.eclipse.tm4e.ui.tests/src/main/java/org/eclipse/tm4e/ui/tests/support/StyleRangesCollector.java

@@ -45,7 +45,12 @@ public void onUninstalled() {
 	public void onColorized(final TextPresentation presentation, final Throwable error) {
 		add(presentation);
 		if (waitForToLineNumber != null) {
-			final int offset = presentation.getExtent().getOffset() + presentation.getExtent().getLength();
+			final int endOffsetExclusive = presentation.getExtent().getOffset() + presentation.getExtent().getLength();
+			// The extent end offset is exclusive. If it points to the start of the next line,
+			// document.getLineOfOffset(endOffsetExclusive) would already return that next line
+			// even though the presentation does not actually cover it.
+			// We therefore check the line number of the last included character.
+			final int offset = presentation.getExtent().getLength() > 0 ? endOffsetExclusive - 1 : endOffsetExclusive;
 			try {
 				if (waitForToLineNumber != document.getLineOfOffset(offset)) {
 					return;

org.eclipse.tm4e.ui.tests/src/main/java/org/eclipse/tm4e/ui/tests/themes/CSSThemeColorizationTest.java

@@ -0,0 +1,103 @@
+/**
+ * Copyright (c) 2025 Vegard IT GmbH and others.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * Sebastian Thomschke (Vegard IT GmbH) - initial implementation
+ */
+package org.eclipse.tm4e.ui.tests.themes;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.eclipse.tm4e.core.grammar.IGrammar;
+import org.eclipse.tm4e.core.registry.IGrammarSource;
+import org.eclipse.tm4e.core.registry.Registry;
+import org.eclipse.tm4e.ui.tests.support.TMEditor;
+import org.eclipse.tm4e.ui.tests.support.TestUtils;
+import org.eclipse.tm4e.ui.themes.css.CSSTokenProvider;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class CSSThemeColorizationTest {
+
+	private static final String SAMPLE_TEXT = "let a = '';\nlet b = 10;\nlet c = true;";
+
+	private IGrammar grammar;
+	private TMEditor editor;
+
+	@BeforeEach
+	void setup() throws Exception {
+		TestUtils.assertNoTM4EThreadsRunning();
+		grammar = new Registry().addGrammar(IGrammarSource.fromResource(getClass(), "/grammars/TypeScript.tmLanguage.json"));
+	}
+
+	@AfterEach
+	void tearDown() throws Exception {
+		if (editor != null) {
+			editor.dispose();
+			editor = null;
+		}
+		TestUtils.assertNoTM4EThreadsRunning();
+	}
+
+	private static void assertStyleRange(
+			final String styleRanges,
+			final int offset,
+			final int length,
+			final String expectedFontStyle,
+			final String expectedForegroundColor) {
+		assertThat(styleRanges).contains(
+				"StyleRange {" + offset + ", " + length + ", fontStyle=" + expectedFontStyle + ", foreground=Color {"
+						+ expectedForegroundColor + ", 255}}");
+	}
+
+	@Test
+	void darkCssThemeColorsAreApplied() throws Exception {
+		try (var in = Files.newInputStream(Path.of("../org.eclipse.tm4e.ui/themes/Dark.css"))) {
+			final var theme = new CSSTokenProvider(in);
+			editor = new TMEditor(grammar, theme, SAMPLE_TEXT);
+
+			final var commands = editor.execute();
+			assertThat(commands).hasSize(1);
+			final String ranges = commands.get(0).getStyleRanges();
+
+			// let -> storage (matches .storage / .storage.type)
+			assertStyleRange(ranges, 0, 3, "normal", "86, 156, 214");
+			// '' -> string
+			assertStyleRange(ranges, 8, 2, "normal", "206, 145, 120");
+			// 10 -> constant.numeric
+			assertStyleRange(ranges, 20, 2, "normal", "181, 206, 168");
+			// true -> constant.language
+			assertStyleRange(ranges, 32, 4, "normal", "86, 156, 214");
+		}
+	}
+
+	@Test
+	void monokaiCssThemeColorsAreApplied() throws Exception {
+		try (var in = Files.newInputStream(Path.of("../org.eclipse.tm4e.ui/themes/Monokai.css"))) {
+			final var theme = new CSSTokenProvider(in);
+			editor = new TMEditor(grammar, theme, SAMPLE_TEXT);
+
+			final var commands = editor.execute();
+			assertThat(commands).hasSize(1);
+			final String ranges = commands.get(0).getStyleRanges();
+
+			// let -> storage.type
+			assertStyleRange(ranges, 0, 3, "italic", "102, 217, 239");
+			// '' -> string
+			assertStyleRange(ranges, 8, 2, "normal", "230, 219, 116");
+			// 10 -> constant.numeric
+			assertStyleRange(ranges, 20, 2, "normal", "174, 129, 255");
+			// true -> constant.language
+			assertStyleRange(ranges, 32, 4, "normal", "174, 129, 255");
+		}
+	}
+}