From 567b4b7917d41bd966fb4b3ccec47e4214d0e214 Mon Sep 17 00:00:00 2001 From: Stefan Kopco Date: Sun, 22 Nov 2026 11:19:11 +0100 Subject: [PATCH 1/2] rx-language-server: semantic tokens say which names are sources and which equations In the projection a source and an equation are both a const, so TypeScript's tokens call both `variable` with `readonly`. An editor that draws semantic tokens over its grammar's colours (Zed with semantic_tokens "combined") then shows state and derived values alike, where the grammar told them apart. The legend has two modifiers more, `source` and `equation`. A variable token gets one when its declaration, followed through an import, is initialised with the projection's __.typed/__.src or __.eq. --- tooling/rx-language-server/README.md | 6 ++ tooling/rx-language-server/bin/server.js | 6 +- tooling/rx-language-server/src/tokens.js | 84 ++++++++++++++++++++++++ tooling/rx-language-server/test/run.js | 28 +++++++- 4 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 tooling/rx-language-server/src/tokens.js diff --git a/tooling/rx-language-server/README.md b/tooling/rx-language-server/README.md index 4eb74fa1..d5095de3 100644 --- a/tooling/rx-language-server/README.md +++ b/tooling/rx-language-server/README.md @@ -14,6 +14,12 @@ errors, else the checker's, each with its rule) are a second service. projection's prelude to `node_modules/.reactivescript/prelude.d.ts`. - `src/index.js` is the language plugin (exact mappings carry every feature; ranges only place type errors) and the diagnostics service. +- `src/tokens.js` marks TypeScript's semantic tokens with what the projection + hides: a source and an equation are both a `const` there, so both come back + as `variable` with `readonly`. A source's name gets the modifier `source` and + an equation's `equation`, wherever it is read or written, also in a module + that imports it. An editor maps the two to colours of its own (Zed: + `semantic_token_rules`; VS Code: `semanticTokenScopes`). - `bin/server.js` is the server, over stdio or IPC. The project needs a `tsconfig.json` with Solid's JSX (`"jsx": "preserve"`, diff --git a/tooling/rx-language-server/bin/server.js b/tooling/rx-language-server/bin/server.js index 6655ac25..b4603aa2 100644 --- a/tooling/rx-language-server/bin/server.js +++ b/tooling/rx-language-server/bin/server.js @@ -1,9 +1,11 @@ #!/usr/bin/env node // The .rx language server over stdio (or whatever transport the client asks for): Volar's TypeScript project with the -// .rx language plugin, TypeScript's services, and the crate's diagnostics. +// .rx language plugin, TypeScript's services (their semantic tokens marked `source` and `equation`), and the crate's +// diagnostics. const { createConnection, createServer, createTypeScriptProject, loadTsdkByPath } = require("@volar/language-server/node"); const { create: createTypeScriptServices } = require("volar-service-typescript"); const { rxLanguagePlugin, rxDiagnostics } = require("../src/index.js"); +const { withRxTokens } = require("../src/tokens.js"); const connection = createConnection(); const server = createServer(connection); @@ -16,7 +18,7 @@ connection.onInitialize(params => { return server.initialize( params, createTypeScriptProject(tsdk.typescript, tsdk.diagnosticMessages, () => ({ languagePlugins: [rxLanguagePlugin(tsdk.typescript)] })), - [...createTypeScriptServices(tsdk.typescript), rxDiagnostics()], + [...withRxTokens(tsdk.typescript, createTypeScriptServices(tsdk.typescript)), rxDiagnostics()], ); }); diff --git a/tooling/rx-language-server/src/tokens.js b/tooling/rx-language-server/src/tokens.js new file mode 100644 index 00000000..91810832 --- /dev/null +++ b/tooling/rx-language-server/src/tokens.js @@ -0,0 +1,84 @@ +// Semantic tokens that say what `.rx` knows and TypeScript does not. In the projection a source and an equation are +// both a `const` (`const cart = __.typed([])`, `const count = __.eq(…, true)`), so TypeScript's tokens call +// both `variable` with `readonly`, and an editor that draws semantic tokens over its grammar's colours shows state and +// derived values alike. Each such token gets a modifier of its own here, `source` or `equation`, wherever the name is +// read or written, in the module that declares it and in one that imports it. + +/** The modifiers this adds to the legend. */ +const MODIFIERS = ["source", "equation"]; + +/** The projection's helper a declaration is initialised with (`project.rs`), and the modifier its name gets. */ +const HELPERS = { typed: "source", src: "source", eq: "equation" }; + +/** The modifier for the name at `node`: its declaration, followed through an import, is a `const` (or a name a + * destructuring `const` binds) initialised with one of the helpers. */ +function roleOf(ts, checker, node) { + let symbol = ts.isShorthandPropertyAssignment(node.parent) ? checker.getShorthandAssignmentValueSymbol(node.parent) : checker.getSymbolAtLocation(node); + if (!symbol) return undefined; + if (symbol.flags & ts.SymbolFlags.Alias) symbol = checker.getAliasedSymbol(symbol); + let declaration = symbol.valueDeclaration; + while (declaration && ts.isBindingElement(declaration)) declaration = declaration.parent.parent; + if (!declaration || !ts.isVariableDeclaration(declaration)) return undefined; + const call = declaration.initializer; + if (!call || !ts.isCallExpression(call)) return undefined; + const helper = call.expression; + if (!ts.isPropertyAccessExpression(helper) || !ts.isIdentifier(helper.expression) || helper.expression.text !== "__") return undefined; + return HELPERS[helper.name.text]; +} + +/** Adds the modifiers to `tokens` (`[line, character, length, type, modifiers]`, as the TypeScript service gives + * them for `document`, the projection). */ +function mark(ts, languageService, fileName, document, legend, tokens) { + const variable = legend.tokenTypes.indexOf("variable"); + const bits = Object.fromEntries(MODIFIERS.map(m => [m, 1 << legend.tokenModifiers.indexOf(m)])); + const program = languageService.getProgram(); + const file = program?.getSourceFile(fileName); + if (!file || variable < 0) return; + const checker = program.getTypeChecker(); + const names = new Map(); + const visit = node => { if (ts.isIdentifier(node)) names.set(node.getStart(file), node); ts.forEachChild(node, visit); }; + visit(file); + for (const token of tokens) { + if (token[3] !== variable) continue; + const node = names.get(document.offsetAt({ line: token[0], character: token[1] })); + if (!node) continue; + const role = roleOf(ts, checker, node); + if (role) token[4] |= bits[role]; + } +} + +/** The services, with the legend's two modifiers more and TypeScript's tokens marked with them. + * @param {typeof import("typescript")} ts */ +function withRxTokens(ts, services) { + return services.map(service => { + const provider = service.capabilities?.semanticTokensProvider; + if (!provider?.legend) return service; + const legend = { ...provider.legend, tokenModifiers: [...provider.legend.tokenModifiers, ...MODIFIERS] }; + return { + ...service, + capabilities: { ...service.capabilities, semanticTokensProvider: { ...provider, legend } }, + create(context, ...rest) { + const instance = service.create(context, ...rest); + if (!instance.provideDocumentSemanticTokens) return instance; + const provide = instance.provideDocumentSemanticTokens.bind(instance); + return { + ...instance, + async provideDocumentSemanticTokens(document, range, legend, ...more) { + const tokens = await provide(document, range, legend, ...more); + if (!tokens?.length) return tokens; + try { + const { URI } = require("vscode-uri"); + const fileName = instance.provide["typescript/documentFileName"](URI.parse(document.uri)); + mark(ts, instance.provide["typescript/languageService"](), fileName, document, legend, tokens); + } catch (e) { + console.error(`rx: marking semantic tokens of ${document.uri} failed: ${e?.stack ?? e}`); // TypeScript's stay + } + return tokens; + }, + }; + }, + }; + }); +} + +module.exports = { withRxTokens, MODIFIERS }; diff --git a/tooling/rx-language-server/test/run.js b/tooling/rx-language-server/test/run.js index 6751e34a..f9ddbb03 100644 --- a/tooling/rx-language-server/test/run.js +++ b/tooling/rx-language-server/test/run.js @@ -38,7 +38,7 @@ const hoverText = h => !h ? "" : typeof h.contents === "string" ? h.contents : A (async () => { // SERVER: another build of it, such as the extension's bundle (tooling/vscode-rx/dist/server.js) const server = startLanguageServer(process.env.SERVER ? path.resolve(process.env.SERVER) : path.join(__dirname, "../bin/server.js"), ws); - await server.initialize(URI.file(ws).toString(), { typescript: { tsdk: path.dirname(require.resolve("typescript")) } }); + const initialized = await server.initialize(URI.file(ws).toString(), { typescript: { tsdk: path.dirname(require.resolve("typescript")) } }); const cartFile = path.join(ws, "cart.rx"); const cart = await server.openTextDocument(cartFile, "reactivescript"); const text = cart.getText(); @@ -63,6 +63,32 @@ const hoverText = h => !h ? "" : typeof h.contents === "string" ? h.contents : A const refs = await server.sendReferencesRequest(cart.uri, at(text, "export count", 8), { includeDeclaration: true }); check("`count` is referenced in cart.rx and beyond", (refs ?? []).length >= 3, refs?.map(r => `${path.basename(r.uri)}:${r.range.start.line + 1}`)); + console.log("semantic tokens"); + const legend = initialized.capabilities.semanticTokensProvider.legend; + const st = (await server.sendSemanticTokensRequest(cart.uri))?.data ?? []; + /** The modifiers of the token at a position, or null where there is none. */ + const modifiersAt = ({ line, character }) => { + for (let i = 0, l = 0, c = 0; i < st.length; i += 5) { + l += st[i]; c = st[i] ? st[i + 1] : c + st[i + 1]; + if (l === line && c === character) return legend.tokenModifiers.filter((_, bit) => st[i + 4] & (1 << bit)); + } + return null; + }; + const m1 = modifiersAt(at(text, "export source cart", 14)); + check("a source's declaration is `source`", m1?.includes("source") && !m1.includes("equation"), m1); + const m2 = modifiersAt(at(text, "count = cart.reduce", 8)); + check("a read of a source is `source`", m2?.includes("source"), m2); + const m3 = modifiersAt(at(text, "export count", 7)); + check("an equation's declaration is `equation`", m3?.includes("equation") && !m3.includes("source"), m3); + const m4 = modifiersAt(at(text, "if (count === 0)", 4)); + check("a read of an equation is `equation`", m4?.includes("equation"), m4); + const m5 = modifiersAt(at(text, "products.reduce")); + check("a source imported from catalog.rx is `source`", m5?.includes("source"), m5); + const m6 = modifiersAt(at(text, "cart := ls")); + check("a write's target is `source`", m6?.includes("source"), m6); + const m7 = modifiersAt(at(text, "let items = cart", 4)); + check("a handler's local is neither", m7 && !m7.includes("source") && !m7.includes("equation"), m7); + console.log("completion while typing"); const lines = text.split("\n"); const end = { line: lines.length - 1, character: lines[lines.length - 1].length }; const typed = await server.updateTextDocument(cart.uri, [{ range: { start: end, end }, newText: "\nnewest = cart." }]); From cae22114a5435b745ad10a4c07cd61ed7151b7e3 Mon Sep 17 00:00:00 2001 From: Stefan Kopco Date: Sun, 22 Nov 2026 13:15:44 +0100 Subject: [PATCH 2/2] rx-language-server: semantic tokens: a shadowing test, and the README says VS Code doesn't use them yet --- tooling/rx-language-server/README.md | 7 +++-- tooling/rx-language-server/src/tokens.js | 12 ++++----- tooling/rx-language-server/test/run.js | 34 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/tooling/rx-language-server/README.md b/tooling/rx-language-server/README.md index d5095de3..9d67b999 100644 --- a/tooling/rx-language-server/README.md +++ b/tooling/rx-language-server/README.md @@ -18,8 +18,11 @@ errors, else the checker's, each with its rule) are a second service. hides: a source and an equation are both a `const` there, so both come back as `variable` with `readonly`. A source's name gets the modifier `source` and an equation's `equation`, wherever it is read or written, also in a module - that imports it. An editor maps the two to colours of its own (Zed: - `semantic_token_rules`; VS Code: `semanticTokenScopes`). + that imports it from a `.rx` file. A module read through its signature (a + `.d.rx`, or an MO-9 stub such as `export declare const n: any`) has no + initialiser, so its names get neither. Zed maps the two to colours with + `semantic_token_rules`; VS Code doesn't use them yet, since + `tooling/vscode-rx` contributes no `semanticTokenScopes`. - `bin/server.js` is the server, over stdio or IPC. The project needs a `tsconfig.json` with Solid's JSX (`"jsx": "preserve"`, diff --git a/tooling/rx-language-server/src/tokens.js b/tooling/rx-language-server/src/tokens.js index 91810832..7546eacf 100644 --- a/tooling/rx-language-server/src/tokens.js +++ b/tooling/rx-language-server/src/tokens.js @@ -4,20 +4,20 @@ // derived values alike. Each such token gets a modifier of its own here, `source` or `equation`, wherever the name is // read or written, in the module that declares it and in one that imports it. +const { URI } = require("vscode-uri"); + /** The modifiers this adds to the legend. */ const MODIFIERS = ["source", "equation"]; /** The projection's helper a declaration is initialised with (`project.rs`), and the modifier its name gets. */ const HELPERS = { typed: "source", src: "source", eq: "equation" }; -/** The modifier for the name at `node`: its declaration, followed through an import, is a `const` (or a name a - * destructuring `const` binds) initialised with one of the helpers. */ +/** The modifier for the name at `node`: its declaration, followed through an import, is a `const` initialised with one of the helpers. */ function roleOf(ts, checker, node) { let symbol = ts.isShorthandPropertyAssignment(node.parent) ? checker.getShorthandAssignmentValueSymbol(node.parent) : checker.getSymbolAtLocation(node); if (!symbol) return undefined; if (symbol.flags & ts.SymbolFlags.Alias) symbol = checker.getAliasedSymbol(symbol); let declaration = symbol.valueDeclaration; - while (declaration && ts.isBindingElement(declaration)) declaration = declaration.parent.parent; if (!declaration || !ts.isVariableDeclaration(declaration)) return undefined; const call = declaration.initializer; if (!call || !ts.isCallExpression(call)) return undefined; @@ -30,7 +30,8 @@ function roleOf(ts, checker, node) { * them for `document`, the projection). */ function mark(ts, languageService, fileName, document, legend, tokens) { const variable = legend.tokenTypes.indexOf("variable"); - const bits = Object.fromEntries(MODIFIERS.map(m => [m, 1 << legend.tokenModifiers.indexOf(m)])); + const bits = {}; + for (const m of MODIFIERS) { const i = legend.tokenModifiers.indexOf(m); if (i >= 0) bits[m] = 1 << i; } const program = languageService.getProgram(); const file = program?.getSourceFile(fileName); if (!file || variable < 0) return; @@ -43,7 +44,7 @@ function mark(ts, languageService, fileName, document, legend, tokens) { const node = names.get(document.offsetAt({ line: token[0], character: token[1] })); if (!node) continue; const role = roleOf(ts, checker, node); - if (role) token[4] |= bits[role]; + if (role && bits[role]) token[4] |= bits[role]; } } @@ -67,7 +68,6 @@ function withRxTokens(ts, services) { const tokens = await provide(document, range, legend, ...more); if (!tokens?.length) return tokens; try { - const { URI } = require("vscode-uri"); const fileName = instance.provide["typescript/documentFileName"](URI.parse(document.uri)); mark(ts, instance.provide["typescript/languageService"](), fileName, document, legend, tokens); } catch (e) { diff --git a/tooling/rx-language-server/test/run.js b/tooling/rx-language-server/test/run.js index f9ddbb03..29afc62e 100644 --- a/tooling/rx-language-server/test/run.js +++ b/tooling/rx-language-server/test/run.js @@ -89,6 +89,40 @@ const hoverText = h => !h ? "" : typeof h.contents === "string" ? h.contents : A const m7 = modifiersAt(at(text, "let items = cart", 4)); check("a handler's local is neither", m7 && !m7.includes("source") && !m7.includes("equation"), m7); + // A parameter and a local named like a source are neither: the name is looked up, not matched. + const shadowText = [ + "source n: number := 0", + "source total: number := 0", + "fn twice(n: number): number => n * 2", + "double = n + 1", + "action bump() {", + " let n = 1", + " total := n", + "}", + "", + ].join("\n"); + fs.writeFileSync(path.join(ws, "shadow.rx"), shadowText); + const shadow = await server.openTextDocument(path.join(ws, "shadow.rx"), "reactivescript"); + const sst = (await server.sendSemanticTokensRequest(shadow.uri))?.data ?? []; + const shadowAt = pos => { + for (let i = 0, l = 0, c = 0; i < sst.length; i += 5) { + l += sst[i]; c = sst[i] ? sst[i + 1] : c + sst[i + 1]; + if (l === pos.line && c === pos.character) return legend.tokenModifiers.filter((_, bit) => sst[i + 4] & (1 << bit)); + } + return null; + }; + const neither = m => m && !m.includes("source") && !m.includes("equation"); + const s1 = shadowAt(at(shadowText, "double = n", 9)); + check("a read of the source `n` is `source`", s1?.includes("source"), s1); + const s2 = shadowAt(at(shadowText, "twice(n", 6)); + check("a parameter named like a source is neither", neither(s2), s2); + const s3 = shadowAt(at(shadowText, "=> n * 2", 3)); + check("a read of that parameter is neither", neither(s3), s3); + const s4 = shadowAt(at(shadowText, "let n", 4)); + check("a local named like a source is neither", neither(s4), s4); + const s5 = shadowAt(at(shadowText, ":= n", 3)); + check("a read of that local is neither", neither(s5), s5); + console.log("completion while typing"); const lines = text.split("\n"); const end = { line: lines.length - 1, character: lines[lines.length - 1].length }; const typed = await server.updateTextDocument(cart.uri, [{ range: { start: end, end }, newText: "\nnewest = cart." }]);