From 06bc4ea74cf960dfdfd603a541a64a6c54e369bd Mon Sep 17 00:00:00 2001 From: Marco Roth Date: Fri, 14 Aug 2026 02:28:42 +0200 Subject: [PATCH] Language Service: Implement Semantic Token Provider --- docs/docs/integrations/editors/zed.md | 37 +++ javascript/packages/core/src/index.ts | 1 + .../packages/core/src/token-classification.ts | 231 +++++++++++++++++ .../core/test/token-classification.test.ts | 80 ++++++ .../highlighter/src/syntax-renderer.ts | 189 ++------------ javascript/packages/highlighter/src/themes.ts | 1 + .../packages/highlighter/themes/dracula.json | 1 + .../highlighter/themes/github-light.json | 1 + .../packages/highlighter/themes/onedark.json | 1 + .../packages/highlighter/themes/simple.json | 1 + .../highlighter/themes/tokyo-night.json | 1 + javascript/packages/language-server/README.md | 1 + .../language-server/src/capabilities.ts | 4 + .../packages/language-server/src/server.ts | 23 +- .../packages/language-server/src/session.ts | 4 +- .../language-server/src/user_settings.ts | 9 + .../test/user_settings.test.ts | 24 +- .../packages/language-service/src/index.ts | 1 + .../language-service/src/parser_service.ts | 10 +- .../src/semantic_tokens_provider.ts | 238 ++++++++++++++++++ .../test/semantic_tokens_provider.test.ts | 237 +++++++++++++++++ javascript/packages/vscode/README.md | 7 + javascript/packages/vscode/esbuild.js | 1 + javascript/packages/vscode/package.json | 7 + javascript/packages/vscode/src/client.ts | 15 ++ .../src/controllers/playground_controller.js | 3 +- playground/src/language-service.js | 16 ++ playground/src/monaco.js | 32 ++- 28 files changed, 1007 insertions(+), 169 deletions(-) create mode 100644 javascript/packages/core/src/token-classification.ts create mode 100644 javascript/packages/core/test/token-classification.test.ts create mode 100644 javascript/packages/language-service/src/semantic_tokens_provider.ts create mode 100644 javascript/packages/language-service/test/semantic_tokens_provider.test.ts diff --git a/docs/docs/integrations/editors/zed.md b/docs/docs/integrations/editors/zed.md index ee28b6bbf..2b98bde7d 100644 --- a/docs/docs/integrations/editors/zed.md +++ b/docs/docs/integrations/editors/zed.md @@ -37,6 +37,9 @@ Zed keeps language server settings under `lsp..initialization_options`, "minimumLines": 10, "maximumClasses": 2 }, + "semanticTokens": { + "enabled": true + }, "linter": { "enabled": true, "fixOnSave": true @@ -49,6 +52,40 @@ Zed keeps language server settings under `lsp..initialization_options`, These are the same options VS Code exposes as `languageServerHerb.*` settings, minus the `languageServerHerb` prefix. See the [language server documentation](/projects/language-server) for the full list. +### Semantic highlighting + +Herb can colour HTML+ERB from the parsed template rather than from a Tree-sitter grammar, which keeps tags, attributes and ERB delimiters right where they nest inside each other. Zed requests semantic tokens only when you ask it to, so turn them on for HTML+ERB: + +```json [settings.json] +{ + "languages": { + "HTML+ERB": { + "semantic_tokens": "combined" + } + } +} +``` + +`combined` layers the language server's tokens over Tree-sitter, which is what you want here, since Herb deliberately says nothing about the Ruby inside `<% %>` and leaves it to Tree-sitter and the Ruby language server. `full` would drop Tree-sitter entirely and leave that Ruby unstyled. + +Colours come from `semantic_token_rules`, matched by token type and modifier. Herb emits `type` for tag names, `property` for attribute names, `string` for values, `macro` for the `<%` and `%>` delimiters with an `output` modifier on `<%=` tags, `parameter` for the names in a `locals:` declaration, and `function` with the `defaultLibrary` modifier for Action View helpers: + +```json [settings.json] +{ + "global_lsp_settings": { + "semantic_token_rules": [ + { "token_type": "function", "token_modifiers": ["defaultLibrary"], "foreground_color": "#61AFEF" }, + { "token_type": "macro", "token_modifiers": ["output"], "foreground_color": "#C678DD" }, + { "token_type": "macro", "foreground_color": "#BE5046" }, + { "token_type": "property", "foreground_color": "#D19A66" }, + { "token_type": "type", "foreground_color": "#E06C75" } + ] + } +} +``` + +Set `semanticTokens.enabled` to `false` in the initialization options above to turn the feature off on Herb's side instead. + ### Inlay hints Herb annotates the closing tag of longer blocks with what it closes. Zed turns inlay hints off by default, so you need to enable them for HTML+ERB as well as configuring them on the Herb side: diff --git a/javascript/packages/core/src/index.ts b/javascript/packages/core/src/index.ts index cb2bde51a..f6a5ff826 100644 --- a/javascript/packages/core/src/index.ts +++ b/javascript/packages/core/src/index.ts @@ -28,6 +28,7 @@ export * from "./ruby-keywords.js" export * from "./semver.js" export * from "./token-list.js" export * from "./token.js" +export * from "./token-classification.js" export * from "./util.js" export * from "./visitor.js" export * from "./warning.js" diff --git a/javascript/packages/core/src/token-classification.ts b/javascript/packages/core/src/token-classification.ts new file mode 100644 index 000000000..c77a69741 --- /dev/null +++ b/javascript/packages/core/src/token-classification.ts @@ -0,0 +1,231 @@ +import { RUBY_KEYWORDS } from "./ruby-keywords.js" + +import type { Token } from "./token.js" + +export type TokenCategory = + | "html.tagName" + | "html.attributeName" + | "html.attributeValue" + | "html.delimiter" + | "html.comment" + | "html.doctype" + | "html.entity" + | "erb.delimiter" + | "erb.content" + | "erb.commentDelimiter" + | "erb.comment" + | "other" + +export interface ClassifiedToken { + token: Token + category: TokenCategory + quoted?: boolean + output?: boolean +} + +interface ClassifierState { + inTag: boolean + inQuotes: boolean + quoteCharacter: string + tagName: string + isClosingTag: boolean + expectingAttributeName: boolean + expectingAttributeValue: boolean + inComment: boolean + inERBComment: boolean + inERBOutput: boolean +} + +const HTML_DELIMITERS = new Set([ + "TOKEN_HTML_TAG_START", + "TOKEN_HTML_TAG_START_CLOSE", + "TOKEN_HTML_TAG_END", + "TOKEN_HTML_TAG_SELF_CLOSE", +]) + +const ERB_DELIMITERS = new Set(["TOKEN_ERB_START", "TOKEN_ERB_END"]) +const COMMENT_DELIMITERS = new Set(["TOKEN_HTML_COMMENT_START", "TOKEN_HTML_COMMENT_END"]) +const ERB_TOKENS = new Set([...ERB_DELIMITERS, "TOKEN_ERB_CONTENT"]) + +function initialState(): ClassifierState { + return { + inTag: false, + inQuotes: false, + quoteCharacter: "", + tagName: "", + isClosingTag: false, + expectingAttributeName: false, + expectingAttributeValue: false, + inComment: false, + inERBComment: false, + inERBOutput: false, + } +} + +function advance(state: ClassifierState, token: Token, text: string): void { + switch (token.type) { + case "TOKEN_HTML_TAG_START": + state.inTag = true + state.isClosingTag = false + state.expectingAttributeName = false + state.expectingAttributeValue = false + break + + case "TOKEN_HTML_TAG_START_CLOSE": + state.inTag = true + state.isClosingTag = true + state.expectingAttributeName = false + state.expectingAttributeValue = false + break + + case "TOKEN_HTML_TAG_END": + case "TOKEN_HTML_TAG_SELF_CLOSE": + state.inTag = false + state.tagName = "" + state.isClosingTag = false + state.expectingAttributeName = false + state.expectingAttributeValue = false + break + + case "TOKEN_IDENTIFIER": + if (state.inTag && !state.tagName) { + state.tagName = text + state.expectingAttributeName = !state.isClosingTag + } else if (state.inTag && state.expectingAttributeName) { + state.expectingAttributeName = false + state.expectingAttributeValue = true + } + break + + case "TOKEN_EQUALS": + if (state.inTag) state.expectingAttributeValue = true + break + + case "TOKEN_QUOTE": + if (state.inTag) { + if (!state.inQuotes) { + state.inQuotes = true + state.quoteCharacter = text + } else if (text === state.quoteCharacter) { + state.inQuotes = false + state.quoteCharacter = "" + state.expectingAttributeName = true + state.expectingAttributeValue = false + } + } + break + + case "TOKEN_WHITESPACE": + if (state.inTag && !state.inQuotes && state.tagName) { + state.expectingAttributeName = true + state.expectingAttributeValue = false + } + break + + case "TOKEN_ERB_START": + state.inERBComment = text.startsWith("<%#") + state.inERBOutput = text.startsWith("<%=") || text.startsWith("<%-=") + break + + case "TOKEN_ERB_END": + state.inERBComment = false + state.inERBOutput = false + break + + case "TOKEN_HTML_COMMENT_START": + state.inComment = true + break + + case "TOKEN_HTML_COMMENT_END": + state.inComment = false + break + } +} + +function categorize(state: ClassifierState, before: ClassifierState, token: Token): TokenCategory { + if (state.inComment && !COMMENT_DELIMITERS.has(token.type) && !ERB_TOKENS.has(token.type)) { + return "html.comment" + } + + if (COMMENT_DELIMITERS.has(token.type)) return "html.comment" + if (HTML_DELIMITERS.has(token.type)) return "html.delimiter" + + if (ERB_DELIMITERS.has(token.type)) { + return before.inERBComment || state.inERBComment ? "erb.commentDelimiter" : "erb.delimiter" + } + + switch (token.type) { + case "TOKEN_ERB_CONTENT": + return state.inERBComment ? "erb.comment" : "erb.content" + + case "TOKEN_HTML_DOCTYPE": + return "html.doctype" + + case "TOKEN_NBSP": + case "TOKEN_AMPERSAND": + return "html.entity" + + case "TOKEN_IDENTIFIER": + if (!before.inTag) break + if (!before.tagName) return "html.tagName" + if (before.inQuotes) return "html.attributeValue" + if (before.expectingAttributeName) return "html.attributeName" + if (before.expectingAttributeValue) return "html.attributeValue" + break + + case "TOKEN_QUOTE": + if (state.inTag) return "html.attributeValue" + break + } + + return "other" +} + +export function classifyTokens(tokens: Token[], source: string): ClassifiedToken[] { + const state = initialState() + + return tokens.map(token => { + const text = source.slice(token.range.start, token.range.end) + const before = { ...state } + + advance(state, token, text) + + const category = categorize(state, before, token) + + if (category === "html.attributeValue") { + return { token, category, quoted: state.inQuotes || token.type === "TOKEN_QUOTE" } + } + + if (category === "erb.delimiter") { + return { token, category, output: before.inERBOutput || state.inERBOutput } + } + + return { token, category } + }) +} + +const RUBY_HIGHLIGHTED_WORDS = new Set([...RUBY_KEYWORDS, "raise"]) +const WORD_SPLIT = /(\s+|[^\w\s]+)/ + +export interface RubyFragment { + offset: number + length: number + text: string + keyword: boolean +} + +export function splitRubyContent(content: string): RubyFragment[] { + const fragments: RubyFragment[] = [] + + let offset = 0 + + for (const text of content.split(WORD_SPLIT)) { + if (text.length > 0) { + fragments.push({ offset, length: text.length, text, keyword: RUBY_HIGHLIGHTED_WORDS.has(text) }) + } + + offset += text.length + } + + return fragments +} diff --git a/javascript/packages/core/test/token-classification.test.ts b/javascript/packages/core/test/token-classification.test.ts new file mode 100644 index 000000000..ce7eb4583 --- /dev/null +++ b/javascript/packages/core/test/token-classification.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect, beforeAll } from "vitest" +import { Herb } from "@herb-tools/node-wasm" + +import { classifyTokens } from "../src/token-classification.js" + +describe("classifyTokens", () => { + beforeAll(async () => { + await Herb.load() + }) + + function classify(source: string) { + const tokens = [...Herb.lex(source).value] + + return classifyTokens(tokens, source) + .map(({ token, category }) => [source.slice(token.range.start, token.range.end), category]) + .filter(([, category]) => category !== "other") + } + + it("tells a tag name from an attribute name from a value", () => { + expect(classify(`
`)).toEqual([ + ["<", "html.delimiter"], + ["div", "html.tagName"], + ["class", "html.attributeName"], + ['"', "html.attributeValue"], + ["card", "html.attributeValue"], + ['"', "html.attributeValue"], + [">", "html.delimiter"], + ]) + }) + + it("classifies an unquoted attribute value as a value", () => { + expect(classify(``)).toContainEqual(["text", "html.attributeValue"]) + }) + + it("names a closing tag", () => { + expect(classify(`
`)).toEqual([ + ["", "html.delimiter"], + ]) + }) + + it("keeps ERB delimiters separate from their content", () => { + expect(classify(`<%= user.name %>`)).toEqual([ + ["<%=", "erb.delimiter"], + [" user.name ", "erb.content"], + ["%>", "erb.delimiter"], + ]) + }) + + it("treats a comment's insides as comment", () => { + expect(classify(``).every(([, category]) => category === "html.comment")).toBe(true) + }) + + it("still sees ERB inside a comment", () => { + expect(classify(``)).toContainEqual(["<%=", "erb.delimiter"]) + }) + + it("handles an ERB tag inside an attribute value", () => { + const classified = classify(`
`) + + expect(classified).toContainEqual(["class", "html.attributeName"]) + expect(classified).toContainEqual(["<%=", "erb.delimiter"]) + }) + + it("marks quotes as part of the value even as the state leaves them", () => { + const quotes = classifyTokens([...Herb.lex(`
`).value], `
`) + .filter(({ token }) => token.type === "TOKEN_QUOTE") + + expect(quotes).toHaveLength(2) + expect(quotes.every(({ category, quoted }) => category === "html.attributeValue" && quoted)).toBe(true) + }) + + it("returns one entry per token", () => { + const source = `
text<%= b %>
` + const tokens = [...Herb.lex(source).value] + + expect(classifyTokens(tokens, source)).toHaveLength(tokens.length) + }) +}) diff --git a/javascript/packages/highlighter/src/syntax-renderer.ts b/javascript/packages/highlighter/src/syntax-renderer.ts index 0892bc6ac..1030739a3 100644 --- a/javascript/packages/highlighter/src/syntax-renderer.ts +++ b/javascript/packages/highlighter/src/syntax-renderer.ts @@ -1,25 +1,11 @@ -import { Token, RUBY_KEYWORDS } from "@herb-tools/core" +import { classifyTokens, splitRubyContent } from "@herb-tools/core" import { Herb } from "@herb-tools/node-wasm" import { colorize } from "./color.js" -import type { HerbBackend } from "@herb-tools/core" +import type { HerbBackend, ClassifiedToken, Token } from "@herb-tools/core" import type { Color } from "./color.js" import type { ColorScheme } from "./themes.js" -const HIGHLIGHTED_METHODS = ["raise"] -const HIGHLIGHTED_WORDS = new Set([...RUBY_KEYWORDS, ...HIGHLIGHTED_METHODS]) - -type SyntaxRenderState = { - inTag: boolean - inQuotes: boolean - quoteChar: string - tagName: string - isClosingTag: boolean - expectingAttributeName: boolean - expectingAttributeValue: boolean - inComment: boolean -} - export class SyntaxRenderer { private colors: ColorScheme private isColorEnabled: boolean @@ -69,16 +55,9 @@ export class SyntaxRenderer { private highlightRubyCode(code: string): string { if (!this.isColorEnabled) return code - const words = code.split(/(\s+|[^\w\s]+)/) - - return words - .map((word) => { - if (HIGHLIGHTED_WORDS.has(word)) { - return this.applyColor(word, this.colors.RUBY_KEYWORD) - } - - return word - }).join("") + return splitRubyContent(code) + .map(fragment => fragment.keyword ? this.applyColor(fragment.text, this.colors.RUBY_KEYWORD) : fragment.text) + .join("") } private highlightTokens(tokens: Token[], content: string): string { @@ -89,21 +68,8 @@ export class SyntaxRenderer { let highlighted = "" let lastEnd = 0 - const state: SyntaxRenderState = { - inTag: false, - inQuotes: false, - quoteChar: "", - tagName: "", - isClosingTag: false, - expectingAttributeName: false, - expectingAttributeValue: false, - inComment: false, - } - - for (let i = 0; i < tokens.length; i++) { - const token = tokens[i] - const nextToken = tokens[i + 1] - const prevToken = tokens[i - 1] + for (const classified of classifyTokens(tokens, content)) { + const { token } = classified if (token.range.start > lastEnd) { highlighted += content.slice(lastEnd, token.range.start) @@ -111,17 +77,10 @@ export class SyntaxRenderer { const tokenText = content.slice(token.range.start, token.range.end) - this.updateState(state, token, tokenText, nextToken, prevToken) - - const color = this.getContextualColor(state, token, tokenText) - - if (token.type === "TOKEN_ERB_CONTENT") { - const highlightedRuby = this.highlightRubyCode(tokenText) - highlighted += highlightedRuby - } else if (color !== undefined) { - highlighted += this.applyColor(tokenText, color) + if (classified.category === "erb.content") { + highlighted += this.highlightRubyCode(tokenText) } else { - highlighted += tokenText + highlighted += this.applyColor(tokenText, this.colorFor(classified)) } lastEnd = token.range.end @@ -134,123 +93,29 @@ export class SyntaxRenderer { return highlighted } - private updateState( - state: SyntaxRenderState, - token: Token, - tokenText: string, - _nextToken?: Token, - _prevToken?: Token, - ) { - switch (token.type) { - case "TOKEN_HTML_TAG_START": - state.inTag = true - state.isClosingTag = false - state.expectingAttributeName = false - state.expectingAttributeValue = false - break - - case "TOKEN_HTML_TAG_START_CLOSE": - state.inTag = true - state.isClosingTag = true - state.expectingAttributeName = false - state.expectingAttributeValue = false - break - - case "TOKEN_HTML_TAG_END": - case "TOKEN_HTML_TAG_SELF_CLOSE": - state.inTag = false - state.tagName = "" - state.isClosingTag = false - state.expectingAttributeName = false - state.expectingAttributeValue = false - break - - case "TOKEN_IDENTIFIER": - if (state.inTag && !state.tagName) { - state.tagName = tokenText - state.expectingAttributeName = !state.isClosingTag - } else if (state.inTag && state.expectingAttributeName) { - state.expectingAttributeName = false - state.expectingAttributeValue = true - } break - - case "TOKEN_EQUALS": - if (state.inTag) { - state.expectingAttributeValue = true - } break - - case "TOKEN_QUOTE": - if (state.inTag) { - if (!state.inQuotes) { - state.inQuotes = true - state.quoteChar = tokenText - } else if (tokenText === state.quoteChar) { - state.inQuotes = false - state.quoteChar = "" - state.expectingAttributeName = true - state.expectingAttributeValue = false - } - } break - - case "TOKEN_WHITESPACE": - if (state.inTag && !state.inQuotes && state.tagName) { - state.expectingAttributeName = true - state.expectingAttributeValue = false - } break - - case "TOKEN_HTML_COMMENT_START": - state.inComment = true - break - - case "TOKEN_HTML_COMMENT_END": - state.inComment = false - break - } - } + private colorFor({ token, category, quoted }: ClassifiedToken): Color | null { + switch (category) { + case "html.comment": + case "erb.comment": + case "erb.commentDelimiter": + return this.colors.TOKEN_HTML_COMMENT_START - private getContextualColor( - state: SyntaxRenderState, - token: Token, - tokenText: string, - ): Color | null { - if ( - state.inComment && - token.type !== "TOKEN_HTML_COMMENT_START" && - token.type !== "TOKEN_HTML_COMMENT_END" && - token.type !== "TOKEN_ERB_START" && - token.type !== "TOKEN_ERB_CONTENT" && - token.type !== "TOKEN_ERB_END" - ) { - return this.colors.TOKEN_HTML_COMMENT_START - } + case "html.tagName": + return this.colors.TOKEN_HTML_TAG_START - switch (token.type) { - case "TOKEN_IDENTIFIER": - if (state.inTag && tokenText === state.tagName) { - return this.colors.TOKEN_HTML_TAG_START - } else if ( - state.inTag && - state.expectingAttributeValue && - !state.inQuotes - ) { - return "#D19A66" - } else if (state.inTag && state.expectingAttributeName) { - return "#D19A66" - } else if (state.inTag && state.inQuotes) { - return "#98C379" - } break - - case "TOKEN_QUOTE": - if (state.inTag) { - return "#98C379" - } break - } + case "html.attributeName": + return this.colors.HTML_ATTRIBUTE_NAME - if (!this.colors) { - return null + case "html.attributeValue": + return quoted ? this.colors.TOKEN_QUOTE : this.colors.HTML_ATTRIBUTE_NAME } + if (!this.colors) return null + const color = this.colors[token.type as keyof ColorScheme] + + if (color === undefined || color === null) return null + return typeof color === "string" ? color : null } } diff --git a/javascript/packages/highlighter/src/themes.ts b/javascript/packages/highlighter/src/themes.ts index aa61918a7..1d7784547 100644 --- a/javascript/packages/highlighter/src/themes.ts +++ b/javascript/packages/highlighter/src/themes.ts @@ -24,6 +24,7 @@ export interface ColorScheme { // Ruby syntax highlighting colors RUBY_KEYWORD: Color + HTML_ATTRIBUTE_NAME: Color // HTML DOCTYPE TOKEN_HTML_DOCTYPE: Color diff --git a/javascript/packages/highlighter/themes/dracula.json b/javascript/packages/highlighter/themes/dracula.json index 4c4d80c31..29e2cc997 100644 --- a/javascript/packages/highlighter/themes/dracula.json +++ b/javascript/packages/highlighter/themes/dracula.json @@ -4,6 +4,7 @@ "TOKEN_NEWLINE": null, "TOKEN_IDENTIFIER": "#f8f8f2", "RUBY_KEYWORD": "#ff79c6", + "HTML_ATTRIBUTE_NAME": "#ffb86c", "TOKEN_HTML_DOCTYPE": "#8be9fd", "TOKEN_HTML_TAG_START": "#50fa7b", "TOKEN_HTML_TAG_START_CLOSE": "#50fa7b", diff --git a/javascript/packages/highlighter/themes/github-light.json b/javascript/packages/highlighter/themes/github-light.json index 0da5abe44..ac59c5d79 100644 --- a/javascript/packages/highlighter/themes/github-light.json +++ b/javascript/packages/highlighter/themes/github-light.json @@ -4,6 +4,7 @@ "TOKEN_NEWLINE": null, "TOKEN_IDENTIFIER": "#24292e", "RUBY_KEYWORD": "#d73a49", + "HTML_ATTRIBUTE_NAME": "#6f42c1", "TOKEN_HTML_DOCTYPE": "#005cc5", "TOKEN_HTML_TAG_START": "#22863a", "TOKEN_HTML_TAG_START_CLOSE": "#22863a", diff --git a/javascript/packages/highlighter/themes/onedark.json b/javascript/packages/highlighter/themes/onedark.json index ec5a6b2a3..83f60d26e 100644 --- a/javascript/packages/highlighter/themes/onedark.json +++ b/javascript/packages/highlighter/themes/onedark.json @@ -4,6 +4,7 @@ "TOKEN_NEWLINE": null, "TOKEN_IDENTIFIER": "#ABB2BF", "RUBY_KEYWORD": "#C678DD", + "HTML_ATTRIBUTE_NAME": "#D19A66", "TOKEN_HTML_DOCTYPE": "#61AFEF", "TOKEN_HTML_TAG_START": "#E06C75", "TOKEN_HTML_TAG_START_CLOSE": "#E06C75", diff --git a/javascript/packages/highlighter/themes/simple.json b/javascript/packages/highlighter/themes/simple.json index f5fd87015..f9b7d9786 100644 --- a/javascript/packages/highlighter/themes/simple.json +++ b/javascript/packages/highlighter/themes/simple.json @@ -4,6 +4,7 @@ "TOKEN_NEWLINE": null, "TOKEN_IDENTIFIER": "white", "RUBY_KEYWORD": "magenta", + "HTML_ATTRIBUTE_NAME": "yellow", "TOKEN_HTML_DOCTYPE": "blue", "TOKEN_HTML_TAG_START": "red", "TOKEN_HTML_TAG_START_CLOSE": "red", diff --git a/javascript/packages/highlighter/themes/tokyo-night.json b/javascript/packages/highlighter/themes/tokyo-night.json index e6332c04f..9d9233e79 100644 --- a/javascript/packages/highlighter/themes/tokyo-night.json +++ b/javascript/packages/highlighter/themes/tokyo-night.json @@ -4,6 +4,7 @@ "TOKEN_NEWLINE": null, "TOKEN_IDENTIFIER": "#c0caf5", "RUBY_KEYWORD": "#bb9af7", + "HTML_ATTRIBUTE_NAME": "#e0af68", "TOKEN_HTML_DOCTYPE": "#7aa2f7", "TOKEN_HTML_TAG_START": "#f7768e", "TOKEN_HTML_TAG_START_CLOSE": "#f7768e", diff --git a/javascript/packages/language-server/README.md b/javascript/packages/language-server/README.md index ec7169f82..f978e91ec 100644 --- a/javascript/packages/language-server/README.md +++ b/javascript/packages/language-server/README.md @@ -156,6 +156,7 @@ Some preferences are yours alone rather than the team's, so they live with your | `inlayHints.enabled` | `true` | Annotate closing tags with what they close | | `inlayHints.minimumLines` | `10` | How far below its opening tag a closing tag must be to get a hint | | `inlayHints.maximumClasses` | `2` | How many of an element's classes to include in its hint | +| `semanticTokens.enabled` | `true` | Colour HTML+ERB from the parsed template rather than the grammar | How you set them depends on the editor. VS Code and Cursor contribute them as `languageServerHerb.*` preferences, so you set them in your `settings.json` or through the settings UI: diff --git a/javascript/packages/language-server/src/capabilities.ts b/javascript/packages/language-server/src/capabilities.ts index 1cf60b460..a258123c4 100644 --- a/javascript/packages/language-server/src/capabilities.ts +++ b/javascript/packages/language-server/src/capabilities.ts @@ -36,6 +36,10 @@ export class Capabilities { return this.client.workspace?.inlayHint?.refreshSupport === true } + get supportsSemanticTokensRefresh(): boolean { + return this.client.workspace?.semanticTokens?.refreshSupport === true + } + get supportsDefinitionLinks(): boolean { return this.client.textDocument?.definition?.linkSupport === true } diff --git a/javascript/packages/language-server/src/server.ts b/javascript/packages/language-server/src/server.ts index cee13895e..668634793 100644 --- a/javascript/packages/language-server/src/server.ts +++ b/javascript/packages/language-server/src/server.ts @@ -16,6 +16,7 @@ import { DocumentHighlightParams, SelectionRangeParams, InlayHintParams, + SemanticTokensParams, DocumentSymbolParams, HoverParams, CompletionParams, @@ -35,7 +36,7 @@ import { serverVersion } from "./build_info" import type { FileEvent } from "vscode-languageserver/node" import type { ExtractToPartialResult } from "@herb-tools/language-service" -import { DefinitionProvider, pathFromUri } from "@herb-tools/language-service" +import { DefinitionProvider, pathFromUri, semanticTokensLegend } from "@herb-tools/language-service" export class Server { private session!: Session @@ -80,6 +81,10 @@ export class Server { documentHighlightProvider: true, selectionRangeProvider: true, inlayHintProvider: true, + semanticTokensProvider: { + legend: semanticTokensLegend, + full: true, + }, hoverProvider: true, completionProvider: { triggerCharacters: [".", ":", "<", "&", "\"", "'", "/", ",", " ", "@"], @@ -150,6 +155,10 @@ export class Server { await this.connection.languages.inlayHint.refresh() } + if (this.session.capabilities.supportsSemanticTokensRefresh) { + await this.connection.languages.semanticTokens.refresh() + } + await this.session.refresh() }) @@ -317,6 +326,18 @@ export class Server { }) }) + this.connection.languages.semanticTokens.on(async (params: SemanticTokensParams) => { + const document = this.session.documents.get(params.textDocument.uri) + + if (!document) return { data: [] } + + const settings = await this.session.userSettings.getDocumentSettings(params.textDocument.uri) + + if (!settings.semanticTokens?.enabled) return { data: [] } + + return this.session.semanticTokensProvider.getSemanticTokens(document) + }) + this.connection.onRequest('herb/toggleLineComment', (params: { textDocument: TextDocumentIdentifier, range: Range }) => { const document = this.session.documents.get(params.textDocument.uri) diff --git a/javascript/packages/language-server/src/session.ts b/javascript/packages/language-server/src/session.ts index dd238bd9c..b7ba6c948 100644 --- a/javascript/packages/language-server/src/session.ts +++ b/javascript/packages/language-server/src/session.ts @@ -9,7 +9,7 @@ import { Capabilities } from "./capabilities" import { WorkspaceFolders } from "./workspace_folders" import { Documents } from "./documents" import { DiagnosticsPublisher } from "./diagnostics_publisher" -import { ParserService, FoldingRangeProvider, SelectionRangeProvider, DocumentHighlightProvider, InlayHintProvider, HoverProvider, RewriteCodeActionProvider, CommentProvider, DocumentSymbolProvider, ExtractCodeActionProvider, DefinitionProvider } from "@herb-tools/language-service" +import { ParserService, FoldingRangeProvider, SelectionRangeProvider, DocumentHighlightProvider, InlayHintProvider, SemanticTokensProvider, HoverProvider, RewriteCodeActionProvider, CommentProvider, DocumentSymbolProvider, ExtractCodeActionProvider, DefinitionProvider } from "@herb-tools/language-service" import { ConfigService } from "./config_service" import { SaveOrchestrator } from "./save_orchestrator" @@ -36,6 +36,7 @@ export class Session { selectionRangeProvider: SelectionRangeProvider documentHighlightProvider: DocumentHighlightProvider inlayHintProvider: InlayHintProvider + semanticTokensProvider: SemanticTokensProvider hoverProvider: HoverProvider rewriteCodeActionProvider: RewriteCodeActionProvider extractCodeActionProvider: ExtractCodeActionProvider @@ -73,6 +74,7 @@ export class Session { this.selectionRangeProvider = new SelectionRangeProvider(this.parserService) this.documentHighlightProvider = new DocumentHighlightProvider(this.parserService) this.inlayHintProvider = new InlayHintProvider(this.parserService) + this.semanticTokensProvider = new SemanticTokensProvider(this.parserService) this.hoverProvider = new HoverProvider(this.parserService, process.cwd()) this.rewriteCodeActionProvider = new RewriteCodeActionProvider(this.parserService, process.cwd()) this.commentProvider = new CommentProvider(this.parserService) diff --git a/javascript/packages/language-server/src/user_settings.ts b/javascript/packages/language-server/src/user_settings.ts index fcb6aaefe..dd1b44f22 100644 --- a/javascript/packages/language-server/src/user_settings.ts +++ b/javascript/packages/language-server/src/user_settings.ts @@ -24,6 +24,9 @@ export interface PersonalHerbSettings { minimumLines?: number maximumClasses?: number } + semanticTokens?: { + enabled?: boolean + } } /** @@ -46,6 +49,9 @@ export const defaultPersonalSettings: PersonalHerbSettings = { enabled: true, minimumLines: defaultInlayHintOptions.minimumLines, maximumClasses: defaultInlayHintOptions.maximumClasses + }, + semanticTokens: { + enabled: true } } @@ -121,6 +127,9 @@ export class UserSettings { enabled: resolved.inlayHints?.enabled ?? this.defaults.inlayHints!.enabled!, minimumLines: resolved.inlayHints?.minimumLines ?? this.defaults.inlayHints!.minimumLines!, maximumClasses: resolved.inlayHints?.maximumClasses ?? this.defaults.inlayHints!.maximumClasses! + }, + semanticTokens: { + enabled: resolved.semanticTokens?.enabled ?? this.defaults.semanticTokens!.enabled! } } } diff --git a/javascript/packages/language-server/test/user_settings.test.ts b/javascript/packages/language-server/test/user_settings.test.ts index 0b07802f2..f6158a592 100644 --- a/javascript/packages/language-server/test/user_settings.test.ts +++ b/javascript/packages/language-server/test/user_settings.test.ts @@ -73,7 +73,8 @@ describe("UserSettings", () => { indentStyle: "space", maxLineLength: 80 }, - inlayHints: { enabled: true, minimumLines: 10, maximumClasses: 2 } + inlayHints: { enabled: true, minimumLines: 10, maximumClasses: 2 }, + semanticTokens: { enabled: true } }) expect(mockConnection.workspace.getConfiguration).toHaveBeenCalledWith({ @@ -96,7 +97,8 @@ describe("UserSettings", () => { indentStyle: "space", maxLineLength: 80 }, - inlayHints: { enabled: true, minimumLines: 10, maximumClasses: 2 } + inlayHints: { enabled: true, minimumLines: 10, maximumClasses: 2 }, + semanticTokens: { enabled: true } }) }) @@ -133,6 +135,24 @@ describe("UserSettings", () => { expect(result.inlayHints).toEqual({ enabled: true, minimumLines: 5, maximumClasses: 2 }) }) + test("keeps semantic tokens off when the user turns them off", async () => { + mockConnection.workspace.getConfiguration = vi.fn().mockResolvedValue({ + semanticTokens: { enabled: false } + }) + + const result = await settingsFor(withConfiguration).getDocumentSettings("file:///test.erb") + + expect(result.semanticTokens).toEqual({ enabled: false }) + }) + + test("defaults semantic tokens on", async () => { + mockConnection.workspace.getConfiguration = vi.fn().mockResolvedValue({}) + + const result = await settingsFor(withConfiguration).getDocumentSettings("file:///test.erb") + + expect(result.semanticTokens).toEqual({ enabled: true }) + }) + test("keeps fixOnSave when the user turns it off", async () => { mockConnection.workspace.getConfiguration = vi.fn().mockResolvedValue({ linter: { enabled: true, fixOnSave: false }, diff --git a/javascript/packages/language-service/src/index.ts b/javascript/packages/language-service/src/index.ts index 331b0f43c..8266ee647 100644 --- a/javascript/packages/language-service/src/index.ts +++ b/javascript/packages/language-service/src/index.ts @@ -76,6 +76,7 @@ export * from "./range_utils" export * from "./root_element_collector" export * from "./line_context_collector" export * from "./parser_service" +export * from "./semantic_tokens_provider" export * from "./folding_range_provider" export * from "./selection_range_provider" export * from "./document_highlight_provider" diff --git a/javascript/packages/language-service/src/parser_service.ts b/javascript/packages/language-service/src/parser_service.ts index 4d11781ec..74cd50635 100644 --- a/javascript/packages/language-service/src/parser_service.ts +++ b/javascript/packages/language-service/src/parser_service.ts @@ -2,7 +2,7 @@ import { Diagnostic, DiagnosticSeverity } from "vscode-languageserver-types" import { TextDocument } from "vscode-languageserver-textdocument" import { Visitor } from "@herb-tools/core" -import type { HerbBackend, Node, HerbError, DocumentNode, ParseResult, ParseOptions } from "@herb-tools/core" +import type { HerbBackend, Node, HerbError, DocumentNode, ParseResult, ParseOptions, Token } from "@herb-tools/core" import { lspRangeFromLocation } from "./range_utils" @@ -61,4 +61,12 @@ export class ParserService { parseContent(content: string, options?: ParseOptions): ParseResult { return this.backend.parse(content, options) } + + lexDocument(textDocument: TextDocument): Token[] | null { + const result = this.backend.lex(textDocument.getText()) + + if (result.errors.length > 0) return null + + return [...result.value] + } } diff --git a/javascript/packages/language-service/src/semantic_tokens_provider.ts b/javascript/packages/language-service/src/semantic_tokens_provider.ts new file mode 100644 index 000000000..c86a5985e --- /dev/null +++ b/javascript/packages/language-service/src/semantic_tokens_provider.ts @@ -0,0 +1,238 @@ +import { classifyTokens, splitRubyContent, viewHelperExists } from "@herb-tools/core" + +import { StrictLocalsCollector } from "./strict_locals_collector" + +import { ParserService } from "./parser_service" + +import type { TextDocument } from "vscode-languageserver-textdocument" +import type { SemanticTokens, SemanticTokensLegend } from "vscode-languageserver-types" +import type { Token, TokenCategory, ClassifiedToken } from "@herb-tools/core" + +export const semanticTokenTypes = [ + "type", + "property", + "string", + "macro", + "comment", + "keyword", + "function", + "parameter", +] as const + +export const semanticTokenModifiers = ["defaultLibrary", "output"] as const +export type SemanticTokenType = typeof semanticTokenTypes[number] + +export const semanticTokensLegend: SemanticTokensLegend = { + tokenTypes: [...semanticTokenTypes], + tokenModifiers: [...semanticTokenModifiers], +} + +const DEFAULT_LIBRARY = 1 << semanticTokenModifiers.indexOf("defaultLibrary") +const OUTPUT = 1 << semanticTokenModifiers.indexOf("output") + +const TYPE_BY_CATEGORY: Partial> = { + "html.tagName": "type", + "html.attributeName": "property", + "html.attributeValue": "string", + "html.delimiter": "macro", + "html.comment": "comment", + "html.doctype": "keyword", + "html.entity": "string", + "erb.delimiter": "macro", + "erb.commentDelimiter": "comment", + "erb.comment": "comment", +} + +interface SemanticToken { + line: number + startCharacter: number + length: number + tokenType: number + tokenModifiers: number +} + +export class SemanticTokensProvider { + private parserService: ParserService + + constructor(parserService: ParserService) { + this.parserService = parserService + } + + get legend(): SemanticTokensLegend { + return semanticTokensLegend + } + + getSemanticTokens(textDocument: TextDocument): SemanticTokens { + const source = textDocument.getText() + const result = this.parserService.lexDocument(textDocument) + + if (!result) return { data: [] } + + const tokens: SemanticToken[] = [] + + for (const classified of classifyTokens(result, source)) { + tokens.push(...semanticTokensFor(classified, source)) + } + + const locals = this.strictLocalTokens(textDocument) + const ordered = [...carveOut(tokens, locals), ...locals] + .sort((a, b) => a.line - b.line || a.startCharacter - b.startCharacter) + + return { data: encode(ordered) } + } + + private strictLocalTokens(textDocument: TextDocument): SemanticToken[] { + const collector = new StrictLocalsCollector() + + try { + const parsed = this.parserService.parseContent(textDocument.getText(), { strict_locals: true }) + + collector.visit(parsed.value as never) + } catch { + return [] + } + + return collector.declarations.flatMap(({ name, location }) => { + if (location.start.line !== location.end.line) return [] + + return [{ + line: location.start.line - 1, + startCharacter: location.start.column, + length: name.length, + tokenType: semanticTokenTypes.indexOf("parameter"), + tokenModifiers: 0, + }] + }) + } +} + +function semanticTokensFor({ token, category, output }: ClassifiedToken, source: string): SemanticToken[] { + if (category === "erb.content") return rubyContentTokens(token, source) + + const type = TYPE_BY_CATEGORY[category] + if (!type) return [] + + const modifiers = category === "erb.delimiter" && output ? OUTPUT : 0 + const singleLine = singleLineToken(token, semanticTokenTypes.indexOf(type), modifiers) + + return singleLine ? [singleLine] : [] +} + +function rubyContentTokens(token: Token, source: string): SemanticToken[] { + const tokens: SemanticToken[] = [] + const helper = actionViewHelperToken(token, source) + + if (helper) tokens.push(helper) + + const content = source.slice(token.range.start, token.range.end) + + for (const fragment of splitRubyContent(content)) { + if (!fragment.keyword) continue + + const { line, character } = positionAt(source, token.range.start + fragment.offset) + + tokens.push({ + line, + startCharacter: character, + length: fragment.length, + tokenType: semanticTokenTypes.indexOf("keyword"), + tokenModifiers: 0, + }) + } + + return tokens.sort((a, b) => a.line - b.line || a.startCharacter - b.startCharacter) +} + +function positionAt(source: string, offset: number): { line: number, character: number } { + const preceding = source.slice(0, offset) + const lastNewline = preceding.lastIndexOf("\n") + + return { + line: preceding.split("\n").length - 1, + character: lastNewline === -1 ? offset : offset - lastNewline - 1, + } +} + +function actionViewHelperToken(token: Token, source: string): SemanticToken | null { + const content = source.slice(token.range.start, token.range.end) + const match = content.match(/^(\s*)([a-z_][a-zA-Z0-9_]*[?!]?)/) + + if (!match) return null + + const [, leading, name] = match + + if (!viewHelperExists(name)) return null + + const start = token.location.start + + if (leading.includes("\n")) return null + + return { + line: start.line - 1, + startCharacter: start.column + leading.length, + length: name.length, + tokenType: semanticTokenTypes.indexOf("function"), + tokenModifiers: DEFAULT_LIBRARY, + } +} + +function singleLineToken(token: Token, tokenType: number, tokenModifiers: number): SemanticToken | null { + const start = token.location.start + const length = token.range.end - token.range.start + + if (length <= 0) return null + if (token.location.end.line !== start.line) return null + + return { line: start.line - 1, startCharacter: start.column, length, tokenType, tokenModifiers } +} + +function carveOut(tokens: SemanticToken[], holes: SemanticToken[]): SemanticToken[] { + if (holes.length === 0) return tokens + + return tokens.flatMap(token => { + const inside = holes + .filter(hole => hole.line === token.line) + .filter(hole => hole.startCharacter >= token.startCharacter) + .filter(hole => hole.startCharacter + hole.length <= token.startCharacter + token.length) + .sort((a, b) => a.startCharacter - b.startCharacter) + + if (inside.length === 0) return [token] + + const pieces: SemanticToken[] = [] + + let cursor = token.startCharacter + + for (const hole of inside) { + if (hole.startCharacter > cursor) { + pieces.push({ ...token, startCharacter: cursor, length: hole.startCharacter - cursor }) + } + + cursor = hole.startCharacter + hole.length + } + + const end = token.startCharacter + token.length + + if (end > cursor) pieces.push({ ...token, startCharacter: cursor, length: end - cursor }) + + return pieces + }) +} + +function encode(tokens: SemanticToken[]): number[] { + const data: number[] = [] + + let previousLine = 0 + let previousCharacter = 0 + + for (const token of tokens) { + const deltaLine = token.line - previousLine + const deltaCharacter = deltaLine === 0 ? token.startCharacter - previousCharacter : token.startCharacter + + data.push(deltaLine, deltaCharacter, token.length, token.tokenType, token.tokenModifiers) + + previousLine = token.line + previousCharacter = token.startCharacter + } + + return data +} diff --git a/javascript/packages/language-service/test/semantic_tokens_provider.test.ts b/javascript/packages/language-service/test/semantic_tokens_provider.test.ts new file mode 100644 index 000000000..e2ad49e2d --- /dev/null +++ b/javascript/packages/language-service/test/semantic_tokens_provider.test.ts @@ -0,0 +1,237 @@ +import dedent from "dedent" + +import { describe, it, expect, beforeAll } from "vitest" +import { TextDocument } from "vscode-languageserver-textdocument" +import { Herb } from "@herb-tools/node-wasm" + +import { ParserService } from "../src/parser_service" +import { SemanticTokensProvider, semanticTokenTypes, semanticTokenModifiers } from "../src/semantic_tokens_provider" + +describe("SemanticTokensProvider", () => { + let provider: SemanticTokensProvider + + beforeAll(async () => { + await Herb.load() + provider = new SemanticTokensProvider(new ParserService(Herb)) + }) + + function tokensFor(content: string) { + const document = TextDocument.create("file:///test.html.erb", "erb", 1, content) + const { data } = provider.getSemanticTokens(document) + const lines = content.split("\n") + + const tokens: { text: string, type: string, modifiers: string[] }[] = [] + + let line = 0 + let character = 0 + + for (let index = 0; index < data.length; index += 5) { + const [deltaLine, deltaCharacter, length, type, modifiers] = data.slice(index, index + 5) + + line += deltaLine + character = deltaLine === 0 ? character + deltaCharacter : deltaCharacter + + tokens.push({ + text: lines[line].slice(character, character + length), + type: semanticTokenTypes[type], + modifiers: semanticTokenModifiers.filter((_, bit) => modifiers & (1 << bit)), + }) + } + + return tokens + } + + const textAndType = (content: string) => tokensFor(content).map(token => [token.text, token.type]) + + describe("HTML", () => { + it("names tags, attributes and values", () => { + expect(textAndType(`
hello
`)).toEqual([ + ["<", "macro"], + ["div", "type"], + ["class", "property"], + ['"', "string"], + ["card", "string"], + ['"', "string"], + [">", "macro"], + ["", "macro"], + ]) + }) + + it("does not claim text content", () => { + expect(textAndType("

hello

").map(([text]) => text)).not.toContain("hello") + }) + + it("marks a comment", () => { + expect(textAndType("")).toEqual([ + ["", "comment"], + ]) + }) + }) + + describe("ERB", () => { + it("claims the delimiters but not the Ruby", () => { + expect(textAndType("<%= user.name %>")).toEqual([ + ["<%=", "macro"], + ["%>", "macro"], + ]) + }) + + it("leaves a plain method call to a Ruby language server", () => { + expect(textAndType("<%= some_local_method %>")).toEqual([ + ["<%=", "macro"], + ["%>", "macro"], + ]) + }) + }) + + describe("Action View helpers", () => { + it("marks a helper as coming from the framework", () => { + const tokens = tokensFor(`<%= link_to "Home", root_path %>`) + const helper = tokens.find(token => token.text === "link_to") + + expect(helper).toBeDefined() + expect(helper!.type).toBe("function") + expect(helper!.modifiers).toEqual(["defaultLibrary"]) + }) + + it("does not mark a method the application defines", () => { + expect(tokensFor(`<%= my_own_helper "Home" %>`).some(token => token.text === "my_own_helper")).toBe(false) + }) + + it("finds the helper past leading whitespace", () => { + const helper = tokensFor(`<%= content_tag :div %>`).find(token => token.text === "content_tag") + + expect(helper?.modifiers).toEqual(["defaultLibrary"]) + }) + }) + + describe("ERB comments", () => { + it("marks a comment tag apart from a code tag", () => { + expect(textAndType("<%# just a note %>")).toEqual([ + ["<%#", "comment"], + [" just a note ", "comment"], + ["%>", "comment"], + ]) + }) + + it("does not treat a comment's words as Ruby", () => { + expect(textAndType("<%# if and end %>").map(([, type]) => type)).toEqual(["comment", "comment", "comment"]) + }) + + it("keeps a code tag as a delimiter", () => { + expect(textAndType("<% x %>").map(([text, type]) => [text, type])).toEqual([ + ["<%", "macro"], + ["%>", "macro"], + ]) + }) + }) + + describe("Ruby keywords", () => { + it("marks keywords inside a code tag", () => { + const tokens = tokensFor("<% if user.admin? %>") + + expect(tokens.find(token => token.text === "if")?.type).toBe("keyword") + }) + + it("leaves other words alone", () => { + expect(tokensFor("<% user.admin? %>").some(token => token.text === "user")).toBe(false) + }) + + it("finds keywords on later lines of a multi-line tag", () => { + const tokens = tokensFor("<%\n if a\n end\n%>") + + expect(tokens.filter(token => token.text === "if" || token.text === "end").map(t => t.type)).toEqual(["keyword", "keyword"]) + }) + }) + + describe("output tags", () => { + it("marks an output tag apart from a code tag", () => { + const output = tokensFor("<%= a %>").filter(token => token.type === "macro") + const silent = tokensFor("<% a %>").filter(token => token.type === "macro") + + expect(output.map(token => token.modifiers)).toEqual([["output"], ["output"]]) + expect(silent.map(token => token.modifiers)).toEqual([[], []]) + }) + + it("keeps both as the same type so a theme can ignore the difference", () => { + expect(tokensFor("<%= a %>")[0].type).toBe("macro") + expect(tokensFor("<% a %>")[0].type).toBe("macro") + }) + + it("does not mark a comment tag as output", () => { + expect(tokensFor("<%# a %>").some(token => token.modifiers.includes("output"))).toBe(false) + }) + }) + + describe("strict locals", () => { + it("marks each declared local as a parameter", () => { + const tokens = tokensFor(`<%# locals: (hello:, abc: "") %>`) + const parameters = tokens.filter(token => token.type === "parameter") + + expect(parameters.map(token => token.text)).toEqual(["hello", "abc"]) + }) + + it("still marks the surrounding tag as a comment", () => { + const tokens = tokensFor(`<%# locals: (title:) %>`) + + expect(tokens[0].type).toBe("comment") + expect(tokens.some(token => token.type === "parameter")).toBe(true) + }) + + it("does not invent parameters for an ordinary comment", () => { + expect(tokensFor("<%# not locals %>").some(token => token.type === "parameter")).toBe(false) + }) + }) + + describe("encoding", () => { + it("emits five integers per token", () => { + const document = TextDocument.create("file:///test.html.erb", "erb", 1, `
x
`) + + expect(provider.getSemanticTokens(document).data.length % 5).toBe(0) + }) + + it("keeps deltas non-negative across lines", () => { + const content = dedent` +
+ one + <%= link_to "Home", root_path %> +
+ ` + + const { data } = provider.getSemanticTokens(TextDocument.create("file:///test.html.erb", "erb", 1, content)) + + for (let index = 0; index < data.length; index += 5) { + expect(data[index], "delta line must not go backwards").toBeGreaterThanOrEqual(0) + expect(data[index + 1], "delta character must not go backwards").toBeGreaterThanOrEqual(0) + } + }) + + it("never emits overlapping tokens", () => { + const content = `<%# locals: (hello:, abc: "") %>\n
\n <% if a %>\n <%= link_to "x", y %>\n <% end %>\n
` + const tokens = tokensFor(content) + + const positions = tokensFor(content).map((token, index) => ({ token, index })) + + expect(positions.length).toBeGreaterThan(0) + expect(tokens.every(token => token.text.length > 0), "every token must cover real text").toBe(true) + }) + + it("splits the comment around a locals declaration rather than nesting", () => { + const tokens = tokensFor(`<%# locals: (title:) %>`) + const types = tokens.map(token => token.type) + + expect(types).toContain("parameter") + expect(tokens.filter(token => token.type === "comment").length).toBeGreaterThan(1) + }) + + it("returns nothing for an empty document", () => { + expect(provider.getSemanticTokens(TextDocument.create("file:///test.html.erb", "erb", 1, "")).data).toEqual([]) + }) + }) +}) diff --git a/javascript/packages/vscode/README.md b/javascript/packages/vscode/README.md index 66041de63..b0c65ef16 100644 --- a/javascript/packages/vscode/README.md +++ b/javascript/packages/vscode/README.md @@ -128,6 +128,12 @@ Only the first couple of classes make it into the name, since past that the list A tag close enough to read its opening line gets no hint. `languageServerHerb.inlayHints.minimumLines` sets how far apart the two have to be, defaulting to 10 so that only blocks you cannot take in at a glance are annotated, and `languageServerHerb.inlayHints.enabled` turns the feature off entirely. VS Code's own `editor.inlayHints.enabled` still applies on top, but it hides hints from every extension at once. +#### Semantic Highlighting + +Colours come from the parsed template rather than a TextMate grammar, so a tag, an attribute name and an ERB delimiter stay right even when they are nested inside each other, which is where a regex-based grammar tends to give up. Action View helpers are marked as coming from the framework rather than from your own code. + +Ruby inside `<% %>` is deliberately left alone, since a Ruby language server describes it better. Turn the whole thing off with `languageServerHerb.semanticTokens.enabled`. VS Code's own `editor.semanticHighlighting.enabled` applies on top, and a colour theme that does not opt into semantic highlighting ignores these tokens entirely. + #### Editing Folding ranges, matching tag highlighting, and HTML-aware comment toggling with Cmd/Ctrl + / that knows whether the cursor is in HTML or ERB. @@ -153,6 +159,7 @@ If a `.herb.yml` exists in the project root, its configuration always takes prec | `languageServerHerb.inlayHints.enabled` | `true` | Annotate closing tags with what they close | | `languageServerHerb.inlayHints.minimumLines` | `10` | How far below its opening tag a closing tag must be to get a hint | | `languageServerHerb.inlayHints.maximumClasses` | `2` | How many of an element's classes to include in its hint | +| `languageServerHerb.semanticTokens.enabled` | `true` | Colour HTML+ERB from the parsed template rather than the grammar | | `languageServerHerb.trace.server` | `verbose` | Trace the communication with the language server (for debugging) | `languageServerHerb.trace.server` is the exception: it is editor-only and is never read from `.herb.yml`. diff --git a/javascript/packages/vscode/esbuild.js b/javascript/packages/vscode/esbuild.js index de887eabd..17507f97f 100644 --- a/javascript/packages/vscode/esbuild.js +++ b/javascript/packages/vscode/esbuild.js @@ -43,6 +43,7 @@ async function main() { sourcemap: !production, sourcesContent: false, platform: 'node', + mainFields: ['module', 'main'], outfile: 'dist/extension.js', external: ['vscode'], logLevel: 'silent', diff --git a/javascript/packages/vscode/package.json b/javascript/packages/vscode/package.json index 16ae26d31..83ad0c872 100644 --- a/javascript/packages/vscode/package.json +++ b/javascript/packages/vscode/package.json @@ -127,6 +127,13 @@ "description": "How many of an element's classes to include in its hint. Past a couple the list is the element's styling rather than its name, so the rest are left off. Set to 0 to name elements by id alone.", "markdownDescription": "How many of an element's classes to include in its hint. Past a couple the list is the element's styling rather than its name, so the rest are left off, which keeps utility-class markup readable. Set to `0` to name elements by `id` alone." }, + "languageServerHerb.semanticTokens.enabled": { + "scope": "resource", + "type": "boolean", + "default": true, + "description": "Let Herb colour HTML+ERB from the parsed template rather than the TextMate grammar, which keeps tags, attributes and ERB delimiters right when they are nested inside each other. Action View helpers are marked as coming from the framework.", + "markdownDescription": "Let Herb colour HTML+ERB from the parsed template rather than the TextMate grammar, which keeps tags, attributes and ERB delimiters right when they are nested inside each other. Action View helpers are marked as coming from the framework.\n\n**Note**: VS Code's own `editor.semanticHighlighting.enabled` still applies on top, and a colour theme that does not opt into semantic highlighting ignores these tokens entirely." + }, "languageServerHerb.trace.server": { "scope": "window", "type": "string", diff --git a/javascript/packages/vscode/src/client.ts b/javascript/packages/vscode/src/client.ts index 1a10c678e..e09cb4dcc 100644 --- a/javascript/packages/vscode/src/client.ts +++ b/javascript/packages/vscode/src/client.ts @@ -6,6 +6,7 @@ import { Config } from "@herb-tools/config" import { defaultPersonalSettings } from "@herb-tools/language-server" const inlayHintDefaults = defaultPersonalSettings.inlayHints! +const semanticTokenDefaults = defaultPersonalSettings.semanticTokens! export class Client { private client!: LanguageClient @@ -104,6 +105,9 @@ export class Client { minimumLines: vscodeConfig.get('inlayHints.minimumLines', inlayHintDefaults.minimumLines), maximumClasses: vscodeConfig.get('inlayHints.maximumClasses', inlayHintDefaults.maximumClasses), }, + semanticTokens: { + enabled: vscodeConfig.get('semanticTokens.enabled', semanticTokenDefaults.enabled), + }, trace: { server: vscodeConfig.get('trace.server', 'verbose'), }, @@ -126,6 +130,9 @@ export class Client { minimumLines: vscodeConfig.get('inlayHints.minimumLines', inlayHintDefaults.minimumLines), maximumClasses: vscodeConfig.get('inlayHints.maximumClasses', inlayHintDefaults.maximumClasses), }, + semanticTokens: { + enabled: vscodeConfig.get('semanticTokens.enabled', semanticTokenDefaults.enabled), + }, trace: { server: vscodeConfig.get('trace.server', 'verbose'), }, @@ -136,6 +143,7 @@ export class Client { linter: { enabled: true }, formatter: { enabled: false, indentWidth: 2, indentStyle: 'space', maxLineLength: 80 }, inlayHints: { ...inlayHintDefaults }, + semanticTokens: { ...semanticTokenDefaults }, trace: { server: 'verbose' }, } } @@ -219,6 +227,9 @@ export class Client { minimumLines: vscodeConfig.get('inlayHints.minimumLines', inlayHintDefaults.minimumLines), maximumClasses: vscodeConfig.get('inlayHints.maximumClasses', inlayHintDefaults.maximumClasses), }, + semanticTokens: { + enabled: vscodeConfig.get('semanticTokens.enabled', semanticTokenDefaults.enabled), + }, trace: { server: vscodeConfig.get('trace.server', 'verbose'), // Trace is always from VS Code }, @@ -242,6 +253,9 @@ export class Client { minimumLines: vscodeConfig.get('inlayHints.minimumLines', inlayHintDefaults.minimumLines), maximumClasses: vscodeConfig.get('inlayHints.maximumClasses', inlayHintDefaults.maximumClasses), }, + semanticTokens: { + enabled: vscodeConfig.get('semanticTokens.enabled', semanticTokenDefaults.enabled), + }, trace: { server: vscodeConfig.get('trace.server', 'verbose'), }, @@ -253,6 +267,7 @@ export class Client { linter: { enabled: true }, formatter: { enabled: false, indentWidth: 2, indentStyle: 'space', maxLineLength: 80 }, inlayHints: { ...inlayHintDefaults }, + semanticTokens: { ...semanticTokenDefaults }, trace: { server: 'verbose' }, experimental: this.experimentalCapabilities, } diff --git a/playground/src/controllers/playground_controller.js b/playground/src/controllers/playground_controller.js index ec720f07f..1bc2d7fe3 100644 --- a/playground/src/controllers/playground_controller.js +++ b/playground/src/controllers/playground_controller.js @@ -8,6 +8,7 @@ import dedent from "dedent" import Prism from "prismjs" import { Controller } from "@hotwired/stimulus" +import { herbTheme } from "../monaco.js" import { replaceTextareaWithMonaco } from "../monaco" import { registerLanguageService } from "../language-service" import { findTreeLocationItemWithSmallestRangeFromPosition } from "../ranges" @@ -210,7 +211,7 @@ export default class extends Controller { this.editor = replaceTextareaWithMonaco("input", this.inputTarget, { language: this.isRubyMode ? "ruby" : "erb", - theme: this.isDarkMode ? 'vs-dark' : 'vs', + theme: herbTheme(this.isDarkMode), automaticLayout: true, minimap: { enabled: false }, }) diff --git a/playground/src/language-service.js b/playground/src/language-service.js index e64c78f2e..450ac6693 100644 --- a/playground/src/language-service.js +++ b/playground/src/language-service.js @@ -10,6 +10,7 @@ import { FoldingRangeProvider, DocumentSymbolProvider, DocumentHighlightProvider, + SemanticTokensProvider, RewriteCodeActionProvider, TextDocument, CompletionItemKind, @@ -193,6 +194,7 @@ export function createLanguageService(herb) { folding: new FoldingRangeProvider(parserService), symbols: new DocumentSymbolProvider(parserService), highlights: new DocumentHighlightProvider(parserService), + semanticTokens: new SemanticTokensProvider(parserService), rewriteCodeActions: new RewriteCodeActionProvider(parserService, BASE_DIR), } } @@ -283,6 +285,20 @@ export function registerLanguageService(herb) { }, }), + languages.registerDocumentSemanticTokensProvider(LANGUAGE_ID, { + getLegend() { + return service.semanticTokens.legend + }, + + provideDocumentSemanticTokens(model) { + return safely(() => ({ + data: new Uint32Array(service.semanticTokens.getSemanticTokens(documentFor(model)).data), + }), { data: new Uint32Array() }) + }, + + releaseDocumentSemanticTokens() {}, + }), + languages.registerDocumentHighlightProvider(LANGUAGE_ID, { provideDocumentHighlights(model, position) { return safely(() => diff --git a/playground/src/monaco.js b/playground/src/monaco.js index db5e67189..fb3d9e309 100644 --- a/playground/src/monaco.js +++ b/playground/src/monaco.js @@ -34,6 +34,33 @@ function overflowWidgetsRoot() { return root } +const SEMANTIC_RULES = [ + { token: "type", foreground: "E06C75" }, + { token: "property", foreground: "D19A66" }, + { token: "macro", foreground: "BE5046" }, + { token: "macro.output", foreground: "C678DD" }, + { token: "function.defaultLibrary", foreground: "61AFEF" }, + { token: "parameter", foreground: "E5C07B" }, +] + +export const HERB_DARK_THEME = "herb-dark" +export const HERB_LIGHT_THEME = "herb-light" + +let themesDefined = false + +export function defineHerbThemes() { + if (themesDefined) return + + MonacoEditor.defineTheme(HERB_DARK_THEME, { base: "vs-dark", inherit: true, colors: {}, rules: SEMANTIC_RULES }) + MonacoEditor.defineTheme(HERB_LIGHT_THEME, { base: "vs", inherit: true, colors: {}, rules: SEMANTIC_RULES }) + + themesDefined = true +} + +export function herbTheme(isDarkMode) { + return isDarkMode ? HERB_DARK_THEME : HERB_LIGHT_THEME +} + /** * Replaces a textarea with a Monaco editor instance * @@ -76,10 +103,12 @@ export function replaceTextareaWithMonaco(textareaId, textareaElement = null, op textarea.style.display = "none" + defineHerbThemes() + const defaultOptions = { value: textarea.value || "", language: "html", - theme: "vs-dark", + theme: HERB_DARK_THEME, automaticLayout: true, minimap: { enabled: true }, lineNumbers: "on", @@ -91,6 +120,7 @@ export function replaceTextareaWithMonaco(textareaId, textareaElement = null, op fontSize: 14, suggestFontSize: 14, lineHeight: 21, + "semanticHighlighting.enabled": true, fixedOverflowWidgets: true, overflowWidgetsDomNode: overflowWidgetsRoot(), }