Language Service: Implement Semantic Token Provider

docs/docs/integrations/editors/zed.md

@@ -37,6 +37,9 @@ Zed keeps language server settings under `lsp.<server>.initialization_options`,
           "minimumLines": 10,
           "maximumClasses": 2
         },
+        "semanticTokens": {
+          "enabled": true
+        },
         "linter": {
           "enabled": true,
           "fixOnSave": true
@@ -49,6 +52,40 @@ Zed keeps language server settings under `lsp.<server>.initialization_options`,
 
 These are the same options VS Code exposes as `languageServerHerb.*` settings, minus the `languageServerHerb` prefix. See the [language server documentation](/projects/language-server) for the full list.
 
+### Semantic highlighting
+
+Herb can colour HTML+ERB from the parsed template rather than from a Tree-sitter grammar, which keeps tags, attributes and ERB delimiters right where they nest inside each other. Zed requests semantic tokens only when you ask it to, so turn them on for HTML+ERB:
+
+```json [settings.json]
+{
+  "languages": {
+    "HTML+ERB": {
+      "semantic_tokens": "combined"
+    }
+  }
+}
+```
+
+`combined` layers the language server's tokens over Tree-sitter, which is what you want here, since Herb deliberately says nothing about the Ruby inside `<% %>` and leaves it to Tree-sitter and the Ruby language server. `full` would drop Tree-sitter entirely and leave that Ruby unstyled.
+
+Colours come from `semantic_token_rules`, matched by token type and modifier. Herb emits `type` for tag names, `property` for attribute names, `string` for values, `macro` for the `<%` and `%>` delimiters with an `output` modifier on `<%=` tags, `parameter` for the names in a `locals:` declaration, and `function` with the `defaultLibrary` modifier for Action View helpers:
+
+```json [settings.json]
+{
+  "global_lsp_settings": {
+    "semantic_token_rules": [
+      { "token_type": "function", "token_modifiers": ["defaultLibrary"], "foreground_color": "#61AFEF" },
+      { "token_type": "macro", "token_modifiers": ["output"], "foreground_color": "#C678DD" },
+      { "token_type": "macro", "foreground_color": "#BE5046" },
+      { "token_type": "property", "foreground_color": "#D19A66" },
+      { "token_type": "type", "foreground_color": "#E06C75" }
+    ]
+  }
+}
+```
+
+Set `semanticTokens.enabled` to `false` in the initialization options above to turn the feature off on Herb's side instead.
+
 ### Inlay hints
 
 Herb annotates the closing tag of longer blocks with what it closes. Zed turns inlay hints off by default, so you need to enable them for HTML+ERB as well as configuring them on the Herb side:

javascript/packages/core/src/index.ts

@@ -28,6 +28,7 @@ export * from "./ruby-keywords.js"
 export * from "./semver.js"
 export * from "./token-list.js"
 export * from "./token.js"
+export * from "./token-classification.js"
 export * from "./util.js"
 export * from "./visitor.js"
 export * from "./warning.js"

javascript/packages/core/src/token-classification.ts

@@ -0,0 +1,231 @@
+import { RUBY_KEYWORDS } from "./ruby-keywords.js"
+
+import type { Token } from "./token.js"
+
+export type TokenCategory =
+  | "html.tagName"
+  | "html.attributeName"
+  | "html.attributeValue"
+  | "html.delimiter"
+  | "html.comment"
+  | "html.doctype"
+  | "html.entity"
+  | "erb.delimiter"
+  | "erb.content"
+  | "erb.commentDelimiter"
+  | "erb.comment"
+  | "other"
+
+export interface ClassifiedToken {
+  token: Token
+  category: TokenCategory
+  quoted?: boolean
+  output?: boolean
+}
+
+interface ClassifierState {
+  inTag: boolean
+  inQuotes: boolean
+  quoteCharacter: string
+  tagName: string
+  isClosingTag: boolean
+  expectingAttributeName: boolean
+  expectingAttributeValue: boolean
+  inComment: boolean
+  inERBComment: boolean
+  inERBOutput: boolean
+}
+
+const HTML_DELIMITERS = new Set([
+  "TOKEN_HTML_TAG_START",
+  "TOKEN_HTML_TAG_START_CLOSE",
+  "TOKEN_HTML_TAG_END",
+  "TOKEN_HTML_TAG_SELF_CLOSE",
+])
+
+const ERB_DELIMITERS = new Set(["TOKEN_ERB_START", "TOKEN_ERB_END"])
+const COMMENT_DELIMITERS = new Set(["TOKEN_HTML_COMMENT_START", "TOKEN_HTML_COMMENT_END"])
+const ERB_TOKENS = new Set([...ERB_DELIMITERS, "TOKEN_ERB_CONTENT"])
+
+function initialState(): ClassifierState {
+  return {
+    inTag: false,
+    inQuotes: false,
+    quoteCharacter: "",
+    tagName: "",
+    isClosingTag: false,
+    expectingAttributeName: false,
+    expectingAttributeValue: false,
+    inComment: false,
+    inERBComment: false,
+    inERBOutput: false,
+  }
+}
+
+function advance(state: ClassifierState, token: Token, text: string): void {
+  switch (token.type) {
+    case "TOKEN_HTML_TAG_START":
+      state.inTag = true
+      state.isClosingTag = false
+      state.expectingAttributeName = false
+      state.expectingAttributeValue = false
+      break
+
+    case "TOKEN_HTML_TAG_START_CLOSE":
+      state.inTag = true
+      state.isClosingTag = true
+      state.expectingAttributeName = false
+      state.expectingAttributeValue = false
+      break
+
+    case "TOKEN_HTML_TAG_END":
+    case "TOKEN_HTML_TAG_SELF_CLOSE":
+      state.inTag = false
+      state.tagName = ""
+      state.isClosingTag = false
+      state.expectingAttributeName = false
+      state.expectingAttributeValue = false
+      break
+
+    case "TOKEN_IDENTIFIER":
+      if (state.inTag && !state.tagName) {
+        state.tagName = text
+        state.expectingAttributeName = !state.isClosingTag
+      } else if (state.inTag && state.expectingAttributeName) {
+        state.expectingAttributeName = false
+        state.expectingAttributeValue = true
+      }
+      break
+
+    case "TOKEN_EQUALS":
+      if (state.inTag) state.expectingAttributeValue = true
+      break
+
+    case "TOKEN_QUOTE":
+      if (state.inTag) {
+        if (!state.inQuotes) {
+          state.inQuotes = true
+          state.quoteCharacter = text
+        } else if (text === state.quoteCharacter) {
+          state.inQuotes = false
+          state.quoteCharacter = ""
+          state.expectingAttributeName = true
+          state.expectingAttributeValue = false
+        }
+      }
+      break
+
+    case "TOKEN_WHITESPACE":
+      if (state.inTag && !state.inQuotes && state.tagName) {
+        state.expectingAttributeName = true
+        state.expectingAttributeValue = false
+      }
+      break
+
+    case "TOKEN_ERB_START":
+      state.inERBComment = text.startsWith("<%#")
+      state.inERBOutput = text.startsWith("<%=") || text.startsWith("<%-=")
+      break
+
+    case "TOKEN_ERB_END":
+      state.inERBComment = false
+      state.inERBOutput = false
+      break
+
+    case "TOKEN_HTML_COMMENT_START":
+      state.inComment = true
+      break
+
+    case "TOKEN_HTML_COMMENT_END":
+      state.inComment = false
+      break
+  }
+}
+
+function categorize(state: ClassifierState, before: ClassifierState, token: Token): TokenCategory {
+  if (state.inComment && !COMMENT_DELIMITERS.has(token.type) && !ERB_TOKENS.has(token.type)) {
+    return "html.comment"
+  }
+
+  if (COMMENT_DELIMITERS.has(token.type)) return "html.comment"
+  if (HTML_DELIMITERS.has(token.type)) return "html.delimiter"
+
+  if (ERB_DELIMITERS.has(token.type)) {
+    return before.inERBComment || state.inERBComment ? "erb.commentDelimiter" : "erb.delimiter"
+  }
+
+  switch (token.type) {
+    case "TOKEN_ERB_CONTENT":
+      return state.inERBComment ? "erb.comment" : "erb.content"
+
+    case "TOKEN_HTML_DOCTYPE":
+      return "html.doctype"
+
+    case "TOKEN_NBSP":
+    case "TOKEN_AMPERSAND":
+      return "html.entity"
+
+    case "TOKEN_IDENTIFIER":
+      if (!before.inTag) break
+      if (!before.tagName) return "html.tagName"
+      if (before.inQuotes) return "html.attributeValue"
+      if (before.expectingAttributeName) return "html.attributeName"
+      if (before.expectingAttributeValue) return "html.attributeValue"
+      break
+
+    case "TOKEN_QUOTE":
+      if (state.inTag) return "html.attributeValue"
+      break
+  }
+
+  return "other"
+}
+
+export function classifyTokens(tokens: Token[], source: string): ClassifiedToken[] {
+  const state = initialState()
+
+  return tokens.map(token => {
+    const text = source.slice(token.range.start, token.range.end)
+    const before = { ...state }
+
+    advance(state, token, text)
+
+    const category = categorize(state, before, token)
+
+    if (category === "html.attributeValue") {
+      return { token, category, quoted: state.inQuotes || token.type === "TOKEN_QUOTE" }
+    }
+
+    if (category === "erb.delimiter") {
+      return { token, category, output: before.inERBOutput || state.inERBOutput }
+    }
+
+    return { token, category }
+  })
+}
+
+const RUBY_HIGHLIGHTED_WORDS = new Set([...RUBY_KEYWORDS, "raise"])
+const WORD_SPLIT = /(\s+|[^\w\s]+)/
+
+export interface RubyFragment {
+  offset: number
+  length: number
+  text: string
+  keyword: boolean
+}
+
+export function splitRubyContent(content: string): RubyFragment[] {
+  const fragments: RubyFragment[] = []
+
+  let offset = 0
+
+  for (const text of content.split(WORD_SPLIT)) {
+    if (text.length > 0) {
+      fragments.push({ offset, length: text.length, text, keyword: RUBY_HIGHLIGHTED_WORDS.has(text) })
+    }
+
+    offset += text.length
+  }
+
+  return fragments
+}

javascript/packages/core/test/token-classification.test.ts

@@ -0,0 +1,80 @@
+import { describe, it, expect, beforeAll } from "vitest"
+import { Herb } from "@herb-tools/node-wasm"
+
+import { classifyTokens } from "../src/token-classification.js"
+
+describe("classifyTokens", () => {
+  beforeAll(async () => {
+    await Herb.load()
+  })
+
+  function classify(source: string) {
+    const tokens = [...Herb.lex(source).value]
+
+    return classifyTokens(tokens, source)
+      .map(({ token, category }) => [source.slice(token.range.start, token.range.end), category])
+      .filter(([, category]) => category !== "other")
+  }
+
+  it("tells a tag name from an attribute name from a value", () => {
+    expect(classify(`<div class="card">`)).toEqual([
+      ["<", "html.delimiter"],
+      ["div", "html.tagName"],
+      ["class", "html.attributeName"],
+      ['"', "html.attributeValue"],
+      ["card", "html.attributeValue"],
+      ['"', "html.attributeValue"],
+      [">", "html.delimiter"],
+    ])
+  })
+
+  it("classifies an unquoted attribute value as a value", () => {
+    expect(classify(`<input type=text>`)).toContainEqual(["text", "html.attributeValue"])
+  })
+
+  it("names a closing tag", () => {
+    expect(classify(`</div>`)).toEqual([
+      ["</", "html.delimiter"],
+      ["div", "html.tagName"],
+      [">", "html.delimiter"],
+    ])
+  })
+
+  it("keeps ERB delimiters separate from their content", () => {
+    expect(classify(`<%= user.name %>`)).toEqual([
+      ["<%=", "erb.delimiter"],
+      [" user.name ", "erb.content"],
+      ["%>", "erb.delimiter"],
+    ])
+  })
+
+  it("treats a comment's insides as comment", () => {
+    expect(classify(`<!-- note -->`).every(([, category]) => category === "html.comment")).toBe(true)
+  })
+
+  it("still sees ERB inside a comment", () => {
+    expect(classify(`<!-- <%= a %> -->`)).toContainEqual(["<%=", "erb.delimiter"])
+  })
+
+  it("handles an ERB tag inside an attribute value", () => {
+    const classified = classify(`<div class="a <%= b %>">`)
+
+    expect(classified).toContainEqual(["class", "html.attributeName"])
+    expect(classified).toContainEqual(["<%=", "erb.delimiter"])
+  })
+
+  it("marks quotes as part of the value even as the state leaves them", () => {
+    const quotes = classifyTokens([...Herb.lex(`<div id="a">`).value], `<div id="a">`)
+      .filter(({ token }) => token.type === "TOKEN_QUOTE")
+
+    expect(quotes).toHaveLength(2)
+    expect(quotes.every(({ category, quoted }) => category === "html.attributeValue" && quoted)).toBe(true)
+  })
+
+  it("returns one entry per token", () => {
+    const source = `<div class="a">text<%= b %></div>`
+    const tokens = [...Herb.lex(source).value]
+
+    expect(classifyTokens(tokens, source)).toHaveLength(tokens.length)
+  })
+})