diff --git a/.changeset/calm-poems-clean.md b/.changeset/calm-poems-clean.md new file mode 100644 index 00000000..2117698f --- /dev/null +++ b/.changeset/calm-poems-clean.md @@ -0,0 +1,5 @@ +--- +'@ryanatkn/fuz_code': minor +--- + +rename `Grammar` to `Syntax_Grammar` diff --git a/.changeset/shy-doors-appear.md b/.changeset/shy-doors-appear.md new file mode 100644 index 00000000..4b4d4fa7 --- /dev/null +++ b/.changeset/shy-doors-appear.md @@ -0,0 +1,5 @@ +--- +'@ryanatkn/fuz_code': minor +--- + +factor out `tokenize_syntax` into its own module diff --git a/CLAUDE.md b/CLAUDE.md index 339ae527..b0e46c0e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,22 +3,9 @@ A performance-focused fork of PrismJS for syntax highlighting, optimized for runtime use with optional CSS Custom Highlight API support. -## Quick Start +## Usage -```typescript -import Code from '$lib/Code.svelte'; -import {syntax_styler_global} from '$lib/syntax_styler_global.js'; - -// Auto-detects and uses CSS Highlight API when available - - -// Force specific rendering mode - // Traditional HTML - // CSS Highlights - -// Direct usage -const html = syntax_styler_global.stylize(code, 'ts'); -``` +See [README.md](./README.md). ## Commands diff --git a/README.md b/README.md index acd80751..64510254 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ The main changes: - has a minimal and explicit API to generate stylized HTML, and knows nothing about the DOM - uses stateless ES modules, instead of globals with side effects and pseudo-module behaviors +- integrates with the + [CSS Custom Highlight API](https://developer.mozilla.org/en-US/docs/Web/API/CSS_Custom_Highlight_API) + when available - has various incompatible changes, so using Prism grammars requires some tweaks - smaller (by 7kB minified and 3kB gzipped, ~1/3 less) - written in TypeScript @@ -47,25 +50,30 @@ and Shiki is designed mainly for buildtime usage. npm i -D @ryanatkn/fuz_code ``` -```ts -import {syntax_styler_global} from '@ryanatkn/fuz_code/syntax_styler_global.js'; - -syntax_styler_global.stylize('

hello world

', 'svelte'); -``` - ```svelte - - - + + + + + + + + +``` + +```ts +import {syntax_styler_global} from '$lib/syntax_styler_global.js'; - - +// Direct usage +const html = syntax_styler_global.stylize(code, 'ts'); - - +// Get raw tokens +import {tokenize_syntax} from '$lib/tokenize_syntax.js'; +const tokens = tokenize_syntax(code, syntax_styler_global.get_lang('ts')); ``` By default the `Code` component automatically uses the diff --git a/package-lock.json b/package-lock.json index 72302520..0ff14cc8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,9 +12,9 @@ "@changesets/changelog-git": "^0.2.1", "@ryanatkn/belt": "^0.34.1", "@ryanatkn/eslint-config": "^0.8.0", - "@ryanatkn/fuz": "^0.145.0", - "@ryanatkn/gro": "^0.165.0", - "@ryanatkn/moss": "^0.33.0", + "@ryanatkn/fuz": "^0.146.0", + "@ryanatkn/gro": "^0.165.1", + "@ryanatkn/moss": "^0.34.0", "@sveltejs/adapter-static": "^3.0.9", "@sveltejs/kit": "^2.37.1", "@sveltejs/package": "^2.5.0", @@ -1452,9 +1452,9 @@ } }, "node_modules/@ryanatkn/fuz": { - "version": "0.145.0", - "resolved": "https://registry.npmjs.org/@ryanatkn/fuz/-/fuz-0.145.0.tgz", - "integrity": "sha512-oy0+hszTKIUrfy8k8Emxp2FDX8zX8Tp5y/YEHUsY2+tuW/560y6D9tm1A+lVf6LhF9gD4gL96cybyQoL6IBFgg==", + "version": "0.146.0", + "resolved": "https://registry.npmjs.org/@ryanatkn/fuz/-/fuz-0.146.0.tgz", + "integrity": "sha512-cj3/askpyl1ENZvh11FCp9UISYkJwT+gM0ozRGN9QqPHTyHQrSGVodKVty5dK4NMWyB9GuxZZC39vf0s6RvHgg==", "dev": true, "license": "MIT", "engines": { @@ -1465,21 +1465,25 @@ }, "peerDependencies": { "@ryanatkn/belt": "*", - "@ryanatkn/gro": "*", + "@ryanatkn/gro": ">=0.165.0", "@ryanatkn/moss": "*", "@sveltejs/kit": "^2.26", + "esm-env": "^1", "svelte": "^5" }, "peerDependenciesMeta": { "@ryanatkn/gro": { "optional": true + }, + "esm-env": { + "optional": true } } }, "node_modules/@ryanatkn/gro": { - "version": "0.165.0", - "resolved": "https://registry.npmjs.org/@ryanatkn/gro/-/gro-0.165.0.tgz", - "integrity": "sha512-FrDuq5+mzRMr3+h5Ooj3QFJmuLsKFkJlkt6Ukbn4f6c2AYwAL9SDHT4bIFwelAhzK63HkJIfYf70Zo2KyipGAA==", + "version": "0.165.1", + "resolved": "https://registry.npmjs.org/@ryanatkn/gro/-/gro-0.165.1.tgz", + "integrity": "sha512-eMFeDOg7um5788mahIHK2EppmuE01lZeHmla/5oiS7RJa9Y29Mg2xfwu0LT/b9kUd4AwRSt0mX25pIeUAw6K4A==", "dev": true, "license": "MIT", "dependencies": { @@ -1525,9 +1529,9 @@ } }, "node_modules/@ryanatkn/moss": { - "version": "0.33.0", - "resolved": "https://registry.npmjs.org/@ryanatkn/moss/-/moss-0.33.0.tgz", - "integrity": "sha512-PCdDyV5N0eQQ3C7Ee0bZHSwMgQ7O99zUzdJmuegA3sdRhT9A66QOBr0gdqUK70o20jUf00ByMRyA4PVAjBmyWA==", + "version": "0.34.0", + "resolved": "https://registry.npmjs.org/@ryanatkn/moss/-/moss-0.34.0.tgz", + "integrity": "sha512-PtC+D8Hy8Oxwuovwfb33l1jGFfgj/MYP9pvRSr1hC4aPhdlFvB6GS97O0o8dEZx0EkQG5zk7oxshmpKn6l/w7Q==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index 14fa05f2..3210da02 100644 --- a/package.json +++ b/package.json @@ -48,9 +48,9 @@ "@changesets/changelog-git": "^0.2.1", "@ryanatkn/belt": "^0.34.1", "@ryanatkn/eslint-config": "^0.8.0", - "@ryanatkn/fuz": "^0.145.0", - "@ryanatkn/gro": "^0.165.0", - "@ryanatkn/moss": "^0.33.0", + "@ryanatkn/fuz": "^0.146.0", + "@ryanatkn/gro": "^0.165.1", + "@ryanatkn/moss": "^0.34.0", "@sveltejs/adapter-static": "^3.0.9", "@sveltejs/kit": "^2.37.1", "@sveltejs/package": "^2.5.0", diff --git a/src/fixtures/helpers.ts b/src/fixtures/helpers.ts index 8989af90..3cccd320 100644 --- a/src/fixtures/helpers.ts +++ b/src/fixtures/helpers.ts @@ -2,7 +2,8 @@ import {readFileSync} from 'node:fs'; import {search_fs} from '@ryanatkn/gro/search_fs.js'; import {basename, join, relative} from 'node:path'; import {syntax_styler_global} from '$lib/syntax_styler_global.js'; -import {tokenize_syntax, type Syntax_Token_Stream, Syntax_Token} from '$lib/syntax_styler.js'; +import {tokenize_syntax} from '$lib/tokenize_syntax.js'; +import {type Syntax_Token_Stream, Syntax_Token} from '$lib/syntax_token.js'; export interface Sample_Spec { lang: string; diff --git a/src/lib/Code.svelte b/src/lib/Code.svelte index ca85fc65..8d86d4ff 100644 --- a/src/lib/Code.svelte +++ b/src/lib/Code.svelte @@ -6,7 +6,8 @@ import {onDestroy, type Snippet} from 'svelte'; import {syntax_styler_global} from '$lib/syntax_styler_global.js'; - import {tokenize_syntax, type Syntax_Styler, type Grammar} from '$lib/syntax_styler.js'; + import type {Syntax_Styler, Syntax_Grammar} from '$lib/syntax_styler.js'; + import {tokenize_syntax} from '$lib/tokenize_syntax.js'; import { Highlight_Manager, supports_css_highlight_api, @@ -29,7 +30,7 @@ mode?: Highlight_Mode; pre_attrs?: any; code_attrs?: any; - grammar?: Grammar | undefined; + grammar?: Syntax_Grammar | undefined; inline?: boolean; syntax_styler?: Syntax_Styler; children?: Snippet<[markup: string]>; diff --git a/src/lib/grammar_clike.ts b/src/lib/grammar_clike.ts index 1364c5af..67f13066 100644 --- a/src/lib/grammar_clike.ts +++ b/src/lib/grammar_clike.ts @@ -1,4 +1,4 @@ -import type {Add_Grammar, Grammar} from '$lib/syntax_styler.js'; +import type {Add_Syntax_Grammar, Syntax_Grammar} from '$lib/syntax_styler.js'; /** * Based on Prism (https://github.com/PrismJS/prism) @@ -8,7 +8,7 @@ import type {Add_Grammar, Grammar} from '$lib/syntax_styler.js'; * * @see LICENSE */ -export const add_grammar_clike: Add_Grammar = (syntax_styler) => { +export const add_grammar_clike: Add_Syntax_Grammar = (syntax_styler) => { const grammar_clike = { comment: [ { @@ -41,7 +41,7 @@ export const add_grammar_clike: Add_Grammar = (syntax_styler) => { number: /\b0x[\da-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i, operator: /[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/, punctuation: /[{}[\];(),.:]/, - } satisfies Grammar; + } satisfies Syntax_Grammar; syntax_styler.add_lang('clike', grammar_clike); }; diff --git a/src/lib/grammar_css.ts b/src/lib/grammar_css.ts index f3bd08f9..c6171d75 100644 --- a/src/lib/grammar_css.ts +++ b/src/lib/grammar_css.ts @@ -1,4 +1,4 @@ -import type {Add_Grammar, Grammar} from '$lib/syntax_styler.js'; +import type {Add_Syntax_Grammar, Syntax_Grammar} from '$lib/syntax_styler.js'; import {grammar_markup_add_attribute, grammar_markup_add_inlined} from '$lib/grammar_markup.js'; var string = /(?:"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n])*')/; @@ -11,7 +11,7 @@ var string = /(?:"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|'(?:\\(?:\r\n|[\s\S])|[^'\\ * * @see LICENSE */ -export const add_grammar_css: Add_Grammar = (syntax_styler) => { +export const add_grammar_css: Add_Syntax_Grammar = (syntax_styler) => { const grammar_css = { comment: /\/\*[\s\S]*?\*\//, atrule: { @@ -35,7 +35,7 @@ export const add_grammar_css: Add_Grammar = (syntax_styler) => { pattern: /(^|[^\w-])(?:and|not|only|or)(?![\w-])/, lookbehind: true, }, - } as Grammar, // see `rest` below + } as Syntax_Grammar, // see `rest` below }, url: { // https://drafts.csswg.org/css-values-3/#urls @@ -73,7 +73,7 @@ export const add_grammar_css: Add_Grammar = (syntax_styler) => { lookbehind: true, }, punctuation: /[(){};:,]/, - } satisfies Grammar; + } satisfies Syntax_Grammar; grammar_css.atrule.inside.rest = grammar_css; diff --git a/src/lib/grammar_js.ts b/src/lib/grammar_js.ts index 68c15c33..18ee65ff 100644 --- a/src/lib/grammar_js.ts +++ b/src/lib/grammar_js.ts @@ -1,4 +1,4 @@ -import type {Add_Grammar} from '$lib/syntax_styler.js'; +import type {Add_Syntax_Grammar} from '$lib/syntax_styler.js'; import {grammar_markup_add_attribute, grammar_markup_add_inlined} from '$lib/grammar_markup.js'; /** @@ -9,7 +9,7 @@ import {grammar_markup_add_attribute, grammar_markup_add_inlined} from '$lib/gra * * @see LICENSE */ -export const add_grammar_js: Add_Grammar = (syntax_styler) => { +export const add_grammar_js: Add_Syntax_Grammar = (syntax_styler) => { const grammar_clike = syntax_styler.get_lang('clike'); const grammar_js = syntax_styler.add_extended_lang('clike', 'js', { diff --git a/src/lib/grammar_json.ts b/src/lib/grammar_json.ts index 647af6e2..f338a3d7 100644 --- a/src/lib/grammar_json.ts +++ b/src/lib/grammar_json.ts @@ -1,4 +1,4 @@ -import type {Add_Grammar, Grammar} from '$lib/syntax_styler.js'; +import type {Add_Syntax_Grammar, Syntax_Grammar} from '$lib/syntax_styler.js'; /** * Based on Prism (https://github.com/PrismJS/prism) @@ -8,7 +8,7 @@ import type {Add_Grammar, Grammar} from '$lib/syntax_styler.js'; * * @see LICENSE */ -export const add_grammar_json: Add_Grammar = (syntax_styler) => { +export const add_grammar_json: Add_Syntax_Grammar = (syntax_styler) => { const grammar_json = { property: { pattern: /(^|[^\\])"(?:\\.|[^\\"\r\n])*"(?=\s*:)/, @@ -32,7 +32,7 @@ export const add_grammar_json: Add_Grammar = (syntax_styler) => { pattern: /\bnull\b/, alias: 'keyword', }, - } satisfies Grammar; + } satisfies Syntax_Grammar; syntax_styler.add_lang('json', grammar_json); }; diff --git a/src/lib/grammar_markup.ts b/src/lib/grammar_markup.ts index 8458dc53..be76592a 100644 --- a/src/lib/grammar_markup.ts +++ b/src/lib/grammar_markup.ts @@ -1,4 +1,9 @@ -import type {Syntax_Styler, Add_Grammar, Grammar, Grammar_Token} from '$lib/syntax_styler.js'; +import type { + Syntax_Styler, + Add_Syntax_Grammar, + Syntax_Grammar, + Syntax_Grammar_Token, +} from '$lib/syntax_styler.js'; /** * Based on Prism (https://github.com/PrismJS/prism) @@ -8,7 +13,7 @@ import type {Syntax_Styler, Add_Grammar, Grammar, Grammar_Token} from '$lib/synt * * @see LICENSE */ -export const add_grammar_markup: Add_Grammar = (syntax_styler) => { +export const add_grammar_markup: Add_Syntax_Grammar = (syntax_styler) => { const grammar_markup = { comment: { pattern: //, @@ -72,7 +77,7 @@ export const add_grammar_markup: Add_Grammar = (syntax_styler) => { }, /&#x?[\da-f]{1,8};/i, ], - } satisfies Grammar; + } satisfies Syntax_Grammar; grammar_markup.tag.inside.attr_value.inside.entity = grammar_markup.entity; @@ -146,8 +151,8 @@ export const grammar_markup_add_attribute = ( lang: string, ): void => { ( - (syntax_styler.get_lang('markup').tag as Grammar_Token).inside! - .special_attr as Array + (syntax_styler.get_lang('markup').tag as Syntax_Grammar_Token).inside! + .special_attr as Array ).push({ pattern: RegExp( /(^|["'\s])/.source + diff --git a/src/lib/grammar_svelte.ts b/src/lib/grammar_svelte.ts index f6995873..4cecce67 100644 --- a/src/lib/grammar_svelte.ts +++ b/src/lib/grammar_svelte.ts @@ -1,4 +1,4 @@ -import type {Add_Grammar, Grammar_Token, Syntax_Styler} from '$lib/syntax_styler.js'; +import type {Add_Syntax_Grammar, Syntax_Grammar_Token, Syntax_Styler} from '$lib/syntax_styler.js'; import {grammar_markup_add_inlined} from '$lib/grammar_markup.js'; const blocks = '(if|else if|await|then|catch|each|html|debug)'; @@ -11,7 +11,7 @@ const blocks = '(if|else if|await|then|catch|each|html|debug)'; * * @see LICENSE */ -export const add_grammar_svelte: Add_Grammar = (syntax_styler) => { +export const add_grammar_svelte: Add_Syntax_Grammar = (syntax_styler) => { const grammar_ts = syntax_styler.get_lang('ts'); const grammar_svelte = syntax_styler.add_extended_lang('markup', 'svelte', { @@ -101,8 +101,9 @@ export const add_grammar_svelte: Add_Grammar = (syntax_styler) => { }); // oof lol - ((grammar_svelte.tag as Grammar_Token).inside!.attr_value as Grammar_Token).inside!.entity = - grammar_svelte.entity; + ( + (grammar_svelte.tag as Syntax_Grammar_Token).inside!.attr_value as Syntax_Grammar_Token + ).inside!.entity = grammar_svelte.entity; grammar_svelte_add_inlined(syntax_styler, 'style', 'css'); grammar_svelte_add_inlined(syntax_styler, 'script', 'ts'); diff --git a/src/lib/grammar_ts.ts b/src/lib/grammar_ts.ts index fe96a9ac..ffe413db 100644 --- a/src/lib/grammar_ts.ts +++ b/src/lib/grammar_ts.ts @@ -1,4 +1,4 @@ -import type {Add_Grammar, Grammar_Token} from '$lib/syntax_styler.js'; +import type {Add_Syntax_Grammar, Syntax_Grammar_Token} from '$lib/syntax_styler.js'; /** * Based on Prism (https://github.com/PrismJS/prism) @@ -8,7 +8,7 @@ import type {Add_Grammar, Grammar_Token} from '$lib/syntax_styler.js'; * * @see LICENSE */ -export const add_grammar_ts: Add_Grammar = (syntax_styler) => { +export const add_grammar_ts: Add_Syntax_Grammar = (syntax_styler) => { const grammar_ts = syntax_styler.add_extended_lang('js', 'ts', { class_name: { pattern: @@ -38,7 +38,7 @@ export const add_grammar_ts: Add_Grammar = (syntax_styler) => { var type_inside = syntax_styler.extend_grammar('ts', {}); (type_inside as any).class_name = undefined; - (grammar_ts.class_name as Grammar_Token).inside = type_inside; + (grammar_ts.class_name as Syntax_Grammar_Token).inside = type_inside; syntax_styler.grammar_insert_before('ts', 'function', { decorator: { diff --git a/src/lib/highlight_manager.ts b/src/lib/highlight_manager.ts index 5421dc24..17fb1bb8 100644 --- a/src/lib/highlight_manager.ts +++ b/src/lib/highlight_manager.ts @@ -1,4 +1,4 @@ -import {Syntax_Token, type Syntax_Token_Stream} from './syntax_styler.js'; +import {Syntax_Token, type Syntax_Token_Stream} from './syntax_token.js'; export type Highlight_Mode = 'auto' | 'ranges' | 'html'; diff --git a/src/lib/syntax_styler.test.ts b/src/lib/syntax_styler.test.ts index 739cf100..bb55117b 100644 --- a/src/lib/syntax_styler.test.ts +++ b/src/lib/syntax_styler.test.ts @@ -1,5 +1,6 @@ import {test, assert, describe} from 'vitest'; -import {Syntax_Styler, tokenize_syntax} from './syntax_styler.js'; +import {Syntax_Styler} from './syntax_styler.js'; +import {tokenize_syntax} from './tokenize_syntax.js'; import {add_grammar_js} from './grammar_js.js'; import {add_grammar_ts} from './grammar_ts.js'; import {add_grammar_css} from './grammar_css.js'; diff --git a/src/lib/syntax_styler.ts b/src/lib/syntax_styler.ts index 24f89ae5..cd694af7 100644 --- a/src/lib/syntax_styler.ts +++ b/src/lib/syntax_styler.ts @@ -1,4 +1,7 @@ -export type Add_Grammar = (syntax_styler: Syntax_Styler) => void; +import {Syntax_Token, type Syntax_Token_Stream} from './syntax_token.js'; +import {tokenize_syntax} from './tokenize_syntax.js'; + +export type Add_Syntax_Grammar = (syntax_styler: Syntax_Styler) => void; /** * Based on Prism (https://github.com/PrismJS/prism) @@ -9,7 +12,7 @@ export type Add_Grammar = (syntax_styler: Syntax_Styler) => void; * @see LICENSE */ export class Syntax_Styler { - langs: Record = { + langs: Record = { plaintext: {}, }; @@ -29,7 +32,7 @@ export class Syntax_Styler { // } // } - add_lang(id: string, grammar: Grammar, aliases?: Array): void { + add_lang(id: string, grammar: Syntax_Grammar, aliases?: Array): void { this.langs[id] = grammar; if (aliases !== undefined) { for (var alias of aliases) { @@ -41,15 +44,15 @@ export class Syntax_Styler { add_extended_lang( base_id: string, extension_id: string, - extension: Grammar, + extension: Syntax_Grammar, aliases?: Array, - ): Grammar { + ): Syntax_Grammar { var grammar = this.extend_grammar(base_id, extension); this.add_lang(extension_id, grammar, aliases); return grammar; } - get_lang(id: string): Grammar { + get_lang(id: string): Syntax_Grammar { var lang = this.langs[id]; if (lang === undefined) { throw Error(`The language "${id}" has no grammar.`); @@ -81,7 +84,11 @@ export class Syntax_Styler { * @example * stylize('var foo = false;', 'ts', custom_grammar); */ - stylize(text: string, lang: string, grammar: Grammar | undefined = this.get_lang(lang)): string { + stylize( + text: string, + lang: string, + grammar: Syntax_Grammar | undefined = this.get_lang(lang), + ): string { var ctx: Hook_Before_Tokenize_Callback_Context = { code: text, grammar, @@ -175,11 +182,11 @@ export class Syntax_Styler { grammar_insert_before( inside: string, before: string, - insert: Grammar, + insert: Syntax_Grammar, root: Record = this.langs, - ): Grammar { + ): Syntax_Grammar { var grammar = root[inside]; - var updated: Grammar = {}; + var updated: Syntax_Grammar = {}; for (var token in grammar) { if (token === before) { @@ -287,7 +294,7 @@ export class Syntax_Styler { * @param extension - The new tokens to append. * @returns the new grammar */ - extend_grammar(base_id: string, extension: Grammar): Grammar { + extend_grammar(base_id: string, extension: Syntax_Grammar): Syntax_Grammar { return {...deep_clone(this.get_lang(base_id)), ...extension}; } @@ -326,9 +333,11 @@ export class Syntax_Styler { } } -export type Grammar_Value = RegExp | Grammar_Token | Array; +export type Syntax_Grammar_Value = RegExp | Syntax_Grammar_Token | Array; -export type Grammar = Record & {rest?: Grammar | undefined}; +export type Syntax_Grammar = Record & { + rest?: Syntax_Grammar | undefined; +}; /** * The expansion of a simple `RegExp` literal to support additional properties. @@ -340,7 +349,7 @@ export type Grammar = Record & {rest?: Grammar | undefine * Note: This can cause infinite recursion. Be careful when you embed different languages or even the same language into * each another. */ -export interface Grammar_Token { +export interface Syntax_Grammar_Token { /** * The regular expression of the token. */ @@ -363,50 +372,9 @@ export interface Grammar_Token { /** * The nested grammar of this token. */ - inside?: Grammar | null; + inside?: Syntax_Grammar | null; } -/** - * Accepts a string of text as input and the language definitions to use, - * and returns an array with the tokenized code. - * - * When the language definition includes nested tokens, the function is called recursively on each of these tokens. - * - * This method could be useful in other contexts as well, as a very crude parser. - * - * @param text - a string with the code to be styled - * @param grammar - an object containing the tokens to use - * - * Usually a language definition like `syntax_styler.get_lang('markup')`. - * - * @returns an array of strings and tokens, a token stream - * - * @example - * var code = `var foo = 0;`; - * var tokens = tokenize_syntax(code, Syntax_Styler.langs.js); - * for (var token of tokens) { - * if (token instanceof Syntax_Token && token.type === 'number') { - * console.log(`Found numeric literal: ${token.content}`); - * } - * } - */ -export const tokenize_syntax = (text: string, grammar: Grammar): Syntax_Token_Stream => { - var {rest} = grammar; - if (rest) { - for (var token in rest) { - grammar[token] = rest[token]; - } - grammar.rest = undefined; // preserve shape - } - - var token_list = new Linked_List(); - add_after(token_list, token_list.head, text); - - match_grammar(text, token_list, grammar, token_list.head, 0); - - return to_array(token_list); -}; - const depth_first_search = ( o: any, cb: (obj: any, key: string, value: any) => void, @@ -429,306 +397,19 @@ const depth_first_search = ( } }; -export class Syntax_Token { - /** - * The type of the token. - * - * This is usually the key of a pattern in a `Grammar`. - */ - type: string; - - /** - * The strings or tokens contained by this token. - * - * This will be a token stream if the pattern matched also defined an `inside` grammar. - */ - content: string | Syntax_Token_Stream; - - /** - * The alias(es) of the token. - */ - alias: string | Array; - - length: number; - - constructor( - type: string, - content: string | Syntax_Token_Stream, - alias: string | Array, - matched_str: string = '', - ) { - this.type = type; - this.content = content; - this.alias = alias; - this.length = matched_str.length; - } -} - -/** - * A token stream is an array of strings and `Syntax_Token` objects. - * - * Syntax token streams have to fulfill a few properties that are assumed by most functions (mostly internal ones) that process - * them. - * - * 1. No adjacent strings. - * 2. No empty strings. - * - * The only exception here is the token stream that only contains the empty string and nothing else. - */ -export type Syntax_Token_Stream = Array; - -const match_pattern = ( - pattern: RegExp, - pos: number, - text: string, - lookbehind: boolean, -): RegExpExecArray | null => { - pattern.lastIndex = pos; - var match = pattern.exec(text); - if (match && lookbehind && match[1]) { - // change the match to remove the text matched by the lookbehind group - var lookbehind_length = match[1].length; - match.index += lookbehind_length; - match[0] = match[0].substring(lookbehind_length); - } - return match; -}; - -interface Rematch_Options { - cause: string; - reach: number; -} - -const match_grammar = ( - text: string, - token_list: Linked_List, - grammar: any, - start_node: Linked_List_Node, - start_pos: number, - rematch?: Rematch_Options, -): void => { - for (var token in grammar) { - var patterns = grammar[token]; - - if (!patterns) { - continue; - } - - patterns = Array.isArray(patterns) ? patterns : [patterns]; - - for (var j = 0; j < patterns.length; ++j) { - if (rematch && rematch.cause === token + ',' + j) { - return; - } - - var pattern_obj = patterns[j]; - var inside = pattern_obj.inside; - var lookbehind = !!pattern_obj.lookbehind; - var greedy = !!pattern_obj.greedy; - var alias = pattern_obj.alias; - - if (greedy && !pattern_obj.pattern.global) { - // Without the global flag, lastIndex won't work - var flags = pattern_obj.pattern.toString().match(/[imsuy]*$/)[0]; - pattern_obj.pattern = RegExp(pattern_obj.pattern.source, flags + 'g'); - } - - var pattern: RegExp = pattern_obj.pattern || pattern_obj; - - for ( - // iterate the token list and keep track of the current token/string position - var current_node = start_node.next, pos = start_pos; - current_node !== token_list.tail; - pos += current_node!.value!.length, current_node = current_node!.next - ) { - if (rematch && pos >= rematch.reach) { - break; - } - - var str = current_node!.value; - - if (token_list.length > text.length) { - // Something went terribly wrong, ABORT, ABORT! - return; - } - - if (str instanceof Syntax_Token) { - continue; - } - - var remove_count = 1; - var match; - - if (greedy) { - match = match_pattern(pattern, pos, text, lookbehind); - if (!match || match.index >= text.length) { - break; - } - - var from = match.index; - var to = match.index + match[0].length; - var p = pos; - - // find the node that contains the match - p += current_node!.value!.length; - while (from >= p) { - current_node = current_node!.next; - p += current_node!.value!.length; - } - // adjust pos (and p) - p -= current_node!.value!.length; - pos = p; - - // the current node is a Token, then the match starts inside another Token, which is invalid - if (current_node!.value instanceof Syntax_Token) { - continue; - } - - // find the last node which is affected by this match - for ( - var k = current_node; - k !== token_list.tail && (p < to || typeof k!.value === 'string'); - k = k!.next - ) { - remove_count++; - p += k!.value!.length; - } - remove_count--; - - // replace with the new match - str = text.substring(pos, p); - match.index -= pos; - } else { - match = match_pattern(pattern, 0, str!, lookbehind); - if (!match) { - continue; - } - } - - var from = match.index; - var match_str = match[0]; - var before = str!.substring(0, from); - var after = str!.substring(from + match_str.length); - - var reach = pos + str!.length; - if (rematch && reach > rematch.reach) { - rematch.reach = reach; - } - - var remove_from = current_node!.prev; - - if (before) { - remove_from = add_after(token_list, remove_from!, before); - pos += before.length; - } - - remove_range(token_list, remove_from!, remove_count); - - var wrapped = new Syntax_Token( - token, - inside ? tokenize_syntax(match_str, inside) : match_str, - alias, - match_str, - ); - current_node = add_after(token_list, remove_from!, wrapped); - - if (after) { - add_after(token_list, current_node, after); - } - - if (remove_count > 1) { - // at least one Token object was removed, so we have to do some rematching - // this can only happen if the current pattern is greedy - - var nested_rematch: Rematch_Options = { - cause: token + ',' + j, - reach, - }; - match_grammar(text, token_list, grammar, current_node.prev!, pos, nested_rematch); - - // the reach might have been extended because of the rematching - if (rematch && nested_rematch.reach > rematch.reach) { - rematch.reach = nested_rematch.reach; - } - } - } - } - } -}; - -class Linked_List { - head: Linked_List_Node; - tail: Linked_List_Node; - length: number = 0; - - constructor() { - this.head = {value: null, prev: null, next: null}; - this.tail = {value: null, prev: this.head, next: null}; - this.head.next = this.tail; - } -} - -interface Linked_List_Node { - value: T | null; - prev: Linked_List_Node | null; - next: Linked_List_Node | null; -} - -/** - * Adds a new node with the given value to the list. - */ -const add_after = ( - list: Linked_List, - node: Linked_List_Node, - value: T, -): Linked_List_Node => { - // assumes that node != list.tail && values.length >= 0 - var next = node.next!; - - var new_node = {value, prev: node, next}; - node.next = new_node; - next.prev = new_node; - list.length++; - - return new_node; -}; - -/** - * Removes `count` nodes after the given node. The given node will not be removed. - */ -const remove_range = (list: Linked_List, node: Linked_List_Node, count: number) => { - var next = node.next; - for (var i = 0; i < count && next !== list.tail; i++) { - next = next!.next; - } - node.next = next; - next!.prev = node; - list.length -= i; -}; - -const to_array = (list: Linked_List): Array => { - var array = []; - var node = list.head.next; - while (node !== list.tail) { - array.push(node!.value!); - node = node!.next; - } - return array; -}; - export type Hook_Before_Tokenize_Callback = (ctx: Hook_Before_Tokenize_Callback_Context) => void; export type Hook_After_Tokenize_Callback = (ctx: Hook_After_Tokenize_Callback_Context) => void; export type Hook_Wrap_Callback = (ctx: Hook_Wrap_Callback_Context) => void; export interface Hook_Before_Tokenize_Callback_Context { code: string; - grammar: Grammar; + grammar: Syntax_Grammar; lang: string; tokens: undefined; } export interface Hook_After_Tokenize_Callback_Context { code: string; - grammar: Grammar; + grammar: Syntax_Grammar; lang: string; tokens: Syntax_Token_Stream; } diff --git a/src/lib/syntax_token.ts b/src/lib/syntax_token.ts new file mode 100644 index 00000000..59e3d2bd --- /dev/null +++ b/src/lib/syntax_token.ts @@ -0,0 +1,47 @@ +export class Syntax_Token { + /** + * The type of the token. + * + * This is usually the key of a pattern in a `Grammar`. + */ + type: string; + + /** + * The strings or tokens contained by this token. + * + * This will be a token stream if the pattern matched also defined an `inside` grammar. + */ + content: string | Syntax_Token_Stream; + + /** + * The alias(es) of the token. + */ + alias: string | Array; + + length: number; + + constructor( + type: string, + content: string | Syntax_Token_Stream, + alias: string | Array, + matched_str: string = '', + ) { + this.type = type; + this.content = content; + this.alias = alias; + this.length = matched_str.length; + } +} + +/** + * A token stream is an array of strings and `Syntax_Token` objects. + * + * Syntax token streams have to fulfill a few properties that are assumed by most functions (mostly internal ones) that process + * them. + * + * 1. No adjacent strings. + * 2. No empty strings. + * + * The only exception here is the token stream that only contains the empty string and nothing else. + */ +export type Syntax_Token_Stream = Array; diff --git a/src/lib/tokenize_syntax.ts b/src/lib/tokenize_syntax.ts new file mode 100644 index 00000000..0781f33c --- /dev/null +++ b/src/lib/tokenize_syntax.ts @@ -0,0 +1,282 @@ +import type {Syntax_Grammar} from './syntax_styler.js'; +import {Syntax_Token, type Syntax_Token_Stream} from './syntax_token.js'; + +/** + * Accepts a string of text as input and the language definitions to use, + * and returns an array with the tokenized code. + * + * When the language definition includes nested tokens, the function is called recursively on each of these tokens. + * + * This method could be useful in other contexts as well, as a very crude parser. + * + * @param text - a string with the code to be styled + * @param grammar - an object containing the tokens to use + * + * Usually a language definition like `syntax_styler.get_lang('markup')`. + * + * @returns an array of strings and tokens, a token stream + * + * @example + * var code = `var foo = 0;`; + * var tokens = tokenize_syntax(code, Syntax_Styler.langs.js); + * for (var token of tokens) { + * if (token instanceof Syntax_Token && token.type === 'number') { + * console.log(`Found numeric literal: ${token.content}`); + * } + * } + */ +export const tokenize_syntax = (text: string, grammar: Syntax_Grammar): Syntax_Token_Stream => { + var {rest} = grammar; + if (rest) { + for (var token in rest) { + grammar[token] = rest[token]; + } + grammar.rest = undefined; // preserve shape + } + + var token_list = new Linked_List(); + add_after(token_list, token_list.head, text); + + match_grammar(text, token_list, grammar, token_list.head, 0); + + return to_array(token_list); +}; + +interface Rematch_Options { + cause: string; + reach: number; +} + +const match_grammar = ( + text: string, + token_list: Linked_List, + grammar: any, + start_node: Linked_List_Node, + start_pos: number, + rematch?: Rematch_Options, +): void => { + for (var token in grammar) { + var patterns = grammar[token]; + + if (!patterns) { + continue; + } + + patterns = Array.isArray(patterns) ? patterns : [patterns]; + + for (var j = 0; j < patterns.length; ++j) { + if (rematch && rematch.cause === token + ',' + j) { + return; + } + + var pattern_obj = patterns[j]; + var inside = pattern_obj.inside; + var lookbehind = !!pattern_obj.lookbehind; + var greedy = !!pattern_obj.greedy; + var alias = pattern_obj.alias; + + if (greedy && !pattern_obj.pattern.global) { + // Without the global flag, lastIndex won't work + var flags = pattern_obj.pattern.toString().match(/[imsuy]*$/)[0]; + pattern_obj.pattern = RegExp(pattern_obj.pattern.source, flags + 'g'); + } + + var pattern: RegExp = pattern_obj.pattern || pattern_obj; + + for ( + // iterate the token list and keep track of the current token/string position + var current_node = start_node.next, pos = start_pos; + current_node !== token_list.tail; + pos += current_node!.value!.length, current_node = current_node!.next + ) { + if (rematch && pos >= rematch.reach) { + break; + } + + var str = current_node!.value; + + if (token_list.length > text.length) { + // Something went terribly wrong, ABORT, ABORT! + return; + } + + if (str instanceof Syntax_Token) { + continue; + } + + var remove_count = 1; + var match; + + if (greedy) { + match = match_pattern(pattern, pos, text, lookbehind); + if (!match || match.index >= text.length) { + break; + } + + var from = match.index; + var to = match.index + match[0].length; + var p = pos; + + // find the node that contains the match + p += current_node!.value!.length; + while (from >= p) { + current_node = current_node!.next; + p += current_node!.value!.length; + } + // adjust pos (and p) + p -= current_node!.value!.length; + pos = p; + + // the current node is a Token, then the match starts inside another Token, which is invalid + if (current_node!.value instanceof Syntax_Token) { + continue; + } + + // find the last node which is affected by this match + for ( + var k = current_node; + k !== token_list.tail && (p < to || typeof k!.value === 'string'); + k = k!.next + ) { + remove_count++; + p += k!.value!.length; + } + remove_count--; + + // replace with the new match + str = text.substring(pos, p); + match.index -= pos; + } else { + match = match_pattern(pattern, 0, str!, lookbehind); + if (!match) { + continue; + } + } + + var from = match.index; + var match_str = match[0]; + var before = str!.substring(0, from); + var after = str!.substring(from + match_str.length); + + var reach = pos + str!.length; + if (rematch && reach > rematch.reach) { + rematch.reach = reach; + } + + var remove_from = current_node!.prev; + + if (before) { + remove_from = add_after(token_list, remove_from!, before); + pos += before.length; + } + + remove_range(token_list, remove_from!, remove_count); + + var wrapped = new Syntax_Token( + token, + inside ? tokenize_syntax(match_str, inside) : match_str, + alias, + match_str, + ); + current_node = add_after(token_list, remove_from!, wrapped); + + if (after) { + add_after(token_list, current_node, after); + } + + if (remove_count > 1) { + // at least one Token object was removed, so we have to do some rematching + // this can only happen if the current pattern is greedy + + var nested_rematch: Rematch_Options = { + cause: token + ',' + j, + reach, + }; + match_grammar(text, token_list, grammar, current_node.prev!, pos, nested_rematch); + + // the reach might have been extended because of the rematching + if (rematch && nested_rematch.reach > rematch.reach) { + rematch.reach = nested_rematch.reach; + } + } + } + } + } +}; + +class Linked_List { + head: Linked_List_Node; + tail: Linked_List_Node; + length: number = 0; + + constructor() { + this.head = {value: null, prev: null, next: null}; + this.tail = {value: null, prev: this.head, next: null}; + this.head.next = this.tail; + } +} + +interface Linked_List_Node { + value: T | null; + prev: Linked_List_Node | null; + next: Linked_List_Node | null; +} + +/** + * Adds a new node with the given value to the list. + */ +const add_after = ( + list: Linked_List, + node: Linked_List_Node, + value: T, +): Linked_List_Node => { + // assumes that node != list.tail && values.length >= 0 + var next = node.next!; + + var new_node = {value, prev: node, next}; + node.next = new_node; + next.prev = new_node; + list.length++; + + return new_node; +}; + +/** + * Removes `count` nodes after the given node. The given node will not be removed. + */ +const remove_range = (list: Linked_List, node: Linked_List_Node, count: number) => { + var next = node.next; + for (var i = 0; i < count && next !== list.tail; i++) { + next = next!.next; + } + node.next = next; + next!.prev = node; + list.length -= i; +}; + +const to_array = (list: Linked_List): Array => { + var array = []; + var node = list.head.next; + while (node !== list.tail) { + array.push(node!.value!); + node = node!.next; + } + return array; +}; + +const match_pattern = ( + pattern: RegExp, + pos: number, + text: string, + lookbehind: boolean, +): RegExpExecArray | null => { + pattern.lastIndex = pos; + var match = pattern.exec(text); + if (match && lookbehind && match[1]) { + // change the match to remove the text matched by the lookbehind group + var lookbehind_length = match[1].length; + match.index += lookbehind_length; + match[0] = match[0].substring(lookbehind_length); + } + return match; +}; diff --git a/src/routes/moss.css b/src/routes/moss.css index ed974e45..578b6bcb 100644 --- a/src/routes/moss.css +++ b/src/routes/moss.css @@ -1,15 +1,5 @@ /* generated by src/routes/moss.gen.css.ts */ -/* * - * File statistics: - * - Total files in filer: 285 - * - External dependencies: 235 - * - Internal project files: 50 - * - Files processed (passed filter): 250 - * - Files with CSS classes: 23 - * - Unique classes found: 41 - */ - .box { display: flex; flex-direction: column; @@ -60,14 +50,14 @@ } .chip { - font-weight: 600; + font-weight: 500; padding-left: var(--space_xs); padding-right: var(--space_xs); background-color: var(--fg_1); border-radius: var(--border_radius_xs); } a.chip { - font-weight: 700; + font-weight: 600; } .pre { diff --git a/src/routes/moss.gen.css.ts b/src/routes/moss.gen.css.ts index 9cd1cecf..cc6bc097 100644 --- a/src/routes/moss.gen.css.ts +++ b/src/routes/moss.gen.css.ts @@ -1,3 +1,3 @@ import {gen_moss_css} from '@ryanatkn/moss/gen_moss_css.js'; -export const gen = gen_moss_css({include_stats: true}); +export const gen = gen_moss_css(); diff --git a/src/routes/package.ts b/src/routes/package.ts index 91af80a7..879eb263 100644 --- a/src/routes/package.ts +++ b/src/routes/package.ts @@ -37,9 +37,9 @@ export const package_json: Package_Json = { '@changesets/changelog-git': '^0.2.1', '@ryanatkn/belt': '^0.34.1', '@ryanatkn/eslint-config': '^0.8.0', - '@ryanatkn/fuz': '^0.145.0', - '@ryanatkn/gro': '^0.165.0', - '@ryanatkn/moss': '^0.33.0', + '@ryanatkn/fuz': '^0.146.0', + '@ryanatkn/gro': '^0.165.1', + '@ryanatkn/moss': '^0.34.0', '@sveltejs/adapter-static': '^3.0.9', '@sveltejs/kit': '^2.37.1', '@sveltejs/package': '^2.5.0', @@ -143,14 +143,11 @@ export const src_json: Src_Json = { './syntax_styler.js': { path: 'syntax_styler.ts', declarations: [ - {name: 'Add_Grammar', kind: 'type'}, + {name: 'Add_Syntax_Grammar', kind: 'type'}, {name: 'Syntax_Styler', kind: 'class'}, - {name: 'Grammar_Value', kind: 'type'}, - {name: 'Grammar', kind: 'type'}, - {name: 'Grammar_Token', kind: 'type'}, - {name: 'tokenize_syntax', kind: 'function'}, - {name: 'Syntax_Token', kind: 'class'}, - {name: 'Syntax_Token_Stream', kind: 'type'}, + {name: 'Syntax_Grammar_Value', kind: 'type'}, + {name: 'Syntax_Grammar', kind: 'type'}, + {name: 'Syntax_Grammar_Token', kind: 'type'}, {name: 'Hook_Before_Tokenize_Callback', kind: 'type'}, {name: 'Hook_After_Tokenize_Callback', kind: 'type'}, {name: 'Hook_Wrap_Callback', kind: 'type'}, @@ -159,6 +156,17 @@ export const src_json: Src_Json = { {name: 'Hook_Wrap_Callback_Context', kind: 'type'}, ], }, + './syntax_token.js': { + path: 'syntax_token.ts', + declarations: [ + {name: 'Syntax_Token', kind: 'class'}, + {name: 'Syntax_Token_Stream', kind: 'type'}, + ], + }, + './tokenize_syntax.js': { + path: 'tokenize_syntax.ts', + declarations: [{name: 'tokenize_syntax', kind: 'function'}], + }, './Code.svelte': {path: 'Code.svelte', declarations: [{name: 'default', kind: 'component'}]}, './theme_variables.css': { path: 'theme_variables.css',