diff --git a/.changeset/floppy-coins-cry.md b/.changeset/floppy-coins-cry.md new file mode 100644 index 00000000..763a763d --- /dev/null +++ b/.changeset/floppy-coins-cry.md @@ -0,0 +1,5 @@ +--- +'@ryanatkn/fuz_code': minor +--- + +improve token styles diff --git a/CLAUDE.md b/CLAUDE.md index b0e46c0e..906316bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ See [README.md](./README.md). ```bash gro test # Run all tests gro test src/fixtures/check.test.ts # Verify fixture generation -npm run update-generated-fixtures # Regenerate fixtures +gro src/fixtures/update # Regenerate fixtures npm run benchmark # Run performance benchmarks npm run benchmark-compare # Compare performance with Prism and Shiki ``` @@ -168,7 +168,7 @@ Generated fixtures in `src/fixtures/{lang}/`: ### Workflow 1. Edit samples in `src/lib/samples/` -2. Run `npm run update-generated-fixtures` to regenerate +2. Run `gro src/fixtures/update` to regenerate 3. Run `gro test src/fixtures/check.test.ts` to verify 4. Review changes with `git diff src/fixtures/` diff --git a/README.md b/README.md index 64510254..8b53a565 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,9 @@ but there are two optional dependencies: based on [`prism-svelte`](https://github.com/pngwn/prism-svelte) and a [Svelte component](src/lib/Code.svelte) for convenient usage. - The [default theme](src/lib/theme.css) integrates - with my CSS library [Moss](https://github.com/ryanatkn/moss) for colors that adapt to the user's runtime `color-scheme` preference, - and [theme_variables.css](src/lib/theme_variables.css) - is also included that uses the less-customizable `light-dark()`. + with my CSS library [Moss](https://github.com/ryanatkn/moss) for colors that adapt to the user's runtime `color-scheme` preference. + Non-Moss users should import [theme_variables.css](src/lib/theme_variables.css) + or otherwise define those variables. Compared to [Shiki](https://github.com/shikijs/shiki), this library is much lighter diff --git a/fixtures/generated/css/css_complex.json b/fixtures/generated/css/css_complex.json deleted file mode 100644 index caf89022..00000000 --- a/fixtures/generated/css/css_complex.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "sample": { - "lang": "css", - "variant": "complex", - "content": ".some_class {\n\tcolor: red;\n}\n\n.hypen-class {\n\tfont-size: 16px;\n}\n\np {\n\tbox-shadow: 0 0 10px rgba(0, 0, 0, 0.1);\n}\n\n/* comment */\n\n/*\nmulti\n.line {\n\ti: 100px\n\n\n\n@media*/\n\n#id {\n\tbackground-color: blue;\n}\n\ndiv > p {\n\tmargin: 10px;\n}\n\n@media (max-width: 600px) {\n\tbody {\n\t\tbackground-color: lightblue;\n\t}\n}\n\n/* patterns in strings are not falsely detected */\n.content::before {\n\tcontent: ' /* not a comment */';\n}\n\n.attr[title='Click: here'] {\n\tcursor: pointer;\n}\n\n.background /* comment*/ {\n\tbackground-image: url('data:image/svg+xml...');\n}\n", - "filepath": "src/lib/samples/sample_complex.css" - }, - "tokens": [ - {"type": "selector", "start": 0, "end": 11}, - {"type": "punctuation", "start": 12, "end": 13}, - {"type": "property", "start": 15, "end": 20}, - {"type": "punctuation", "start": 20, "end": 21}, - {"type": "punctuation", "start": 25, "end": 26}, - {"type": "punctuation", "start": 27, "end": 28}, - {"type": "selector", "start": 30, "end": 42}, - {"type": "punctuation", "start": 43, "end": 44}, - {"type": "property", "start": 46, "end": 55}, - {"type": "punctuation", "start": 55, "end": 56}, - {"type": "punctuation", "start": 61, "end": 62}, - {"type": "punctuation", "start": 63, "end": 64}, - {"type": "selector", "start": 66, "end": 67}, - {"type": "punctuation", "start": 68, "end": 69}, - {"type": "property", "start": 71, "end": 81}, - {"type": "punctuation", "start": 81, "end": 82}, - {"type": "function", "start": 92, "end": 96}, - {"type": "punctuation", "start": 96, "end": 97}, - {"type": "punctuation", "start": 98, "end": 99}, - {"type": "punctuation", "start": 101, "end": 102}, - {"type": "punctuation", "start": 104, "end": 105}, - {"type": "punctuation", "start": 109, "end": 110}, - {"type": "punctuation", "start": 110, "end": 111}, - {"type": "punctuation", "start": 112, "end": 113}, - {"type": "comment", "start": 115, "end": 128}, - {"type": "comment", "start": 130, "end": 176}, - {"type": "selector", "start": 178, "end": 181}, - {"type": "punctuation", "start": 182, "end": 183}, - {"type": "property", "start": 185, "end": 201}, - {"type": "punctuation", "start": 201, "end": 202}, - {"type": "punctuation", "start": 207, "end": 208}, - {"type": "punctuation", "start": 209, "end": 210}, - {"type": "selector", "start": 212, "end": 219}, - {"type": "punctuation", "start": 220, "end": 221}, - {"type": "property", "start": 223, "end": 229}, - {"type": "punctuation", "start": 229, "end": 230}, - {"type": "punctuation", "start": 235, "end": 236}, - {"type": "punctuation", "start": 237, "end": 238}, - {"type": "atrule", "start": 240, "end": 265}, - {"type": "rule", "start": 240, "end": 246}, - {"type": "punctuation", "start": 247, "end": 248}, - {"type": "property", "start": 248, "end": 257}, - {"type": "punctuation", "start": 257, "end": 258}, - {"type": "punctuation", "start": 264, "end": 265}, - {"type": "punctuation", "start": 266, "end": 267}, - {"type": "selector", "start": 269, "end": 273}, - {"type": "punctuation", "start": 274, "end": 275}, - {"type": "property", "start": 278, "end": 294}, - {"type": "punctuation", "start": 294, "end": 295}, - {"type": "punctuation", "start": 305, "end": 306}, - {"type": "punctuation", "start": 308, "end": 309}, - {"type": "punctuation", "start": 310, "end": 311}, - {"type": "comment", "start": 313, "end": 363}, - {"type": "selector", "start": 364, "end": 380}, - {"type": "punctuation", "start": 381, "end": 382}, - {"type": "property", "start": 384, "end": 391}, - {"type": "punctuation", "start": 391, "end": 392}, - {"type": "string", "start": 393, "end": 423}, - {"type": "punctuation", "start": 423, "end": 424}, - {"type": "punctuation", "start": 425, "end": 426}, - {"type": "selector", "start": 428, "end": 454}, - {"type": "punctuation", "start": 455, "end": 456}, - {"type": "property", "start": 458, "end": 464}, - {"type": "punctuation", "start": 464, "end": 465}, - {"type": "punctuation", "start": 473, "end": 474}, - {"type": "punctuation", "start": 475, "end": 476}, - {"type": "comment", "start": 491, "end": 503}, - {"type": "punctuation", "start": 504, "end": 505}, - {"type": "property", "start": 507, "end": 523}, - {"type": "punctuation", "start": 523, "end": 524}, - {"type": "url", "start": 525, "end": 553}, - {"type": "function", "start": 525, "end": 528}, - {"type": "punctuation", "start": 528, "end": 529}, - {"type": "string", "start": 529, "end": 552}, - {"type": "punctuation", "start": 552, "end": 553}, - {"type": "punctuation", "start": 553, "end": 554}, - {"type": "punctuation", "start": 555, "end": 556} - ], - "html": ".some_class {\n\tcolor: red;\n}\n\n.hypen-class {\n\tfont-size: 16px;\n}\n\np {\n\tbox-shadow: 0 0 10px rgba(0, 0, 0, 0.1);\n}\n\n/* comment */\n\n/*\nmulti\n.line {\n\ti: 100px\n\n</style>\n\n@media*/\n\n#id {\n\tbackground-color: blue;\n}\n\ndiv > p {\n\tmargin: 10px;\n}\n\n@media (max-width: 600px) {\n\tbody {\n\t\tbackground-color: lightblue;\n\t}\n}\n\n/* patterns in strings are not falsely detected */\n.content::before {\n\tcontent: '</style> /* not a comment */';\n}\n\n.attr[title='Click: here'] {\n\tcursor: pointer;\n}\n\n.background /* comment*/ {\n\tbackground-image: url('data:image/svg+xml...');\n}\n" -} diff --git a/fixtures/generated/html/html_complex.json b/fixtures/generated/html/html_complex.json deleted file mode 100644 index 72a85484..00000000 --- a/fixtures/generated/html/html_complex.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "sample": { - "lang": "html", - "variant": "complex", - "content": "\n\n
\n\t

hello world!

\n
\n\n

some text

\n\n\n\n\n\n
\n\n
\n\n\"access\"\n\n\n\n\n\n\n\n ]]>\n", - "filepath": "src/lib/samples/sample_complex.html" - }, - "tokens": [ - {"type": "doctype", "start": 0, "end": 15}, - {"type": "tag", "start": 17, "end": 35}, - {"type": "tag", "start": 17, "end": 21}, - {"type": "punctuation", "start": 17, "end": 18}, - {"type": "attr_name", "start": 22, "end": 27}, - {"type": "attr_value", "start": 27, "end": 34}, - {"type": "punctuation", "start": 27, "end": 28}, - {"type": "punctuation", "start": 28, "end": 29}, - {"type": "punctuation", "start": 33, "end": 34}, - {"type": "punctuation", "start": 34, "end": 35}, - {"type": "tag", "start": 37, "end": 40}, - {"type": "tag", "start": 37, "end": 39}, - {"type": "punctuation", "start": 37, "end": 38}, - {"type": "punctuation", "start": 39, "end": 40}, - {"type": "tag", "start": 52, "end": 56}, - {"type": "tag", "start": 52, "end": 55}, - {"type": "punctuation", "start": 52, "end": 54}, - {"type": "punctuation", "start": 55, "end": 56}, - {"type": "tag", "start": 57, "end": 63}, - {"type": "tag", "start": 57, "end": 62}, - {"type": "punctuation", "start": 57, "end": 59}, - {"type": "punctuation", "start": 62, "end": 63}, - {"type": "tag", "start": 65, "end": 99}, - {"type": "tag", "start": 65, "end": 67}, - {"type": "punctuation", "start": 65, "end": 66}, - {"type": "attr_name", "start": 68, "end": 73}, - {"type": "attr_value", "start": 73, "end": 98}, - {"type": "punctuation", "start": 73, "end": 74}, - {"type": "punctuation", "start": 74, "end": 75}, - {"type": "punctuation", "start": 97, "end": 98}, - {"type": "punctuation", "start": 98, "end": 99}, - {"type": "tag", "start": 104, "end": 124}, - {"type": "tag", "start": 104, "end": 109}, - {"type": "punctuation", "start": 104, "end": 105}, - {"type": "attr_name", "start": 110, "end": 115}, - {"type": "attr_value", "start": 115, "end": 123}, - {"type": "punctuation", "start": 115, "end": 116}, - {"type": "punctuation", "start": 116, "end": 117}, - {"type": "punctuation", "start": 122, "end": 123}, - {"type": "punctuation", "start": 123, "end": 124}, - {"type": "tag", "start": 128, "end": 135}, - {"type": "tag", "start": 128, "end": 134}, - {"type": "punctuation", "start": 128, "end": 130}, - {"type": "punctuation", "start": 134, "end": 135}, - {"type": "tag", "start": 135, "end": 139}, - {"type": "tag", "start": 135, "end": 138}, - {"type": "punctuation", "start": 135, "end": 137}, - {"type": "punctuation", "start": 138, "end": 139}, - {"type": "tag", "start": 141, "end": 172}, - {"type": "tag", "start": 141, "end": 148}, - {"type": "punctuation", "start": 141, "end": 142}, - {"type": "attr_name", "start": 149, "end": 153}, - {"type": "attr_value", "start": 153, "end": 162}, - {"type": "punctuation", "start": 153, "end": 154}, - {"type": "punctuation", "start": 154, "end": 155}, - {"type": "punctuation", "start": 161, "end": 162}, - {"type": "attr_name", "start": 163, "end": 171}, - {"type": "punctuation", "start": 171, "end": 172}, - {"type": "tag", "start": 180, "end": 189}, - {"type": "tag", "start": 180, "end": 188}, - {"type": "punctuation", "start": 180, "end": 182}, - {"type": "punctuation", "start": 188, "end": 189}, - {"type": "comment", "start": 191, "end": 237}, - {"type": "tag", "start": 239, "end": 245}, - {"type": "tag", "start": 239, "end": 242}, - {"type": "punctuation", "start": 239, "end": 240}, - {"type": "punctuation", "start": 243, "end": 245}, - {"type": "tag", "start": 247, "end": 253}, - {"type": "tag", "start": 247, "end": 250}, - {"type": "punctuation", "start": 247, "end": 248}, - {"type": "punctuation", "start": 251, "end": 253}, - {"type": "tag", "start": 255, "end": 291}, - {"type": "tag", "start": 255, "end": 259}, - {"type": "punctuation", "start": 255, "end": 256}, - {"type": "attr_name", "start": 260, "end": 263}, - {"type": "attr_value", "start": 263, "end": 275}, - {"type": "punctuation", "start": 263, "end": 264}, - {"type": "punctuation", "start": 264, "end": 265}, - {"type": "punctuation", "start": 274, "end": 275}, - {"type": "attr_name", "start": 276, "end": 279}, - {"type": "attr_value", "start": 279, "end": 288}, - {"type": "punctuation", "start": 279, "end": 280}, - {"type": "punctuation", "start": 280, "end": 281}, - {"type": "punctuation", "start": 287, "end": 288}, - {"type": "punctuation", "start": 289, "end": 291}, - {"type": "tag", "start": 293, "end": 297}, - {"type": "tag", "start": 293, "end": 296}, - {"type": "punctuation", "start": 293, "end": 294}, - {"type": "punctuation", "start": 296, "end": 297}, - {"type": "tag", "start": 299, "end": 303}, - {"type": "tag", "start": 299, "end": 302}, - {"type": "punctuation", "start": 299, "end": 300}, - {"type": "punctuation", "start": 302, "end": 303}, - {"type": "tag", "start": 314, "end": 319}, - {"type": "tag", "start": 314, "end": 318}, - {"type": "punctuation", "start": 314, "end": 316}, - {"type": "punctuation", "start": 318, "end": 319}, - {"type": "tag", "start": 321, "end": 325}, - {"type": "tag", "start": 321, "end": 324}, - {"type": "punctuation", "start": 321, "end": 322}, - {"type": "punctuation", "start": 324, "end": 325}, - {"type": "tag", "start": 336, "end": 341}, - {"type": "tag", "start": 336, "end": 340}, - {"type": "punctuation", "start": 336, "end": 338}, - {"type": "punctuation", "start": 340, "end": 341}, - {"type": "tag", "start": 342, "end": 347}, - {"type": "tag", "start": 342, "end": 346}, - {"type": "punctuation", "start": 342, "end": 344}, - {"type": "punctuation", "start": 346, "end": 347}, - {"type": "tag", "start": 349, "end": 380}, - {"type": "tag", "start": 349, "end": 356}, - {"type": "punctuation", "start": 349, "end": 350}, - {"type": "attr_name", "start": 357, "end": 361}, - {"type": "attr_value", "start": 361, "end": 379}, - {"type": "punctuation", "start": 361, "end": 362}, - {"type": "punctuation", "start": 362, "end": 363}, - {"type": "punctuation", "start": 378, "end": 379}, - {"type": "punctuation", "start": 379, "end": 380}, - {"type": "script", "start": 380, "end": 404}, - {"type": "lang_js", "start": 380, "end": 404}, - {"type": "keyword", "start": 382, "end": 387}, - {"type": "operator", "start": 391, "end": 392}, - {"type": "string", "start": 393, "end": 402}, - {"type": "punctuation", "start": 402, "end": 403}, - {"type": "tag", "start": 404, "end": 413}, - {"type": "tag", "start": 404, "end": 412}, - {"type": "punctuation", "start": 404, "end": 406}, - {"type": "punctuation", "start": 412, "end": 413}, - {"type": "tag", "start": 415, "end": 438}, - {"type": "tag", "start": 415, "end": 421}, - {"type": "punctuation", "start": 415, "end": 416}, - {"type": "attr_name", "start": 422, "end": 426}, - {"type": "attr_value", "start": 426, "end": 437}, - {"type": "punctuation", "start": 426, "end": 427}, - {"type": "punctuation", "start": 427, "end": 428}, - {"type": "punctuation", "start": 436, "end": 437}, - {"type": "punctuation", "start": 437, "end": 438}, - {"type": "style", "start": 438, "end": 482}, - {"type": "lang_css", "start": 438, "end": 482}, - {"type": "selector", "start": 440, "end": 456}, - {"type": "punctuation", "start": 457, "end": 458}, - {"type": "property", "start": 461, "end": 468}, - {"type": "punctuation", "start": 468, "end": 469}, - {"type": "string", "start": 470, "end": 477}, - {"type": "punctuation", "start": 477, "end": 478}, - {"type": "punctuation", "start": 480, "end": 481}, - {"type": "tag", "start": 482, "end": 490}, - {"type": "tag", "start": 482, "end": 489}, - {"type": "punctuation", "start": 482, "end": 484}, - {"type": "punctuation", "start": 489, "end": 490}, - {"type": "cdata", "start": 492, "end": 540} - ], - "html": "<!doctype html>\n\n<div class=\"test\">\n\t<p>hello world!</p>\n</div>\n\n<p class=\"some_class hypen-class\">some <span class=\"a b c\">text</span></p>\n\n<button type=\"button\" disabled>click me</button>\n\n<!-- comment <div>a<br /> b</div> <script> -->\n\n<br />\n\n<hr />\n\n<img src=\"image.jpg\" alt=\"access\" />\n\n<ul>\n\t<li>list item 1</li>\n\t<li>list item 2</li>\n</ul>\n\n<script type=\"text/javascript\">\n\tconst ok = '<style>';\n</script>\n\n<style type=\"text/css\">\n\t.special::before {\n\t\tcontent: '< & >';\n\t}\n</style>\n\n<![CDATA[ if (a < 0) alert(\"b\"); <not-a-tag> ]]>\n" -} diff --git a/fixtures/generated/html/html_complex.txt b/fixtures/generated/html/html_complex.txt deleted file mode 100644 index cd8b4eba..00000000 --- a/fixtures/generated/html/html_complex.txt +++ /dev/null @@ -1,174 +0,0 @@ -=== TOKENS === - 0-15 doctype - 17-35 tag
- 17-21 tag
- 37-40 tag

- 37-39 tag

- 52-56 tag

- 52-55 tag

- 57-63 tag
- 57-62 tag
- 65-99 tag

- 65-67 tag

- 104-124 tag - 104-109 tag - 128-135 tag - 128-134 tag - 135-139 tag

- 135-138 tag

- 141-172 tag - 180-188 tag - 191-237 comment - 239-245 tag
- 239-242 tag
- 247-253 tag
- 247-250 tag
- 255-291 tag access - 255-259 tag - 293-297 tag - 349-380 tag - 404-412 tag - 415-438 tag - 482-489 tag - 492-540 cdata ]]> - -=== STATS === -Total tokens: 151 -Sample length: 541 characters - -Token types: - punctuation: 75 - tag: 46 - attr_name: 9 - attr_value: 8 - string: 2 - doctype: 1 - comment: 1 - script: 1 - lang_js: 1 - keyword: 1 - operator: 1 - style: 1 - lang_css: 1 - selector: 1 - property: 1 - cdata: 1 diff --git a/fixtures/generated/json/json_complex.json b/fixtures/generated/json/json_complex.json deleted file mode 100644 index 2b43c70d..00000000 --- a/fixtures/generated/json/json_complex.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "sample": { - "lang": "json", - "variant": "complex", - "content": "{\n\t\"string\": \"a string\",\n\t\"number\": 12345,\n\t\"boolean\": true,\n\t\"null\": null,\n\t\"empty\": \"\",\n\t\"escaped\": \"quote: \\\"test\\\" and backslash: \\\\\",\n\t\"object\": {\n\t\t\"array\": [1, \"b\", false],\n\t\t\"strings\": [\"1\", \"2\", \"3\"],\n\t\t\"mixed\": [\"start\", 123, true, \"middle\", null, \"end\"],\n\t\t\"nested\": [[\"a\", \"str\", \"\"], {\"key\": \"nested value\"}]\n\t}\n}\n", - "filepath": "src/lib/samples/sample_complex.json" - }, - "tokens": [ - {"type": "punctuation", "start": 0, "end": 1}, - {"type": "property", "start": 3, "end": 11}, - {"type": "operator", "start": 11, "end": 12}, - {"type": "string", "start": 13, "end": 23}, - {"type": "punctuation", "start": 23, "end": 24}, - {"type": "property", "start": 26, "end": 34}, - {"type": "operator", "start": 34, "end": 35}, - {"type": "number", "start": 36, "end": 41}, - {"type": "punctuation", "start": 41, "end": 42}, - {"type": "property", "start": 44, "end": 53}, - {"type": "operator", "start": 53, "end": 54}, - {"type": "boolean", "start": 55, "end": 59}, - {"type": "punctuation", "start": 59, "end": 60}, - {"type": "property", "start": 62, "end": 68}, - {"type": "operator", "start": 68, "end": 69}, - {"type": "null", "start": 70, "end": 74}, - {"type": "punctuation", "start": 74, "end": 75}, - {"type": "property", "start": 77, "end": 84}, - {"type": "operator", "start": 84, "end": 85}, - {"type": "string", "start": 86, "end": 88}, - {"type": "punctuation", "start": 88, "end": 89}, - {"type": "property", "start": 91, "end": 100}, - {"type": "operator", "start": 100, "end": 101}, - {"type": "string", "start": 102, "end": 137}, - {"type": "punctuation", "start": 137, "end": 138}, - {"type": "property", "start": 140, "end": 148}, - {"type": "operator", "start": 148, "end": 149}, - {"type": "punctuation", "start": 150, "end": 151}, - {"type": "property", "start": 154, "end": 161}, - {"type": "operator", "start": 161, "end": 162}, - {"type": "punctuation", "start": 163, "end": 164}, - {"type": "number", "start": 164, "end": 165}, - {"type": "punctuation", "start": 165, "end": 166}, - {"type": "string", "start": 167, "end": 170}, - {"type": "punctuation", "start": 170, "end": 171}, - {"type": "boolean", "start": 172, "end": 177}, - {"type": "punctuation", "start": 177, "end": 178}, - {"type": "punctuation", "start": 178, "end": 179}, - {"type": "property", "start": 182, "end": 191}, - {"type": "operator", "start": 191, "end": 192}, - {"type": "punctuation", "start": 193, "end": 194}, - {"type": "string", "start": 194, "end": 197}, - {"type": "punctuation", "start": 197, "end": 198}, - {"type": "string", "start": 199, "end": 202}, - {"type": "punctuation", "start": 202, "end": 203}, - {"type": "string", "start": 204, "end": 207}, - {"type": "punctuation", "start": 207, "end": 208}, - {"type": "punctuation", "start": 208, "end": 209}, - {"type": "property", "start": 212, "end": 219}, - {"type": "operator", "start": 219, "end": 220}, - {"type": "punctuation", "start": 221, "end": 222}, - {"type": "string", "start": 222, "end": 229}, - {"type": "punctuation", "start": 229, "end": 230}, - {"type": "number", "start": 231, "end": 234}, - {"type": "punctuation", "start": 234, "end": 235}, - {"type": "boolean", "start": 236, "end": 240}, - {"type": "punctuation", "start": 240, "end": 241}, - {"type": "string", "start": 242, "end": 250}, - {"type": "punctuation", "start": 250, "end": 251}, - {"type": "null", "start": 252, "end": 256}, - {"type": "punctuation", "start": 256, "end": 257}, - {"type": "string", "start": 258, "end": 263}, - {"type": "punctuation", "start": 263, "end": 264}, - {"type": "punctuation", "start": 264, "end": 265}, - {"type": "property", "start": 268, "end": 276}, - {"type": "operator", "start": 276, "end": 277}, - {"type": "punctuation", "start": 278, "end": 279}, - {"type": "punctuation", "start": 279, "end": 280}, - {"type": "string", "start": 280, "end": 283}, - {"type": "punctuation", "start": 283, "end": 284}, - {"type": "string", "start": 285, "end": 290}, - {"type": "punctuation", "start": 290, "end": 291}, - {"type": "string", "start": 292, "end": 294}, - {"type": "punctuation", "start": 294, "end": 295}, - {"type": "punctuation", "start": 295, "end": 296}, - {"type": "punctuation", "start": 297, "end": 298}, - {"type": "property", "start": 298, "end": 303}, - {"type": "operator", "start": 303, "end": 304}, - {"type": "string", "start": 305, "end": 319}, - {"type": "punctuation", "start": 319, "end": 320}, - {"type": "punctuation", "start": 320, "end": 321}, - {"type": "punctuation", "start": 323, "end": 324}, - {"type": "punctuation", "start": 325, "end": 326} - ], - "html": "{\n\t\"string\": \"a string\",\n\t\"number\": 12345,\n\t\"boolean\": true,\n\t\"null\": null,\n\t\"empty\": \"\",\n\t\"escaped\": \"quote: \\\"test\\\" and backslash: \\\\\",\n\t\"object\": {\n\t\t\"array\": [1, \"b\", false],\n\t\t\"strings\": [\"1\", \"2\", \"3\"],\n\t\t\"mixed\": [\"start\", 123, true, \"middle\", null, \"end\"],\n\t\t\"nested\": [[\"a\", \"str\", \"\"], {\"key\": \"nested value\"}]\n\t}\n}\n" -} diff --git a/fixtures/generated/svelte/svelte_complex.json b/fixtures/generated/svelte/svelte_complex.json deleted file mode 100644 index 16d8e811..00000000 --- a/fixtures/generated/svelte/svelte_complex.json +++ /dev/null @@ -1,587 +0,0 @@ -{ - "sample": { - "lang": "svelte", - "variant": "complex", - "content": "\n\n\n\n

hello {HELLO}!

\n\n{#each thing_keys as key (key)}\n\t{@const value = thing[key]}\n\t{value}\n{/each}\n\n{#if c}\n\t\n{:else}\n\t (c = !c)}>\n\t\t{@render children()}\n\t\n{/if}\n\n\n\n
\n\t

hello world!

\n
\n\n

\n\tsome text\n

\n\n\n\n\n{a}\n{b}\n{bound}\n{D}\n\n
\n\n
\n\n\"access\"\n\n\n\n\n", - "filepath": "src/lib/samples/sample_complex.svelte" - }, - "tokens": [ - {"type": "tag", "start": 0, "end": 25}, - {"type": "tag", "start": 0, "end": 7}, - {"type": "punctuation", "start": 0, "end": 1}, - {"type": "attr_name", "start": 8, "end": 12}, - {"type": "attr_value", "start": 12, "end": 17}, - {"type": "punctuation", "start": 12, "end": 13}, - {"type": "punctuation", "start": 13, "end": 14}, - {"type": "punctuation", "start": 16, "end": 17}, - {"type": "attr_name", "start": 18, "end": 24}, - {"type": "punctuation", "start": 24, "end": 25}, - {"type": "script", "start": 25, "end": 57}, - {"type": "lang_ts", "start": 25, "end": 57}, - {"type": "keyword", "start": 27, "end": 33}, - {"type": "keyword", "start": 34, "end": 39}, - {"type": "constant", "start": 40, "end": 45}, - {"type": "operator", "start": 46, "end": 47}, - {"type": "string", "start": 48, "end": 55}, - {"type": "punctuation", "start": 55, "end": 56}, - {"type": "tag", "start": 57, "end": 66}, - {"type": "tag", "start": 57, "end": 65}, - {"type": "punctuation", "start": 57, "end": 59}, - {"type": "punctuation", "start": 65, "end": 66}, - {"type": "tag", "start": 68, "end": 86}, - {"type": "tag", "start": 68, "end": 75}, - {"type": "punctuation", "start": 68, "end": 69}, - {"type": "attr_name", "start": 76, "end": 80}, - {"type": "attr_value", "start": 80, "end": 85}, - {"type": "punctuation", "start": 80, "end": 81}, - {"type": "punctuation", "start": 81, "end": 82}, - {"type": "punctuation", "start": 84, "end": 85}, - {"type": "punctuation", "start": 85, "end": 86}, - {"type": "script", "start": 86, "end": 1268}, - {"type": "lang_ts", "start": 86, "end": 1268}, - {"type": "comment", "start": 88, "end": 107}, - {"type": "keyword", "start": 109, "end": 115}, - {"type": "keyword", "start": 122, "end": 126}, - {"type": "string", "start": 127, "end": 146}, - {"type": "punctuation", "start": 146, "end": 147}, - {"type": "keyword", "start": 149, "end": 155}, - {"type": "keyword", "start": 156, "end": 160}, - {"type": "punctuation", "start": 161, "end": 162}, - {"type": "punctuation", "start": 169, "end": 170}, - {"type": "keyword", "start": 171, "end": 175}, - {"type": "string", "start": 176, "end": 184}, - {"type": "punctuation", "start": 184, "end": 185}, - {"type": "keyword", "start": 188, "end": 193}, - {"type": "punctuation", "start": 194, "end": 195}, - {"type": "punctuation", "start": 203, "end": 204}, - {"type": "operator", "start": 213, "end": 214}, - {"type": "function", "start": 215, "end": 224}, - {"type": "punctuation", "start": 224, "end": 225}, - {"type": "boolean", "start": 225, "end": 229}, - {"type": "punctuation", "start": 229, "end": 230}, - {"type": "punctuation", "start": 230, "end": 231}, - {"type": "punctuation", "start": 242, "end": 243}, - {"type": "punctuation", "start": 245, "end": 246}, - {"type": "operator", "start": 246, "end": 247}, - {"type": "punctuation", "start": 248, "end": 249}, - {"type": "operator", "start": 257, "end": 258}, - {"type": "operator", "start": 265, "end": 266}, - {"type": "builtin", "start": 266, "end": 272}, - {"type": "punctuation", "start": 272, "end": 273}, - {"type": "builtin", "start": 274, "end": 277}, - {"type": "operator", "start": 277, "end": 278}, - {"type": "punctuation", "start": 278, "end": 279}, - {"type": "operator", "start": 287, "end": 288}, - {"type": "operator", "start": 288, "end": 289}, - {"type": "builtin", "start": 290, "end": 297}, - {"type": "punctuation", "start": 297, "end": 298}, - {"type": "operator", "start": 309, "end": 310}, - {"type": "punctuation", "start": 318, "end": 319}, - {"type": "punctuation", "start": 321, "end": 322}, - {"type": "operator", "start": 323, "end": 324}, - {"type": "function", "start": 325, "end": 331}, - {"type": "punctuation", "start": 331, "end": 332}, - {"type": "punctuation", "start": 332, "end": 333}, - {"type": "punctuation", "start": 333, "end": 334}, - {"type": "keyword", "start": 337, "end": 342}, - {"type": "operator", "start": 354, "end": 355}, - {"type": "function", "start": 356, "end": 364}, - {"type": "punctuation", "start": 364, "end": 365}, - {"type": "punctuation", "start": 371, "end": 372}, - {"type": "function", "start": 372, "end": 376}, - {"type": "punctuation", "start": 376, "end": 377}, - {"type": "punctuation", "start": 382, "end": 383}, - {"type": "punctuation", "start": 383, "end": 384}, - {"type": "punctuation", "start": 384, "end": 385}, - {"type": "keyword", "start": 388, "end": 393}, - {"type": "operator", "start": 396, "end": 397}, - {"type": "number", "start": 398, "end": 399}, - {"type": "punctuation", "start": 399, "end": 400}, - {"type": "keyword", "start": 403, "end": 408}, - {"type": "operator", "start": 411, "end": 412}, - {"type": "string", "start": 413, "end": 416}, - {"type": "punctuation", "start": 416, "end": 417}, - {"type": "keyword", "start": 420, "end": 423}, - {"type": "operator", "start": 425, "end": 426}, - {"type": "builtin", "start": 427, "end": 434}, - {"type": "operator", "start": 435, "end": 436}, - {"type": "function", "start": 437, "end": 443}, - {"type": "punctuation", "start": 443, "end": 444}, - {"type": "boolean", "start": 444, "end": 448}, - {"type": "punctuation", "start": 448, "end": 449}, - {"type": "punctuation", "start": 449, "end": 450}, - {"type": "keyword", "start": 453, "end": 459}, - {"type": "keyword", "start": 460, "end": 464}, - {"type": "class_name", "start": 465, "end": 474}, - {"type": "operator", "start": 475, "end": 476}, - {"type": "number", "start": 477, "end": 478}, - {"type": "operator", "start": 479, "end": 480}, - {"type": "string", "start": 481, "end": 484}, - {"type": "operator", "start": 485, "end": 486}, - {"type": "boolean", "start": 487, "end": 491}, - {"type": "punctuation", "start": 491, "end": 492}, - {"type": "keyword", "start": 495, "end": 500}, - {"type": "class_name", "start": 501, "end": 502}, - {"type": "constant", "start": 501, "end": 502}, - {"type": "punctuation", "start": 503, "end": 504}, - {"type": "operator", "start": 509, "end": 510}, - {"type": "builtin", "start": 511, "end": 517}, - {"type": "operator", "start": 518, "end": 519}, - {"type": "string", "start": 520, "end": 523}, - {"type": "punctuation", "start": 523, "end": 524}, - {"type": "operator", "start": 529, "end": 530}, - {"type": "builtin", "start": 531, "end": 537}, - {"type": "punctuation", "start": 537, "end": 538}, - {"type": "operator", "start": 544, "end": 545}, - {"type": "function", "start": 546, "end": 552}, - {"type": "punctuation", "start": 552, "end": 553}, - {"type": "boolean", "start": 553, "end": 558}, - {"type": "punctuation", "start": 558, "end": 559}, - {"type": "punctuation", "start": 559, "end": 560}, - {"type": "function", "start": 564, "end": 575}, - {"type": "punctuation", "start": 575, "end": 576}, - {"type": "operator", "start": 578, "end": 579}, - {"type": "builtin", "start": 580, "end": 586}, - {"type": "punctuation", "start": 586, "end": 587}, - {"type": "punctuation", "start": 588, "end": 589}, - {"type": "keyword", "start": 593, "end": 597}, - {"type": "punctuation", "start": 597, "end": 598}, - {"type": "operator", "start": 601, "end": 602}, - {"type": "punctuation", "start": 605, "end": 606}, - {"type": "punctuation", "start": 609, "end": 610}, - {"type": "function", "start": 614, "end": 626}, - {"type": "punctuation", "start": 626, "end": 627}, - {"type": "punctuation", "start": 627, "end": 628}, - {"type": "operator", "start": 628, "end": 629}, - {"type": "builtin", "start": 630, "end": 636}, - {"type": "punctuation", "start": 637, "end": 638}, - {"type": "keyword", "start": 642, "end": 648}, - {"type": "template_string", "start": 649, "end": 668}, - {"type": "template_punctuation", "start": 649, "end": 650}, - {"type": "string", "start": 650, "end": 657}, - {"type": "interpolation", "start": 657, "end": 667}, - {"type": "interpolation_punctuation", "start": 657, "end": 659}, - {"type": "keyword", "start": 659, "end": 663}, - {"type": "punctuation", "start": 663, "end": 664}, - {"type": "interpolation_punctuation", "start": 666, "end": 667}, - {"type": "template_punctuation", "start": 667, "end": 668}, - {"type": "punctuation", "start": 668, "end": 669}, - {"type": "punctuation", "start": 672, "end": 673}, - {"type": "function_variable", "start": 677, "end": 692}, - {"type": "operator", "start": 693, "end": 694}, - {"type": "punctuation", "start": 695, "end": 696}, - {"type": "punctuation", "start": 696, "end": 697}, - {"type": "operator", "start": 698, "end": 700}, - {"type": "punctuation", "start": 701, "end": 702}, - {"type": "comment", "start": 706, "end": 715}, - {"type": "keyword", "start": 719, "end": 723}, - {"type": "punctuation", "start": 723, "end": 724}, - {"type": "function", "start": 724, "end": 739}, - {"type": "punctuation", "start": 739, "end": 740}, - {"type": "punctuation", "start": 740, "end": 741}, - {"type": "punctuation", "start": 741, "end": 742}, - {"type": "comment", "start": 746, "end": 752}, - {"type": "punctuation", "start": 755, "end": 756}, - {"type": "punctuation", "start": 756, "end": 757}, - {"type": "function", "start": 761, "end": 776}, - {"type": "punctuation", "start": 776, "end": 777}, - {"type": "punctuation", "start": 777, "end": 778}, - {"type": "punctuation", "start": 779, "end": 780}, - {"type": "keyword", "start": 784, "end": 789}, - {"type": "keyword", "start": 790, "end": 793}, - {"type": "class_name", "start": 794, "end": 799}, - {"type": "punctuation", "start": 799, "end": 800}, - {"type": "template_string", "start": 800, "end": 816}, - {"type": "template_punctuation", "start": 800, "end": 801}, - {"type": "interpolation", "start": 801, "end": 811}, - {"type": "interpolation_punctuation", "start": 801, "end": 803}, - {"type": "keyword", "start": 803, "end": 807}, - {"type": "punctuation", "start": 807, "end": 808}, - {"type": "interpolation_punctuation", "start": 810, "end": 811}, - {"type": "string", "start": 811, "end": 815}, - {"type": "template_punctuation", "start": 815, "end": 816}, - {"type": "punctuation", "start": 816, "end": 817}, - {"type": "punctuation", "start": 817, "end": 818}, - {"type": "punctuation", "start": 821, "end": 822}, - {"type": "keyword", "start": 826, "end": 835}, - {"type": "function", "start": 836, "end": 852}, - {"type": "punctuation", "start": 852, "end": 853}, - {"type": "punctuation", "start": 853, "end": 854}, - {"type": "punctuation", "start": 855, "end": 856}, - {"type": "builtin", "start": 860, "end": 867}, - {"type": "punctuation", "start": 867, "end": 868}, - {"type": "function", "start": 868, "end": 871}, - {"type": "punctuation", "start": 871, "end": 872}, - {"type": "keyword", "start": 872, "end": 875}, - {"type": "class_name", "start": 876, "end": 880}, - {"type": "punctuation", "start": 880, "end": 881}, - {"type": "number", "start": 881, "end": 884}, - {"type": "punctuation", "start": 884, "end": 885}, - {"type": "punctuation", "start": 885, "end": 886}, - {"type": "punctuation", "start": 886, "end": 887}, - {"type": "comment", "start": 888, "end": 921}, - {"type": "punctuation", "start": 924, "end": 925}, - {"type": "punctuation", "start": 927, "end": 928}, - {"type": "comment", "start": 931, "end": 941}, - {"type": "comment", "start": 944, "end": 990}, - {"type": "comment", "start": 993, "end": 1019}, - {"type": "keyword", "start": 1022, "end": 1028}, - {"type": "keyword", "start": 1029, "end": 1038}, - {"type": "class_name", "start": 1039, "end": 1045}, - {"type": "punctuation", "start": 1046, "end": 1047}, - {"type": "operator", "start": 1054, "end": 1055}, - {"type": "builtin", "start": 1056, "end": 1062}, - {"type": "punctuation", "start": 1062, "end": 1063}, - {"type": "operator", "start": 1069, "end": 1070}, - {"type": "builtin", "start": 1071, "end": 1077}, - {"type": "punctuation", "start": 1077, "end": 1078}, - {"type": "punctuation", "start": 1080, "end": 1081}, - {"type": "keyword", "start": 1084, "end": 1090}, - {"type": "keyword", "start": 1091, "end": 1096}, - {"type": "operator", "start": 1103, "end": 1104}, - {"type": "operator", "start": 1112, "end": 1113}, - {"type": "punctuation", "start": 1114, "end": 1115}, - {"type": "operator", "start": 1119, "end": 1120}, - {"type": "string", "start": 1121, "end": 1128}, - {"type": "punctuation", "start": 1128, "end": 1129}, - {"type": "operator", "start": 1133, "end": 1134}, - {"type": "number", "start": 1135, "end": 1138}, - {"type": "punctuation", "start": 1138, "end": 1139}, - {"type": "punctuation", "start": 1139, "end": 1140}, - {"type": "keyword", "start": 1143, "end": 1149}, - {"type": "keyword", "start": 1150, "end": 1158}, - {"type": "function", "start": 1159, "end": 1162}, - {"type": "punctuation", "start": 1162, "end": 1163}, - {"type": "operator", "start": 1164, "end": 1165}, - {"type": "builtin", "start": 1166, "end": 1172}, - {"type": "punctuation", "start": 1172, "end": 1173}, - {"type": "operator", "start": 1175, "end": 1176}, - {"type": "builtin", "start": 1177, "end": 1183}, - {"type": "punctuation", "start": 1183, "end": 1184}, - {"type": "operator", "start": 1184, "end": 1185}, - {"type": "builtin", "start": 1186, "end": 1192}, - {"type": "punctuation", "start": 1193, "end": 1194}, - {"type": "keyword", "start": 1197, "end": 1203}, - {"type": "operator", "start": 1206, "end": 1207}, - {"type": "punctuation", "start": 1209, "end": 1210}, - {"type": "punctuation", "start": 1212, "end": 1213}, - {"type": "keyword", "start": 1216, "end": 1222}, - {"type": "keyword", "start": 1223, "end": 1228}, - {"type": "operator", "start": 1234, "end": 1235}, - {"type": "punctuation", "start": 1236, "end": 1237}, - {"type": "operator", "start": 1238, "end": 1239}, - {"type": "builtin", "start": 1240, "end": 1243}, - {"type": "punctuation", "start": 1243, "end": 1244}, - {"type": "operator", "start": 1246, "end": 1247}, - {"type": "builtin", "start": 1248, "end": 1251}, - {"type": "punctuation", "start": 1251, "end": 1252}, - {"type": "operator", "start": 1252, "end": 1253}, - {"type": "builtin", "start": 1254, "end": 1257}, - {"type": "operator", "start": 1258, "end": 1260}, - {"type": "operator", "start": 1263, "end": 1264}, - {"type": "punctuation", "start": 1266, "end": 1267}, - {"type": "tag", "start": 1268, "end": 1277}, - {"type": "tag", "start": 1268, "end": 1276}, - {"type": "punctuation", "start": 1268, "end": 1270}, - {"type": "punctuation", "start": 1276, "end": 1277}, - {"type": "tag", "start": 1279, "end": 1283}, - {"type": "tag", "start": 1279, "end": 1282}, - {"type": "punctuation", "start": 1279, "end": 1280}, - {"type": "punctuation", "start": 1282, "end": 1283}, - {"type": "lang_ts", "start": 1289, "end": 1296}, - {"type": "punctuation", "start": 1289, "end": 1290}, - {"type": "constant", "start": 1290, "end": 1295}, - {"type": "punctuation", "start": 1295, "end": 1296}, - {"type": "tag", "start": 1297, "end": 1302}, - {"type": "tag", "start": 1297, "end": 1301}, - {"type": "punctuation", "start": 1297, "end": 1299}, - {"type": "punctuation", "start": 1301, "end": 1302}, - {"type": "each", "start": 1304, "end": 1335}, - {"type": "punctuation", "start": 1304, "end": 1305}, - {"type": "keyword", "start": 1305, "end": 1310}, - {"type": "lang_ts", "start": 1311, "end": 1322}, - {"type": "keyword", "start": 1322, "end": 1324}, - {"type": "lang_ts", "start": 1325, "end": 1329}, - {"type": "lang_ts", "start": 1329, "end": 1334}, - {"type": "punctuation", "start": 1329, "end": 1330}, - {"type": "punctuation", "start": 1333, "end": 1334}, - {"type": "punctuation", "start": 1334, "end": 1335}, - {"type": "lang_ts", "start": 1337, "end": 1364}, - {"type": "punctuation", "start": 1337, "end": 1338}, - {"type": "keyword", "start": 1339, "end": 1344}, - {"type": "operator", "start": 1351, "end": 1352}, - {"type": "punctuation", "start": 1358, "end": 1359}, - {"type": "punctuation", "start": 1362, "end": 1363}, - {"type": "punctuation", "start": 1363, "end": 1364}, - {"type": "lang_ts", "start": 1366, "end": 1373}, - {"type": "punctuation", "start": 1366, "end": 1367}, - {"type": "punctuation", "start": 1372, "end": 1373}, - {"type": "each", "start": 1374, "end": 1381}, - {"type": "punctuation", "start": 1374, "end": 1375}, - {"type": "keyword", "start": 1375, "end": 1380}, - {"type": "punctuation", "start": 1380, "end": 1381}, - {"type": "lang_ts", "start": 1383, "end": 1390}, - {"type": "punctuation", "start": 1383, "end": 1384}, - {"type": "keyword", "start": 1385, "end": 1387}, - {"type": "punctuation", "start": 1389, "end": 1390}, - {"type": "tag", "start": 1392, "end": 1433}, - {"type": "tag", "start": 1392, "end": 1398}, - {"type": "punctuation", "start": 1392, "end": 1393}, - {"type": "attr_name", "start": 1399, "end": 1410}, - {"type": "attr_value", "start": 1410, "end": 1414}, - {"type": "punctuation", "start": 1410, "end": 1411}, - {"type": "punctuation", "start": 1411, "end": 1412}, - {"type": "punctuation", "start": 1413, "end": 1414}, - {"type": "attr_name", "start": 1415, "end": 1427}, - {"type": "lang_ts", "start": 1427, "end": 1430}, - {"type": "punctuation", "start": 1427, "end": 1428}, - {"type": "number", "start": 1428, "end": 1429}, - {"type": "punctuation", "start": 1429, "end": 1430}, - {"type": "punctuation", "start": 1431, "end": 1433}, - {"type": "lang_ts", "start": 1434, "end": 1441}, - {"type": "punctuation", "start": 1434, "end": 1435}, - {"type": "operator", "start": 1435, "end": 1436}, - {"type": "keyword", "start": 1436, "end": 1440}, - {"type": "punctuation", "start": 1440, "end": 1441}, - {"type": "tag", "start": 1443, "end": 1507}, - {"type": "tag", "start": 1443, "end": 1449}, - {"type": "punctuation", "start": 1443, "end": 1444}, - {"type": "attr_name", "start": 1450, "end": 1461}, - {"type": "attr_value", "start": 1461, "end": 1465}, - {"type": "punctuation", "start": 1461, "end": 1462}, - {"type": "punctuation", "start": 1462, "end": 1463}, - {"type": "punctuation", "start": 1464, "end": 1465}, - {"type": "attr_name", "start": 1466, "end": 1478}, - {"type": "lang_ts", "start": 1478, "end": 1481}, - {"type": "punctuation", "start": 1478, "end": 1479}, - {"type": "number", "start": 1479, "end": 1480}, - {"type": "punctuation", "start": 1480, "end": 1481}, - {"type": "attr_name", "start": 1482, "end": 1490}, - {"type": "lang_ts", "start": 1490, "end": 1506}, - {"type": "punctuation", "start": 1490, "end": 1491}, - {"type": "punctuation", "start": 1491, "end": 1492}, - {"type": "punctuation", "start": 1492, "end": 1493}, - {"type": "operator", "start": 1494, "end": 1496}, - {"type": "punctuation", "start": 1497, "end": 1498}, - {"type": "operator", "start": 1500, "end": 1501}, - {"type": "operator", "start": 1502, "end": 1503}, - {"type": "punctuation", "start": 1504, "end": 1505}, - {"type": "punctuation", "start": 1505, "end": 1506}, - {"type": "punctuation", "start": 1506, "end": 1507}, - {"type": "lang_ts", "start": 1510, "end": 1530}, - {"type": "punctuation", "start": 1510, "end": 1511}, - {"type": "decorator", "start": 1511, "end": 1518}, - {"type": "at", "start": 1511, "end": 1512}, - {"type": "function", "start": 1512, "end": 1518}, - {"type": "function", "start": 1519, "end": 1527}, - {"type": "punctuation", "start": 1527, "end": 1528}, - {"type": "punctuation", "start": 1528, "end": 1529}, - {"type": "punctuation", "start": 1529, "end": 1530}, - {"type": "tag", "start": 1532, "end": 1540}, - {"type": "tag", "start": 1532, "end": 1539}, - {"type": "punctuation", "start": 1532, "end": 1534}, - {"type": "punctuation", "start": 1539, "end": 1540}, - {"type": "lang_ts", "start": 1541, "end": 1546}, - {"type": "punctuation", "start": 1541, "end": 1542}, - {"type": "operator", "start": 1542, "end": 1543}, - {"type": "keyword", "start": 1543, "end": 1545}, - {"type": "punctuation", "start": 1545, "end": 1546}, - {"type": "doctype", "start": 1548, "end": 1563}, - {"type": "tag", "start": 1565, "end": 1606}, - {"type": "tag", "start": 1565, "end": 1569}, - {"type": "punctuation", "start": 1565, "end": 1566}, - {"type": "attr_name", "start": 1570, "end": 1575}, - {"type": "attr_value", "start": 1575, "end": 1590}, - {"type": "punctuation", "start": 1575, "end": 1576}, - {"type": "punctuation", "start": 1576, "end": 1577}, - {"type": "punctuation", "start": 1589, "end": 1590}, - {"type": "attr_name", "start": 1591, "end": 1593}, - {"type": "attr_value", "start": 1593, "end": 1605}, - {"type": "punctuation", "start": 1593, "end": 1594}, - {"type": "punctuation", "start": 1594, "end": 1595}, - {"type": "punctuation", "start": 1604, "end": 1605}, - {"type": "punctuation", "start": 1605, "end": 1606}, - {"type": "tag", "start": 1608, "end": 1611}, - {"type": "tag", "start": 1608, "end": 1610}, - {"type": "punctuation", "start": 1608, "end": 1609}, - {"type": "punctuation", "start": 1610, "end": 1611}, - {"type": "tag", "start": 1623, "end": 1627}, - {"type": "tag", "start": 1623, "end": 1626}, - {"type": "punctuation", "start": 1623, "end": 1625}, - {"type": "punctuation", "start": 1626, "end": 1627}, - {"type": "tag", "start": 1628, "end": 1634}, - {"type": "tag", "start": 1628, "end": 1633}, - {"type": "punctuation", "start": 1628, "end": 1630}, - {"type": "punctuation", "start": 1633, "end": 1634}, - {"type": "tag", "start": 1636, "end": 1670}, - {"type": "tag", "start": 1636, "end": 1638}, - {"type": "punctuation", "start": 1636, "end": 1637}, - {"type": "attr_name", "start": 1639, "end": 1644}, - {"type": "attr_value", "start": 1644, "end": 1669}, - {"type": "punctuation", "start": 1644, "end": 1645}, - {"type": "punctuation", "start": 1645, "end": 1646}, - {"type": "punctuation", "start": 1668, "end": 1669}, - {"type": "punctuation", "start": 1669, "end": 1670}, - {"type": "tag", "start": 1677, "end": 1697}, - {"type": "tag", "start": 1677, "end": 1682}, - {"type": "punctuation", "start": 1677, "end": 1678}, - {"type": "attr_name", "start": 1683, "end": 1688}, - {"type": "attr_value", "start": 1688, "end": 1696}, - {"type": "punctuation", "start": 1688, "end": 1689}, - {"type": "punctuation", "start": 1689, "end": 1690}, - {"type": "punctuation", "start": 1695, "end": 1696}, - {"type": "punctuation", "start": 1696, "end": 1697}, - {"type": "tag", "start": 1701, "end": 1708}, - {"type": "tag", "start": 1701, "end": 1707}, - {"type": "punctuation", "start": 1701, "end": 1703}, - {"type": "punctuation", "start": 1707, "end": 1708}, - {"type": "tag", "start": 1709, "end": 1713}, - {"type": "tag", "start": 1709, "end": 1712}, - {"type": "punctuation", "start": 1709, "end": 1711}, - {"type": "punctuation", "start": 1712, "end": 1713}, - {"type": "tag", "start": 1715, "end": 1746}, - {"type": "tag", "start": 1715, "end": 1722}, - {"type": "punctuation", "start": 1715, "end": 1716}, - {"type": "attr_name", "start": 1723, "end": 1727}, - {"type": "attr_value", "start": 1727, "end": 1736}, - {"type": "punctuation", "start": 1727, "end": 1728}, - {"type": "punctuation", "start": 1728, "end": 1729}, - {"type": "punctuation", "start": 1735, "end": 1736}, - {"type": "attr_name", "start": 1737, "end": 1745}, - {"type": "punctuation", "start": 1745, "end": 1746}, - {"type": "tag", "start": 1756, "end": 1765}, - {"type": "tag", "start": 1756, "end": 1764}, - {"type": "punctuation", "start": 1756, "end": 1758}, - {"type": "punctuation", "start": 1764, "end": 1765}, - {"type": "comment", "start": 1767, "end": 1804}, - {"type": "lang_ts", "start": 1805, "end": 1808}, - {"type": "punctuation", "start": 1805, "end": 1806}, - {"type": "punctuation", "start": 1807, "end": 1808}, - {"type": "lang_ts", "start": 1809, "end": 1812}, - {"type": "punctuation", "start": 1809, "end": 1810}, - {"type": "punctuation", "start": 1811, "end": 1812}, - {"type": "lang_ts", "start": 1813, "end": 1820}, - {"type": "punctuation", "start": 1813, "end": 1814}, - {"type": "punctuation", "start": 1819, "end": 1820}, - {"type": "lang_ts", "start": 1821, "end": 1824}, - {"type": "punctuation", "start": 1821, "end": 1822}, - {"type": "constant", "start": 1822, "end": 1823}, - {"type": "punctuation", "start": 1823, "end": 1824}, - {"type": "tag", "start": 1826, "end": 1832}, - {"type": "tag", "start": 1826, "end": 1829}, - {"type": "punctuation", "start": 1826, "end": 1827}, - {"type": "punctuation", "start": 1830, "end": 1832}, - {"type": "tag", "start": 1834, "end": 1840}, - {"type": "tag", "start": 1834, "end": 1837}, - {"type": "punctuation", "start": 1834, "end": 1835}, - {"type": "punctuation", "start": 1838, "end": 1840}, - {"type": "tag", "start": 1842, "end": 1878}, - {"type": "tag", "start": 1842, "end": 1846}, - {"type": "punctuation", "start": 1842, "end": 1843}, - {"type": "attr_name", "start": 1847, "end": 1850}, - {"type": "attr_value", "start": 1850, "end": 1862}, - {"type": "punctuation", "start": 1850, "end": 1851}, - {"type": "punctuation", "start": 1851, "end": 1852}, - {"type": "punctuation", "start": 1861, "end": 1862}, - {"type": "attr_name", "start": 1863, "end": 1866}, - {"type": "attr_value", "start": 1866, "end": 1875}, - {"type": "punctuation", "start": 1866, "end": 1867}, - {"type": "punctuation", "start": 1867, "end": 1868}, - {"type": "punctuation", "start": 1874, "end": 1875}, - {"type": "punctuation", "start": 1876, "end": 1878}, - {"type": "tag", "start": 1880, "end": 1884}, - {"type": "tag", "start": 1880, "end": 1883}, - {"type": "punctuation", "start": 1880, "end": 1881}, - {"type": "punctuation", "start": 1883, "end": 1884}, - {"type": "tag", "start": 1886, "end": 1890}, - {"type": "tag", "start": 1886, "end": 1889}, - {"type": "punctuation", "start": 1886, "end": 1887}, - {"type": "punctuation", "start": 1889, "end": 1890}, - {"type": "tag", "start": 1901, "end": 1906}, - {"type": "tag", "start": 1901, "end": 1905}, - {"type": "punctuation", "start": 1901, "end": 1903}, - {"type": "punctuation", "start": 1905, "end": 1906}, - {"type": "tag", "start": 1908, "end": 1912}, - {"type": "tag", "start": 1908, "end": 1911}, - {"type": "punctuation", "start": 1908, "end": 1909}, - {"type": "punctuation", "start": 1911, "end": 1912}, - {"type": "tag", "start": 1923, "end": 1928}, - {"type": "tag", "start": 1923, "end": 1927}, - {"type": "punctuation", "start": 1923, "end": 1925}, - {"type": "punctuation", "start": 1927, "end": 1928}, - {"type": "tag", "start": 1929, "end": 1934}, - {"type": "tag", "start": 1929, "end": 1933}, - {"type": "punctuation", "start": 1929, "end": 1931}, - {"type": "punctuation", "start": 1933, "end": 1934}, - {"type": "tag", "start": 1936, "end": 1943}, - {"type": "tag", "start": 1936, "end": 1942}, - {"type": "punctuation", "start": 1936, "end": 1937}, - {"type": "punctuation", "start": 1942, "end": 1943}, - {"type": "style", "start": 1943, "end": 2324}, - {"type": "lang_css", "start": 1943, "end": 2324}, - {"type": "selector", "start": 1945, "end": 1956}, - {"type": "punctuation", "start": 1957, "end": 1958}, - {"type": "property", "start": 1961, "end": 1966}, - {"type": "punctuation", "start": 1966, "end": 1967}, - {"type": "punctuation", "start": 1971, "end": 1972}, - {"type": "punctuation", "start": 1974, "end": 1975}, - {"type": "selector", "start": 1978, "end": 1990}, - {"type": "punctuation", "start": 1991, "end": 1992}, - {"type": "property", "start": 1995, "end": 2004}, - {"type": "punctuation", "start": 2004, "end": 2005}, - {"type": "punctuation", "start": 2010, "end": 2011}, - {"type": "punctuation", "start": 2013, "end": 2014}, - {"type": "selector", "start": 2017, "end": 2018}, - {"type": "punctuation", "start": 2019, "end": 2020}, - {"type": "property", "start": 2023, "end": 2033}, - {"type": "punctuation", "start": 2033, "end": 2034}, - {"type": "function", "start": 2044, "end": 2048}, - {"type": "punctuation", "start": 2048, "end": 2049}, - {"type": "punctuation", "start": 2050, "end": 2051}, - {"type": "punctuation", "start": 2053, "end": 2054}, - {"type": "punctuation", "start": 2056, "end": 2057}, - {"type": "punctuation", "start": 2061, "end": 2062}, - {"type": "punctuation", "start": 2062, "end": 2063}, - {"type": "punctuation", "start": 2065, "end": 2066}, - {"type": "comment", "start": 2069, "end": 2082}, - {"type": "comment", "start": 2085, "end": 2117}, - {"type": "selector", "start": 2120, "end": 2130}, - {"type": "punctuation", "start": 2131, "end": 2132}, - {"type": "property", "start": 2135, "end": 2151}, - {"type": "punctuation", "start": 2151, "end": 2152}, - {"type": "punctuation", "start": 2157, "end": 2158}, - {"type": "punctuation", "start": 2160, "end": 2161}, - {"type": "selector", "start": 2164, "end": 2171}, - {"type": "punctuation", "start": 2172, "end": 2173}, - {"type": "property", "start": 2176, "end": 2182}, - {"type": "punctuation", "start": 2182, "end": 2183}, - {"type": "punctuation", "start": 2188, "end": 2189}, - {"type": "punctuation", "start": 2191, "end": 2192}, - {"type": "atrule", "start": 2195, "end": 2220}, - {"type": "rule", "start": 2195, "end": 2201}, - {"type": "punctuation", "start": 2202, "end": 2203}, - {"type": "property", "start": 2203, "end": 2212}, - {"type": "punctuation", "start": 2212, "end": 2213}, - {"type": "punctuation", "start": 2219, "end": 2220}, - {"type": "punctuation", "start": 2221, "end": 2222}, - {"type": "selector", "start": 2225, "end": 2238}, - {"type": "punctuation", "start": 2239, "end": 2240}, - {"type": "property", "start": 2244, "end": 2260}, - {"type": "punctuation", "start": 2260, "end": 2261}, - {"type": "punctuation", "start": 2271, "end": 2272}, - {"type": "punctuation", "start": 2275, "end": 2276}, - {"type": "punctuation", "start": 2278, "end": 2279}, - {"type": "selector", "start": 2282, "end": 2298}, - {"type": "punctuation", "start": 2299, "end": 2300}, - {"type": "property", "start": 2303, "end": 2310}, - {"type": "punctuation", "start": 2310, "end": 2311}, - {"type": "string", "start": 2312, "end": 2319}, - {"type": "punctuation", "start": 2319, "end": 2320}, - {"type": "punctuation", "start": 2322, "end": 2323}, - {"type": "tag", "start": 2324, "end": 2332}, - {"type": "tag", "start": 2324, "end": 2331}, - {"type": "punctuation", "start": 2324, "end": 2326}, - {"type": "punctuation", "start": 2331, "end": 2332} - ], - "html": "<script lang=\"ts\" module>\n\texport const HELLO = 'world';\n</script>\n\n<script lang=\"ts\">\n\t// @ts-expect-error\n\timport Thing from '$lib/Thing.svelte';\n\timport type {Snippet} from 'svelte';\n\n\tconst {\n\t\tthing,\n\t\tbound = $bindable(true),\n\t\tchildren,\n\t}: {\n\t\tthing: Record<string, any>;\n\t\tbound?: boolean;\n\t\tchildren: Snippet;\n\t} = $props();\n\n\tconst thing_keys = $derived(Object.keys(thing));\n\n\tconst a = 1;\n\n\tconst b = 'b';\n\n\tlet c: boolean = $state(true);\n\n\texport type Some_Type = 1 | 'b' | true;\n\n\tclass D {\n\t\td1: string = 'd';\n\t\td2: number;\n\t\td3 = $state(false);\n\n\t\tconstructor(d2: number) {\n\t\t\tthis.d2 = d2;\n\t\t}\n\n\t\tclass_method(): string {\n\t\t\treturn `Hello, ${this.d1}`;\n\t\t}\n\n\t\tinstance_method = () => {\n\t\t\t/* ... */\n\t\t\tthis.#private_method();\n\t\t\t// foo\n\t\t};\n\n\t\t#private_method() {\n\t\t\tthrow new Error(`${this.d1} etc`);\n\t\t}\n\n\t\tprotected protected_method() {\n\t\t\tconsole.log(new Date(123)); // eslint-disable-line no-console\n\t\t}\n\t}\n\n\t// comment\n\n\t/*\n\tother comment\n\n\tconst comment = false;\n\t*/\n\n\t/**\n\t * JSDoc comment\n\t */\n\n\texport interface Some_E {\n\t\tname: string;\n\t\tage: number;\n\t}\n\n\texport const some_e: Some_E = {name: 'A. H.', age: 100};\n\n\texport function add(x: number, y: number): number {\n\t\treturn x + y;\n\t}\n\n\texport const plus = (a: any, b: any): any => a + b;\n</script>\n\n<h1>hello {HELLO}!</h1>\n\n{#each thing_keys as key (key)}\n\t{@const value = thing[key]}\n\t{value}\n{/each}\n\n{#if c}\n\t<Thing string_prop=\"a\" number_prop={1} />\n{:else}\n\t<Thing string_prop=\"b\" number_prop={2} onthing={() => (c = !c)}>\n\t\t{@render children()}\n\t</Thing>\n{/if}\n\n<!DOCTYPE html>\n\n<div class=\"test special\" id=\"unique_id\">\n\t<p>hello world!</p>\n</div>\n\n<p class=\"some_class hypen-class\">\n\tsome <span class=\"a b c\">text</span>\n</p>\n\n<button type=\"button\" disabled> click me </button>\n\n<!-- comment <div>a<br /> b</div> -->\n{a}\n{b}\n{bound}\n{D}\n\n<br />\n\n<hr />\n\n<img src=\"image.jpg\" alt=\"access\" />\n\n<ul>\n\t<li>list item 1</li>\n\t<li>list item 2</li>\n</ul>\n\n<style>\n\t.some_class {\n\t\tcolor: red;\n\t}\n\n\t.hypen-class {\n\t\tfont-size: 16px;\n\t}\n\n\tp {\n\t\tbox-shadow: 0 0 10px rgba(0, 0, 0, 0.1);\n\t}\n\n\t/* comment */\n\n\t/*\n\tmulti\n\tline\n\n\t<comment>\n\n\t*/\n\n\t#unique_id {\n\t\tbackground-color: blue;\n\t}\n\n\tdiv > p {\n\t\tmargin: 10px;\n\t}\n\n\t@media (max-width: 600px) {\n\t\t:global(body) {\n\t\t\tbackground-color: lightblue;\n\t\t}\n\t}\n\n\t.special::before {\n\t\tcontent: '< & >';\n\t}\n</style>\n" -} diff --git a/fixtures/generated/svelte/svelte_complex.txt b/fixtures/generated/svelte/svelte_complex.txt deleted file mode 100644 index 354d0065..00000000 --- a/fixtures/generated/svelte/svelte_complex.txt +++ /dev/null @@ -1,614 +0,0 @@ -=== TOKENS === - 0-25 tag - 57-65 tag - 68-86 tag -1268-1276 tag -1279-1283 tag

-1279-1282 tag

-1289-1296 lang_ts {HELLO} -1289-1290 punctuation { -1290-1295 constant HELLO -1295-1296 punctuation } -1297-1302 tag

-1297-1301 tag -1304-1335 each {#each thing_keys as key (key)} -1304-1305 punctuation { -1305-1310 keyword #each -1311-1322 lang_ts thing_keys -1322-1324 keyword as -1325-1329 lang_ts key -1329-1334 lang_ts (key) -1329-1330 punctuation ( -1333-1334 punctuation ) -1334-1335 punctuation } -1337-1364 lang_ts {@const value = thing[key]} -1337-1338 punctuation { -1339-1344 keyword const -1351-1352 operator = -1358-1359 punctuation [ -1362-1363 punctuation ] -1363-1364 punctuation } -1366-1373 lang_ts {value} -1366-1367 punctuation { -1372-1373 punctuation } -1374-1381 each {/each} -1374-1375 punctuation { -1375-1380 keyword /each -1380-1381 punctuation } -1383-1390 lang_ts {#if c} -1383-1384 punctuation { -1385-1387 keyword if -1389-1390 punctuation } -1392-1433 tag -1392-1398 tag -1434-1441 lang_ts {:else} -1434-1435 punctuation { -1435-1436 operator : -1436-1440 keyword else -1440-1441 punctuation } -1443-1507 tag (c = !c)}> -1443-1449 tag (c = !c)} -1490-1491 punctuation { -1491-1492 punctuation ( -1492-1493 punctuation ) -1494-1496 operator => -1497-1498 punctuation ( -1500-1501 operator = -1502-1503 operator ! -1504-1505 punctuation ) -1505-1506 punctuation } -1506-1507 punctuation > -1510-1530 lang_ts {@render children()} -1510-1511 punctuation { -1511-1518 decorator @render -1511-1512 at @ -1512-1518 function render -1519-1527 function children -1527-1528 punctuation ( -1528-1529 punctuation ) -1529-1530 punctuation } -1532-1540 tag -1532-1539 tag -1541-1546 lang_ts {/if} -1541-1542 punctuation { -1542-1543 operator / -1543-1545 keyword if -1545-1546 punctuation } -1548-1563 doctype -1565-1606 tag
-1565-1569 tag
-1608-1611 tag

-1608-1610 tag

-1623-1627 tag

-1623-1626 tag

-1628-1634 tag
-1628-1633 tag
-1636-1670 tag

-1636-1638 tag

-1677-1697 tag -1677-1682 tag -1701-1708 tag -1701-1707 tag -1709-1713 tag

-1709-1712 tag

-1715-1746 tag -1756-1764 tag -1767-1804 comment -1805-1808 lang_ts {a} -1805-1806 punctuation { -1807-1808 punctuation } -1809-1812 lang_ts {b} -1809-1810 punctuation { -1811-1812 punctuation } -1813-1820 lang_ts {bound} -1813-1814 punctuation { -1819-1820 punctuation } -1821-1824 lang_ts {D} -1821-1822 punctuation { -1822-1823 constant D -1823-1824 punctuation } -1826-1832 tag
-1826-1829 tag
-1834-1840 tag
-1834-1837 tag
-1842-1878 tag access -1842-1846 tag -1880-1884 tag
    -1880-1883 tag
      -1886-1890 tag
    • -1886-1889 tag
    • -1901-1906 tag
    • -1901-1905 tag -1908-1912 tag
    • -1908-1911 tag
    • -1923-1928 tag
    • -1923-1927 tag -1929-1934 tag
    -1929-1933 tag
-1936-1943 tag -2324-2331 tag - -=== STATS === -Total tokens: 576 -Sample length: 2333 characters - -Token types: - punctuation: 270 - tag: 60 - operator: 49 - keyword: 40 - lang_ts: 19 - builtin: 17 - attr_name: 16 - function: 16 - attr_value: 11 - string: 10 - comment: 10 - property: 8 - selector: 7 - number: 6 - class_name: 5 - constant: 4 - boolean: 4 - template_punctuation: 4 - interpolation_punctuation: 4 - script: 2 - template_string: 2 - interpolation: 2 - each: 2 - function_variable: 1 - decorator: 1 - at: 1 - doctype: 1 - style: 1 - lang_css: 1 - atrule: 1 - rule: 1 diff --git a/fixtures/generated/ts/ts_complex.json b/fixtures/generated/ts/ts_complex.json deleted file mode 100644 index e3b567ab..00000000 --- a/fixtures/generated/ts/ts_complex.json +++ /dev/null @@ -1,248 +0,0 @@ -{ - "sample": { - "lang": "ts", - "variant": "complex", - "content": "const a = 1;\n\nconst b = 'b';\n\nconst c = true;\n\nexport type Some_Type = 1 | 'b' | true;\n\nclass D {\n\td1: string = 'd';\n\td2: number;\n\td3 = $state(false);\n\n\tconstructor(d2: number) {\n\t\tthis.d2 = d2;\n\t}\n\n\tclass_method(): string {\n\t\treturn `Hello, ${this.d1}`;\n\t}\n\n\tinstance_method = (): void => {\n\t\t/* ... */\n\t\tthis.#private_method();\n\t\t// foo\n\t};\n\n\t#private_method() {\n\t\tthrow new Error(`${this.d1} \n\t\t\tmultiline\n\t\t\tetc\n\t\t`);\n\t}\n\n\tprotected protected_method(): void {\n\t\tconsole.log(new Date()); // eslint-disable-line no-console\n\t}\n}\n\nexport {a, b, c, D};\n\n// comment\n\n/*\nother comment\n\nconst comment = false;\n*/\n\n/**\n * JSDoc comment\n */\n\nexport interface Some_E {\n\tname: string;\n\tage: number;\n}\n\nexport const some_e: Some_E = {name: 'A. H.', age: 100};\n\nexport function add(x: number, y: number): number {\n\treturn x + y;\n}\n\nexport const plus = (a: any, b: any): any => a + b;\n\n// boundary test cases\nexport const str_with_keywords = 'const class function string';\nexport const str_with_comment = '// this is not a comment';\nexport const template_with_expr = `Value: ${1 + 2}`;\n\n// regex that looks like comment\nexport const regex = /\\/\\/.*/g;\nexport const complex_regex = /^(?:\\/\\*.*?\\*\\/|\\/\\/.*|[^/])+$/;\n\n// string in comment should not be highlighted as string\n// const commented = \"this string is in a comment\";\n", - "filepath": "src/lib/samples/sample_complex.ts" - }, - "tokens": [ - {"type": "keyword", "start": 0, "end": 5}, - {"type": "operator", "start": 8, "end": 9}, - {"type": "number", "start": 10, "end": 11}, - {"type": "punctuation", "start": 11, "end": 12}, - {"type": "keyword", "start": 14, "end": 19}, - {"type": "operator", "start": 22, "end": 23}, - {"type": "string", "start": 24, "end": 27}, - {"type": "punctuation", "start": 27, "end": 28}, - {"type": "keyword", "start": 30, "end": 35}, - {"type": "operator", "start": 38, "end": 39}, - {"type": "boolean", "start": 40, "end": 44}, - {"type": "punctuation", "start": 44, "end": 45}, - {"type": "keyword", "start": 47, "end": 53}, - {"type": "keyword", "start": 54, "end": 58}, - {"type": "class_name", "start": 59, "end": 68}, - {"type": "operator", "start": 69, "end": 70}, - {"type": "number", "start": 71, "end": 72}, - {"type": "operator", "start": 73, "end": 74}, - {"type": "string", "start": 75, "end": 78}, - {"type": "operator", "start": 79, "end": 80}, - {"type": "boolean", "start": 81, "end": 85}, - {"type": "punctuation", "start": 85, "end": 86}, - {"type": "keyword", "start": 88, "end": 93}, - {"type": "class_name", "start": 94, "end": 95}, - {"type": "constant", "start": 94, "end": 95}, - {"type": "punctuation", "start": 96, "end": 97}, - {"type": "operator", "start": 101, "end": 102}, - {"type": "builtin", "start": 103, "end": 109}, - {"type": "operator", "start": 110, "end": 111}, - {"type": "string", "start": 112, "end": 115}, - {"type": "punctuation", "start": 115, "end": 116}, - {"type": "operator", "start": 120, "end": 121}, - {"type": "builtin", "start": 122, "end": 128}, - {"type": "punctuation", "start": 128, "end": 129}, - {"type": "operator", "start": 134, "end": 135}, - {"type": "function", "start": 136, "end": 142}, - {"type": "punctuation", "start": 142, "end": 143}, - {"type": "boolean", "start": 143, "end": 148}, - {"type": "punctuation", "start": 148, "end": 149}, - {"type": "punctuation", "start": 149, "end": 150}, - {"type": "function", "start": 153, "end": 164}, - {"type": "punctuation", "start": 164, "end": 165}, - {"type": "operator", "start": 167, "end": 168}, - {"type": "builtin", "start": 169, "end": 175}, - {"type": "punctuation", "start": 175, "end": 176}, - {"type": "punctuation", "start": 177, "end": 178}, - {"type": "keyword", "start": 181, "end": 185}, - {"type": "punctuation", "start": 185, "end": 186}, - {"type": "operator", "start": 189, "end": 190}, - {"type": "punctuation", "start": 193, "end": 194}, - {"type": "punctuation", "start": 196, "end": 197}, - {"type": "function", "start": 200, "end": 212}, - {"type": "punctuation", "start": 212, "end": 213}, - {"type": "punctuation", "start": 213, "end": 214}, - {"type": "operator", "start": 214, "end": 215}, - {"type": "builtin", "start": 216, "end": 222}, - {"type": "punctuation", "start": 223, "end": 224}, - {"type": "keyword", "start": 227, "end": 233}, - {"type": "template_string", "start": 234, "end": 253}, - {"type": "template_punctuation", "start": 234, "end": 235}, - {"type": "string", "start": 235, "end": 242}, - {"type": "interpolation", "start": 242, "end": 252}, - {"type": "interpolation_punctuation", "start": 242, "end": 244}, - {"type": "keyword", "start": 244, "end": 248}, - {"type": "punctuation", "start": 248, "end": 249}, - {"type": "interpolation_punctuation", "start": 251, "end": 252}, - {"type": "template_punctuation", "start": 252, "end": 253}, - {"type": "punctuation", "start": 253, "end": 254}, - {"type": "punctuation", "start": 256, "end": 257}, - {"type": "operator", "start": 276, "end": 277}, - {"type": "punctuation", "start": 278, "end": 279}, - {"type": "punctuation", "start": 279, "end": 280}, - {"type": "operator", "start": 280, "end": 281}, - {"type": "keyword", "start": 282, "end": 286}, - {"type": "operator", "start": 287, "end": 289}, - {"type": "punctuation", "start": 290, "end": 291}, - {"type": "comment", "start": 294, "end": 303}, - {"type": "keyword", "start": 306, "end": 310}, - {"type": "punctuation", "start": 310, "end": 311}, - {"type": "function", "start": 311, "end": 326}, - {"type": "punctuation", "start": 326, "end": 327}, - {"type": "punctuation", "start": 327, "end": 328}, - {"type": "punctuation", "start": 328, "end": 329}, - {"type": "comment", "start": 332, "end": 338}, - {"type": "punctuation", "start": 340, "end": 341}, - {"type": "punctuation", "start": 341, "end": 342}, - {"type": "function", "start": 345, "end": 360}, - {"type": "punctuation", "start": 360, "end": 361}, - {"type": "punctuation", "start": 361, "end": 362}, - {"type": "punctuation", "start": 363, "end": 364}, - {"type": "keyword", "start": 367, "end": 372}, - {"type": "keyword", "start": 373, "end": 376}, - {"type": "class_name", "start": 377, "end": 382}, - {"type": "punctuation", "start": 382, "end": 383}, - {"type": "template_string", "start": 383, "end": 419}, - {"type": "template_punctuation", "start": 383, "end": 384}, - {"type": "interpolation", "start": 384, "end": 394}, - {"type": "interpolation_punctuation", "start": 384, "end": 386}, - {"type": "keyword", "start": 386, "end": 390}, - {"type": "punctuation", "start": 390, "end": 391}, - {"type": "interpolation_punctuation", "start": 393, "end": 394}, - {"type": "string", "start": 394, "end": 418}, - {"type": "template_punctuation", "start": 418, "end": 419}, - {"type": "punctuation", "start": 419, "end": 420}, - {"type": "punctuation", "start": 420, "end": 421}, - {"type": "punctuation", "start": 423, "end": 424}, - {"type": "keyword", "start": 427, "end": 436}, - {"type": "function", "start": 437, "end": 453}, - {"type": "punctuation", "start": 453, "end": 454}, - {"type": "punctuation", "start": 454, "end": 455}, - {"type": "operator", "start": 455, "end": 456}, - {"type": "keyword", "start": 457, "end": 461}, - {"type": "punctuation", "start": 462, "end": 463}, - {"type": "builtin", "start": 466, "end": 473}, - {"type": "punctuation", "start": 473, "end": 474}, - {"type": "function", "start": 474, "end": 477}, - {"type": "punctuation", "start": 477, "end": 478}, - {"type": "keyword", "start": 478, "end": 481}, - {"type": "class_name", "start": 482, "end": 486}, - {"type": "punctuation", "start": 486, "end": 487}, - {"type": "punctuation", "start": 487, "end": 488}, - {"type": "punctuation", "start": 488, "end": 489}, - {"type": "punctuation", "start": 489, "end": 490}, - {"type": "comment", "start": 491, "end": 524}, - {"type": "punctuation", "start": 526, "end": 527}, - {"type": "punctuation", "start": 528, "end": 529}, - {"type": "keyword", "start": 531, "end": 537}, - {"type": "punctuation", "start": 538, "end": 539}, - {"type": "punctuation", "start": 540, "end": 541}, - {"type": "punctuation", "start": 543, "end": 544}, - {"type": "punctuation", "start": 546, "end": 547}, - {"type": "constant", "start": 548, "end": 549}, - {"type": "punctuation", "start": 549, "end": 550}, - {"type": "punctuation", "start": 550, "end": 551}, - {"type": "comment", "start": 553, "end": 563}, - {"type": "comment", "start": 565, "end": 608}, - {"type": "comment", "start": 610, "end": 634}, - {"type": "keyword", "start": 636, "end": 642}, - {"type": "keyword", "start": 643, "end": 652}, - {"type": "class_name", "start": 653, "end": 659}, - {"type": "punctuation", "start": 660, "end": 661}, - {"type": "operator", "start": 667, "end": 668}, - {"type": "builtin", "start": 669, "end": 675}, - {"type": "punctuation", "start": 675, "end": 676}, - {"type": "operator", "start": 681, "end": 682}, - {"type": "builtin", "start": 683, "end": 689}, - {"type": "punctuation", "start": 689, "end": 690}, - {"type": "punctuation", "start": 691, "end": 692}, - {"type": "keyword", "start": 694, "end": 700}, - {"type": "keyword", "start": 701, "end": 706}, - {"type": "operator", "start": 713, "end": 714}, - {"type": "operator", "start": 722, "end": 723}, - {"type": "punctuation", "start": 724, "end": 725}, - {"type": "operator", "start": 729, "end": 730}, - {"type": "string", "start": 731, "end": 738}, - {"type": "punctuation", "start": 738, "end": 739}, - {"type": "operator", "start": 743, "end": 744}, - {"type": "number", "start": 745, "end": 748}, - {"type": "punctuation", "start": 748, "end": 749}, - {"type": "punctuation", "start": 749, "end": 750}, - {"type": "keyword", "start": 752, "end": 758}, - {"type": "keyword", "start": 759, "end": 767}, - {"type": "function", "start": 768, "end": 771}, - {"type": "punctuation", "start": 771, "end": 772}, - {"type": "operator", "start": 773, "end": 774}, - {"type": "builtin", "start": 775, "end": 781}, - {"type": "punctuation", "start": 781, "end": 782}, - {"type": "operator", "start": 784, "end": 785}, - {"type": "builtin", "start": 786, "end": 792}, - {"type": "punctuation", "start": 792, "end": 793}, - {"type": "operator", "start": 793, "end": 794}, - {"type": "builtin", "start": 795, "end": 801}, - {"type": "punctuation", "start": 802, "end": 803}, - {"type": "keyword", "start": 805, "end": 811}, - {"type": "operator", "start": 814, "end": 815}, - {"type": "punctuation", "start": 817, "end": 818}, - {"type": "punctuation", "start": 819, "end": 820}, - {"type": "keyword", "start": 822, "end": 828}, - {"type": "keyword", "start": 829, "end": 834}, - {"type": "operator", "start": 840, "end": 841}, - {"type": "punctuation", "start": 842, "end": 843}, - {"type": "operator", "start": 844, "end": 845}, - {"type": "builtin", "start": 846, "end": 849}, - {"type": "punctuation", "start": 849, "end": 850}, - {"type": "operator", "start": 852, "end": 853}, - {"type": "builtin", "start": 854, "end": 857}, - {"type": "punctuation", "start": 857, "end": 858}, - {"type": "operator", "start": 858, "end": 859}, - {"type": "builtin", "start": 860, "end": 863}, - {"type": "operator", "start": 864, "end": 866}, - {"type": "operator", "start": 869, "end": 870}, - {"type": "punctuation", "start": 872, "end": 873}, - {"type": "comment", "start": 875, "end": 897}, - {"type": "keyword", "start": 898, "end": 904}, - {"type": "keyword", "start": 905, "end": 910}, - {"type": "operator", "start": 929, "end": 930}, - {"type": "string", "start": 931, "end": 960}, - {"type": "punctuation", "start": 960, "end": 961}, - {"type": "keyword", "start": 962, "end": 968}, - {"type": "keyword", "start": 969, "end": 974}, - {"type": "operator", "start": 992, "end": 993}, - {"type": "string", "start": 994, "end": 1020}, - {"type": "punctuation", "start": 1020, "end": 1021}, - {"type": "keyword", "start": 1022, "end": 1028}, - {"type": "keyword", "start": 1029, "end": 1034}, - {"type": "operator", "start": 1054, "end": 1055}, - {"type": "template_string", "start": 1056, "end": 1073}, - {"type": "template_punctuation", "start": 1056, "end": 1057}, - {"type": "string", "start": 1057, "end": 1064}, - {"type": "interpolation", "start": 1064, "end": 1072}, - {"type": "interpolation_punctuation", "start": 1064, "end": 1066}, - {"type": "number", "start": 1066, "end": 1067}, - {"type": "operator", "start": 1068, "end": 1069}, - {"type": "number", "start": 1070, "end": 1071}, - {"type": "interpolation_punctuation", "start": 1071, "end": 1072}, - {"type": "template_punctuation", "start": 1072, "end": 1073}, - {"type": "punctuation", "start": 1073, "end": 1074}, - {"type": "comment", "start": 1076, "end": 1108}, - {"type": "keyword", "start": 1109, "end": 1115}, - {"type": "keyword", "start": 1116, "end": 1121}, - {"type": "operator", "start": 1128, "end": 1129}, - {"type": "regex", "start": 1130, "end": 1139}, - {"type": "regex_delimiter", "start": 1130, "end": 1131}, - {"type": "regex_source", "start": 1131, "end": 1137}, - {"type": "regex_delimiter", "start": 1137, "end": 1138}, - {"type": "regex_flags", "start": 1138, "end": 1139}, - {"type": "punctuation", "start": 1139, "end": 1140}, - {"type": "keyword", "start": 1141, "end": 1147}, - {"type": "keyword", "start": 1148, "end": 1153}, - {"type": "operator", "start": 1168, "end": 1169}, - {"type": "regex", "start": 1170, "end": 1202}, - {"type": "regex_delimiter", "start": 1170, "end": 1171}, - {"type": "regex_source", "start": 1171, "end": 1201}, - {"type": "regex_delimiter", "start": 1201, "end": 1202}, - {"type": "punctuation", "start": 1202, "end": 1203}, - {"type": "comment", "start": 1205, "end": 1261}, - {"type": "comment", "start": 1262, "end": 1313} - ], - "html": "const a = 1;\n\nconst b = 'b';\n\nconst c = true;\n\nexport type Some_Type = 1 | 'b' | true;\n\nclass D {\n\td1: string = 'd';\n\td2: number;\n\td3 = $state(false);\n\n\tconstructor(d2: number) {\n\t\tthis.d2 = d2;\n\t}\n\n\tclass_method(): string {\n\t\treturn `Hello, ${this.d1}`;\n\t}\n\n\tinstance_method = (): void => {\n\t\t/* ... */\n\t\tthis.#private_method();\n\t\t// foo\n\t};\n\n\t#private_method() {\n\t\tthrow new Error(`${this.d1} \n\t\t\tmultiline\n\t\t\tetc\n\t\t`);\n\t}\n\n\tprotected protected_method(): void {\n\t\tconsole.log(new Date()); // eslint-disable-line no-console\n\t}\n}\n\nexport {a, b, c, D};\n\n// comment\n\n/*\nother comment\n\nconst comment = false;\n*/\n\n/**\n * JSDoc comment\n */\n\nexport interface Some_E {\n\tname: string;\n\tage: number;\n}\n\nexport const some_e: Some_E = {name: 'A. H.', age: 100};\n\nexport function add(x: number, y: number): number {\n\treturn x + y;\n}\n\nexport const plus = (a: any, b: any): any => a + b;\n\n// boundary test cases\nexport const str_with_keywords = 'const class function string';\nexport const str_with_comment = '// this is not a comment';\nexport const template_with_expr = `Value: ${1 + 2}`;\n\n// regex that looks like comment\nexport const regex = /\\/\\/.*/g;\nexport const complex_regex = /^(?:\\/\\*.*?\\*\\/|\\/\\/.*|[^/])+$/;\n\n// string in comment should not be highlighted as string\n// const commented = \"this string is in a comment\";\n" -} diff --git a/fixtures/generated/ts/ts_complex.txt b/fixtures/generated/ts/ts_complex.txt deleted file mode 100644 index 9188a0e4..00000000 --- a/fixtures/generated/ts/ts_complex.txt +++ /dev/null @@ -1,263 +0,0 @@ -=== TOKENS === - 0-5 keyword const - 8-9 operator = - 10-11 number 1 - 11-12 punctuation ; - 14-19 keyword const - 22-23 operator = - 24-27 string 'b' - 27-28 punctuation ; - 30-35 keyword const - 38-39 operator = - 40-44 boolean true - 44-45 punctuation ; - 47-53 keyword export - 54-58 keyword type - 59-68 class_name Some_Type - 69-70 operator = - 71-72 number 1 - 73-74 operator | - 75-78 string 'b' - 79-80 operator | - 81-85 boolean true - 85-86 punctuation ; - 88-93 keyword class - 94-95 class_name D - 94-95 constant D - 96-97 punctuation { - 101-102 operator : - 103-109 builtin string - 110-111 operator = - 112-115 string 'd' - 115-116 punctuation ; - 120-121 operator : - 122-128 builtin number - 128-129 punctuation ; - 134-135 operator = - 136-142 function $state - 142-143 punctuation ( - 143-148 boolean false - 148-149 punctuation ) - 149-150 punctuation ; - 153-164 function constructor - 164-165 punctuation ( - 167-168 operator : - 169-175 builtin number - 175-176 punctuation ) - 177-178 punctuation { - 181-185 keyword this - 185-186 punctuation . - 189-190 operator = - 193-194 punctuation ; - 196-197 punctuation } - 200-212 function class_method - 212-213 punctuation ( - 213-214 punctuation ) - 214-215 operator : - 216-222 builtin string - 223-224 punctuation { - 227-233 keyword return - 234-253 template_string `Hello, ${this.d1}` - 234-235 template_punctuation ` - 235-242 string Hello, - 242-252 interpolation ${this.d1} - 242-244 interpolation_punctuation ${ - 244-248 keyword this - 248-249 punctuation . - 251-252 interpolation_punctuation } - 252-253 template_punctuation ` - 253-254 punctuation ; - 256-257 punctuation } - 276-277 operator = - 278-279 punctuation ( - 279-280 punctuation ) - 280-281 operator : - 282-286 keyword void - 287-289 operator => - 290-291 punctuation { - 294-303 comment /* ... */ - 306-310 keyword this - 310-311 punctuation . - 311-326 function #private_method - 326-327 punctuation ( - 327-328 punctuation ) - 328-329 punctuation ; - 332-338 comment // foo - 340-341 punctuation } - 341-342 punctuation ; - 345-360 function #private_method - 360-361 punctuation ( - 361-362 punctuation ) - 363-364 punctuation { - 367-372 keyword throw - 373-376 keyword new - 377-382 class_name Error - 382-383 punctuation ( - 383-419 template_string `${this.d1} \n\t\t\tmultiline\n\t\t\tetc\n\t\t` - 383-384 template_punctuation ` - 384-394 interpolation ${this.d1} - 384-386 interpolation_punctuation ${ - 386-390 keyword this - 390-391 punctuation . - 393-394 interpolation_punctuation } - 394-418 string \n\t\t\tmultiline\n\t\t\tetc\n\t\t - 418-419 template_punctuation ` - 419-420 punctuation ) - 420-421 punctuation ; - 423-424 punctuation } - 427-436 keyword protected - 437-453 function protected_method - 453-454 punctuation ( - 454-455 punctuation ) - 455-456 operator : - 457-461 keyword void - 462-463 punctuation { - 466-473 builtin console - 473-474 punctuation . - 474-477 function log - 477-478 punctuation ( - 478-481 keyword new - 482-486 class_name Date - 486-487 punctuation ( - 487-488 punctuation ) - 488-489 punctuation ) - 489-490 punctuation ; - 491-524 comment // eslint-disable-line no-console - 526-527 punctuation } - 528-529 punctuation } - 531-537 keyword export - 538-539 punctuation { - 540-541 punctuation , - 543-544 punctuation , - 546-547 punctuation , - 548-549 constant D - 549-550 punctuation } - 550-551 punctuation ; - 553-563 comment // comment - 565-608 comment /*\nother comment\n\nconst comment = false;\n*/ - 610-634 comment /**\n * JSDoc comment\n */ - 636-642 keyword export - 643-652 keyword interface - 653-659 class_name Some_E - 660-661 punctuation { - 667-668 operator : - 669-675 builtin string - 675-676 punctuation ; - 681-682 operator : - 683-689 builtin number - 689-690 punctuation ; - 691-692 punctuation } - 694-700 keyword export - 701-706 keyword const - 713-714 operator : - 722-723 operator = - 724-725 punctuation { - 729-730 operator : - 731-738 string 'A. H.' - 738-739 punctuation , - 743-744 operator : - 745-748 number 100 - 748-749 punctuation } - 749-750 punctuation ; - 752-758 keyword export - 759-767 keyword function - 768-771 function add - 771-772 punctuation ( - 773-774 operator : - 775-781 builtin number - 781-782 punctuation , - 784-785 operator : - 786-792 builtin number - 792-793 punctuation ) - 793-794 operator : - 795-801 builtin number - 802-803 punctuation { - 805-811 keyword return - 814-815 operator + - 817-818 punctuation ; - 819-820 punctuation } - 822-828 keyword export - 829-834 keyword const - 840-841 operator = - 842-843 punctuation ( - 844-845 operator : - 846-849 builtin any - 849-850 punctuation , - 852-853 operator : - 854-857 builtin any - 857-858 punctuation ) - 858-859 operator : - 860-863 builtin any - 864-866 operator => - 869-870 operator + - 872-873 punctuation ; - 875-897 comment // boundary test cases - 898-904 keyword export - 905-910 keyword const - 929-930 operator = - 931-960 string 'const class function string' - 960-961 punctuation ; - 962-968 keyword export - 969-974 keyword const - 992-993 operator = - 994-1020 string '// this is not a comment' -1020-1021 punctuation ; -1022-1028 keyword export -1029-1034 keyword const -1054-1055 operator = -1056-1073 template_string `Value: ${1 + 2}` -1056-1057 template_punctuation ` -1057-1064 string Value: -1064-1072 interpolation ${1 + 2} -1064-1066 interpolation_punctuation ${ -1066-1067 number 1 -1068-1069 operator + -1070-1071 number 2 -1071-1072 interpolation_punctuation } -1072-1073 template_punctuation ` -1073-1074 punctuation ; -1076-1108 comment // regex that looks like comment -1109-1115 keyword export -1116-1121 keyword const -1128-1129 operator = -1130-1139 regex /\/\/.*/g -1130-1131 regex_delimiter / -1131-1137 regex_source \/\/.* -1137-1138 regex_delimiter / -1138-1139 regex_flags g -1139-1140 punctuation ; -1141-1147 keyword export -1148-1153 keyword const -1168-1169 operator = -1170-1202 regex /^(?:\/\*.*?\*\/|\/\/.*|[^/])+$/ -1170-1171 regex_delimiter / -1171-1201 regex_source ^(?:\/\*.*?\*\/|\/\/.*|[^/])+$ -1201-1202 regex_delimiter / -1202-1203 punctuation ; -1205-1261 comment // string in comment should not be highlighted as string -1262-1313 comment // const commented = "this string is in a comment"; - -=== STATS === -Total tokens: 237 -Sample length: 1314 characters - -Token types: - punctuation: 79 - operator: 39 - keyword: 37 - builtin: 13 - comment: 10 - string: 9 - function: 8 - template_punctuation: 6 - interpolation_punctuation: 6 - number: 5 - class_name: 5 - regex_delimiter: 4 - boolean: 3 - template_string: 3 - interpolation: 3 - constant: 2 - regex: 2 - regex_source: 2 - regex_flags: 1 diff --git a/package.json b/package.json index d3bc07a0..6f359d95 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,12 @@ "options": { "useTabs": false } + }, + { + "files": "src/fixtures/generated/**", + "options": { + "requirePragma": true + } } ] }, diff --git a/src/fixtures/check.test.ts b/src/fixtures/check.test.ts index 93c73bbf..8388ab86 100644 --- a/src/fixtures/check.test.ts +++ b/src/fixtures/check.test.ts @@ -1,11 +1,6 @@ import {test, assert, describe} from 'vitest'; import {readFileSync, existsSync} from 'node:fs'; -import { - discover_samples, - process_sample, - get_fixture_path, - type Generated_Output, -} from './helpers.js'; +import {discover_samples, process_sample, get_fixture_path} from './helpers.js'; import {sample_langs} from '$lib/code_sample.js'; /** @@ -28,13 +23,13 @@ describe('generated fixtures match runtime', () => { // Generate test for each sample for (const sample of samples) { describe(`${sample.lang}_${sample.variant}`, () => { - const fixture_path = get_fixture_path(sample.lang, sample.variant, 'json'); + const html_fixture_path = get_fixture_path(sample.lang, sample.variant, 'html'); test('fixture file exists', () => { // Basic sanity check - fixtures must be generated before tests can run assert.ok( - existsSync(fixture_path), - `Fixture file missing: ${fixture_path}. Run 'npm run task src/fixtures/update' to generate.`, + existsSync(html_fixture_path), + `Fixture file missing: ${html_fixture_path}. Run 'npm run task src/fixtures/update' to generate.`, ); }); @@ -49,17 +44,17 @@ describe('generated fixtures match runtime', () => { * - No overlapping spans * - Performance metrics (time, memory) */ - if (!existsSync(fixture_path)) { - console.warn(`Skipping test - fixture missing: ${fixture_path}`); // eslint-disable-line no-console + if (!existsSync(html_fixture_path)) { + console.warn(`Skipping test - fixture missing: ${html_fixture_path}`); // eslint-disable-line no-console return; } - const fixture: Generated_Output = JSON.parse(readFileSync(fixture_path, 'utf-8')); + const fixture_html = readFileSync(html_fixture_path, 'utf-8'); const runtime_output = process_sample(sample); assert.strictEqual( runtime_output.html, - fixture.html, + fixture_html, `HTML output mismatch for ${sample.lang}_${sample.variant}`, ); @@ -114,7 +109,7 @@ describe('generated fixtures match runtime', () => { } }); - test('token data matches fixture', () => { + test('token data is deterministic', () => { /** * Ensures token positions are consistent between runs * @@ -123,18 +118,14 @@ describe('generated fixtures match runtime', () => { * - Token types are correctly identified * - Tokenization is deterministic */ - if (!existsSync(fixture_path)) { - console.warn(`Skipping test - fixture missing: ${fixture_path}`); // eslint-disable-line no-console - return; - } - - const fixture: Generated_Output = JSON.parse(readFileSync(fixture_path, 'utf-8')); - const runtime_output = process_sample(sample); + // Generate tokens twice and ensure they match + const runtime_output1 = process_sample(sample); + const runtime_output2 = process_sample(sample); assert.deepEqual( - runtime_output.tokens, - fixture.tokens, - `Token data mismatch for ${sample.lang}_${sample.variant}`, + runtime_output1.tokens, + runtime_output2.tokens, + `Token data not deterministic for ${sample.lang}_${sample.variant}`, ); }); }); diff --git a/src/fixtures/generated/css/css_complex.html b/src/fixtures/generated/css/css_complex.html new file mode 100644 index 00000000..b6f51f53 --- /dev/null +++ b/src/fixtures/generated/css/css_complex.html @@ -0,0 +1,45 @@ +.some_class { + color: red; +} + +.hypen-class { + font-size: 16px; +} + +p { + box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); +} + +/* comment */ + +/* +multi +.line { + i: 100px + +</style> + +@media*/ + +#id { + background-color: blue; +} + +div > p { + margin: 10px; +} + +@media (max-width: 600px) { + body { + background-color: light-dark(lightblue, darkblue); + } +} + +.content::before { + /* prettier-ignore */ + content: "</style> /* not a comment */"; +} + +.attr[title='Click: here'] { + background-image: url('data:image/svg+xml...'); +} diff --git a/fixtures/generated/css/css_complex.txt b/src/fixtures/generated/css/css_complex.txt similarity index 60% rename from fixtures/generated/css/css_complex.txt rename to src/fixtures/generated/css/css_complex.txt index 9ae37ae4..37a7a612 100644 --- a/fixtures/generated/css/css_complex.txt +++ b/src/fixtures/generated/css/css_complex.txt @@ -1,3 +1,18 @@ +=== STATS === +Sample length: 502 characters +Total tokens: 75 + +Token types (9 unique): + punctuation: 47 + property: 9 + selector: 8 + function: 3 + comment: 3 + string: 2 + atrule: 1 + rule: 1 + url: 1 + === TOKENS === 0-11 selector .some_class 12-13 punctuation { @@ -48,46 +63,29 @@ 274-275 punctuation { 278-294 property background-color 294-295 punctuation : - 305-306 punctuation ; - 308-309 punctuation } - 310-311 punctuation } - 313-363 comment /* patterns in strings are not falsely detected */ - 364-380 selector .content::before - 381-382 punctuation { - 384-391 property content - 391-392 punctuation : - 393-423 string ' /* not a comment */' - 423-424 punctuation ; - 425-426 punctuation } - 428-454 selector .attr[title='Click: here'] - 455-456 punctuation { - 458-464 property cursor - 464-465 punctuation : - 473-474 punctuation ; - 475-476 punctuation } - 491-503 comment /* comment*/ - 504-505 punctuation { - 507-523 property background-image - 523-524 punctuation : - 525-553 url url('data:image/svg+xml...') - 525-528 function url - 528-529 punctuation ( - 529-552 string 'data:image/svg+xml...' - 552-553 punctuation ) - 553-554 punctuation ; - 555-556 punctuation } - -=== STATS === -Total tokens: 77 -Sample length: 557 characters - -Token types: - punctuation: 48 - property: 10 - selector: 8 - comment: 4 - function: 2 - string: 2 - atrule: 1 - rule: 1 - url: 1 + 296-306 function light-dark + 306-307 punctuation ( + 316-317 punctuation , + 326-327 punctuation ) + 327-328 punctuation ; + 330-331 punctuation } + 332-333 punctuation } + 335-351 selector .content::before + 352-353 punctuation { + 355-376 comment /* prettier-ignore */ + 378-385 property content + 385-386 punctuation : + 387-417 string " /* not a comment */" + 417-418 punctuation ; + 419-420 punctuation } + 422-448 selector .attr[title='Click: here'] + 449-450 punctuation { + 452-468 property background-image + 468-469 punctuation : + 470-498 url url('data:image/svg+xml...') + 470-473 function url + 473-474 punctuation ( + 474-497 string 'data:image/svg+xml...' + 497-498 punctuation ) + 498-499 punctuation ; + 500-501 punctuation } diff --git a/src/fixtures/generated/html/html_complex.html b/src/fixtures/generated/html/html_complex.html new file mode 100644 index 00000000..90084059 --- /dev/null +++ b/src/fixtures/generated/html/html_complex.html @@ -0,0 +1,42 @@ +<!doctype html> + +<div class="test"> + <p>hello world!</p> +</div> + +<p class="some_class hypen-class">some <span class="a b c">text</span></p> + +<button type="button" disabled>click me</button> + +<!-- comment <div>a<br /> b</div> <script> --> + +<br /> + +<hr onclick="console.log('hi')" /> + +<img src="image.jpg" alt="access" /> + +<ul> + <li>list item 1</li> + <li>list item 2</li> +</ul> + +<form action="/submit" method="post"> + <input type="text" name="username" placeholder="Enter name" /> + <select name="option" data-role="dropdown"> + <option value="1">First</option> + <option value="2">Second</option> + </select> +</form> + +<script> + const ok = '<style>'; +</script> + +<style type="text/css"> + .special::before { + content: '< & >'; + } +</style> + +<![CDATA[ if (a < 0) alert("b"); <not-a-tag> ]]> diff --git a/src/fixtures/generated/html/html_complex.txt b/src/fixtures/generated/html/html_complex.txt new file mode 100644 index 00000000..29f367a3 --- /dev/null +++ b/src/fixtures/generated/html/html_complex.txt @@ -0,0 +1,265 @@ +=== STATS === +Sample length: 784 characters +Total tokens: 239 + +Token types (19 unique): + punctuation: 123 + tag: 64 + attr_name: 18 + attr_value: 17 + string: 3 + doctype: 1 + comment: 1 + special_attr: 1 + value: 1 + function: 1 + script: 1 + lang_js: 1 + keyword: 1 + operator: 1 + style: 1 + lang_css: 1 + selector: 1 + property: 1 + cdata: 1 + +=== TOKENS === + 0-15 doctype + 17-35 tag
+ 17-21 tag
+ 37-40 tag

+ 37-39 tag

+ 52-56 tag

+ 52-55 tag

+ 57-63 tag
+ 57-62 tag
+ 65-99 tag

+ 65-67 tag

+ 104-124 tag + 104-109 tag + 128-135 tag + 128-134 tag + 135-139 tag

+ 135-138 tag

+ 141-172 tag + 180-188 tag + 191-237 comment + 239-245 tag
+ 239-242 tag
+ 247-281 tag
+ 247-250 tag
+ 283-319 tag access + 283-287 tag + 321-325 tag
    + 321-324 tag
      + 327-331 tag
    • + 327-330 tag
    • + 342-347 tag
    • + 342-346 tag + 349-353 tag
    • + 349-352 tag
    • + 364-369 tag
    • + 364-368 tag + 370-375 tag
    + 370-374 tag
+ 377-414 tag
+ 377-382 tag + 416-478 tag + 416-422 tag + 480-523 tag + 526-544 tag + 549-557 tag + 561-579 tag + 585-593 tag + 596-605 tag + 596-604 tag + 606-613 tag
+ 606-612 tag + 615-623 tag + 647-655 tag + 658-681 tag + 725-732 tag + 735-783 cdata ]]> diff --git a/src/fixtures/generated/json/json_complex.html b/src/fixtures/generated/json/json_complex.html new file mode 100644 index 00000000..7e3dd385 --- /dev/null +++ b/src/fixtures/generated/json/json_complex.html @@ -0,0 +1,14 @@ +{ + "string": "a string", + "number": 12345, + "boolean": true, + "null": null, + "empty": "", + "escaped": "quote: \"test\" and backslash: \\", + "object": { + "array": [1, "b", false], + "strings": ["1", "2", "3"], + "mixed": ["start", 123, true, "middle", null, "end"], + "nested": [["a", "str", ""], {"key": "nested value"}] + } +} diff --git a/fixtures/generated/json/json_complex.txt b/src/fixtures/generated/json/json_complex.txt similarity index 99% rename from fixtures/generated/json/json_complex.txt rename to src/fixtures/generated/json/json_complex.txt index 082fc2fa..e32ee2b3 100644 --- a/fixtures/generated/json/json_complex.txt +++ b/src/fixtures/generated/json/json_complex.txt @@ -1,3 +1,16 @@ +=== STATS === +Sample length: 327 characters +Total tokens: 83 + +Token types (7 unique): + punctuation: 37 + string: 14 + property: 12 + operator: 12 + number: 3 + boolean: 3 + null: 2 + === TOKENS === 0-1 punctuation { 3-11 property "string" @@ -82,16 +95,3 @@ 320-321 punctuation ] 323-324 punctuation } 325-326 punctuation } - -=== STATS === -Total tokens: 83 -Sample length: 327 characters - -Token types: - punctuation: 37 - string: 14 - property: 12 - operator: 12 - number: 3 - boolean: 3 - null: 2 diff --git a/src/fixtures/generated/svelte/svelte_complex.html b/src/fixtures/generated/svelte/svelte_complex.html new file mode 100644 index 00000000..7814766d --- /dev/null +++ b/src/fixtures/generated/svelte/svelte_complex.html @@ -0,0 +1,141 @@ +<script lang="ts" module> + export const HELLO = 'world'; +</script> + +<script lang="ts"> + // @ts-expect-error + import Thing from '$lib/Thing.svelte'; + import type {Snippet} from 'svelte'; + + const { + thing, + bound = $bindable(true), + children, + onclick, + }: { + thing: Record<string, any>; + bound?: boolean; + children: Snippet; + onclick?: () => void; + } = $props(); + + const thing_keys = $derived(Object.entries(thing)); + + const a = 1 as number; + + const b = 'b'; + + let c: boolean = $state(true); + + const f = (p: any) => p; + + const attachment = (_p1: string, _p2: number) => { + return (el: HTMLElement) => { + element_ref !== el; + }; + }; + + let value = $state(thing['']); + let element_ref: HTMLElement; +</script> + +<h1 bind:this={element_ref}>hello {HELLO}!</h1> + +{#each thing_keys as [k, { t, u }] (f(k))} + {@const v = Math.round(t[k + f(u)])} + {f(v)} +{/each} + +{#if f(c)} + <Thing string_prop="a {f('s')} b" number_prop={f(1)} /> +{:else if f(a > 0)} + bigger +{:else} + <Thing string_prop="b" onthing={() => (c = f(!c))}> + {@render children()} + </Thing> +{/if} + +<!-- eslint-disable-next-line svelte/no-at-html-tags --> +{@html '<strong>raw html</strong>'} + +<input bind:value type="text" class:active={c} /> + +<span {@attach attachment('param', f(42))}>...</span> + +{@render my_snippet('p')} + +{#snippet my_snippet(p: string)} + <button type="button" {onclick}>{p}</button> +{/snippet} + +<p class="some_class hypen-class" id="unique_id"> + some <span class="a b c">text</span> +</p> + +<button type="button" disabled> click me </button> + +<!-- comment <div>a<br /> {b}</div> --> +{b.repeat(2)} +{bound} + +<br /> + +<hr /> + +<img src="image.jpg" alt="access" /> + +<ul> + <li>list item 1</li> + <li>list item 2</li> +</ul> + +<!-- embedded tags for boundary testing --> +<div> + <script> + const inline_js = 'no lang attr'; + </script> + <style> + .inline { + color: blue; + } + </style> +</div> + +<style> + .some_class { + color: red; + } + + .hypen-class { + font-size: 16px; + } + + p { + box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); + } + + /* comment */ + + /* + multi + line + + <comment> + + */ + + #unique_id { + background-color: blue; + } + + @media (max-width: 600px) { + :global(body) { + background-color: light-dark(lightblue, darkblue); + } + } + + :global(.escapehatch)::before { + content: '< & >'; + } +</style> diff --git a/src/fixtures/generated/svelte/svelte_complex.txt b/src/fixtures/generated/svelte/svelte_complex.txt new file mode 100644 index 00000000..2e0f7160 --- /dev/null +++ b/src/fixtures/generated/svelte/svelte_complex.txt @@ -0,0 +1,675 @@ +=== STATS === +Sample length: 2266 characters +Total tokens: 635 + +Token types (33 unique): + punctuation: 301 + tag: 74 + operator: 37 + svelte_expression: 24 + lang_ts: 23 + function: 23 + attr_name: 20 + special_keyword: 15 + keyword: 14 + attr_value: 11 + string: 11 + builtin: 9 + capitalized_identifier: 8 + property: 8 + block: 7 + selector: 7 + comment: 6 + number: 5 + at_directive: 5 + at_keyword: 5 + script: 3 + namespace: 3 + constant: 2 + boolean: 2 + function_variable: 2 + style: 2 + lang_css: 2 + type_annotation: 1 + :: 1 + type: 1 + each: 1 + atrule: 1 + rule: 1 + +=== TOKENS === + 0-25 tag + 57-65 tag + 68-86 tag + 707-715 tag + 718-746 tag

+ 718-721 tag

+ 752-759 svelte_expression {HELLO} + 752-753 punctuation { + 753-758 lang_ts HELLO + 753-758 constant HELLO + 758-759 punctuation } + 760-765 tag

+ 760-764 tag + 767-809 svelte_expression {#each thing_keys as [k, { t, u }] (f(k))} + 767-809 each {#each thing_keys as [k, { t, u }] (f(k))} + 767-768 punctuation { + 768-773 special_keyword #each + 785-787 keyword as + 788-809 lang_ts [k, { t, u }] (f(k))} + 788-789 punctuation [ + 790-791 punctuation , + 792-793 punctuation { + 795-796 punctuation , + 799-800 punctuation } + 800-801 punctuation ] + 802-803 punctuation ( + 803-804 function f + 804-805 punctuation ( + 806-807 punctuation ) + 807-808 punctuation ) + 808-809 punctuation } + 811-847 svelte_expression {@const v = Math.round(t[k + f(u)])} + 811-847 at_directive {@const v = Math.round(t[k + f(u)])} + 811-812 punctuation { + 812-818 at_keyword @const + 819-847 lang_ts v = Math.round(t[k + f(u)])} + 821-822 operator = + 823-827 capitalized_identifier Math + 827-828 punctuation . + 828-833 function round + 833-834 punctuation ( + 835-836 punctuation [ + 838-839 operator + + 840-841 function f + 841-842 punctuation ( + 843-844 punctuation ) + 844-845 punctuation ] + 845-846 punctuation ) + 846-847 punctuation } + 849-855 svelte_expression {f(v)} + 849-850 punctuation { + 850-854 lang_ts f(v) + 850-851 function f + 851-852 punctuation ( + 853-854 punctuation ) + 854-855 punctuation } + 856-863 svelte_expression {/each} + 856-863 block {/each} + 856-857 punctuation { + 857-862 special_keyword /each + 862-863 punctuation } + 865-875 svelte_expression {#if f(c)} + 865-875 block {#if f(c)} + 865-866 punctuation { + 866-869 special_keyword #if + 869-874 lang_ts f(c) + 870-871 function f + 871-872 punctuation ( + 873-874 punctuation ) + 874-875 punctuation } + 877-932 tag + 877-883 tag + 933-952 svelte_expression {:else if f(a > 0)} + 933-952 block {:else if f(a > 0)} + 933-934 punctuation { + 934-942 special_keyword :else if + 942-951 lang_ts f(a > 0) + 943-944 function f + 944-945 punctuation ( + 947-948 operator > + 949-950 number 0 + 950-951 punctuation ) + 951-952 punctuation } + 961-968 svelte_expression {:else} + 961-968 block {:else} + 961-962 punctuation { + 962-967 special_keyword :else + 967-968 punctuation } + 970-1021 tag (c = f(!c))}> + 970-976 tag (c = f(!c))} +1001-1002 punctuation { +1002-1019 lang_ts () => (c = f(!c)) +1002-1003 punctuation ( +1003-1004 punctuation ) +1005-1007 operator => +1008-1009 punctuation ( +1011-1012 operator = +1013-1014 function f +1014-1015 punctuation ( +1015-1016 operator ! +1017-1018 punctuation ) +1018-1019 punctuation ) +1019-1020 punctuation } +1020-1021 punctuation > +1024-1044 svelte_expression {@render children()} +1024-1044 at_directive {@render children()} +1024-1025 punctuation { +1025-1032 at_keyword @render +1033-1044 lang_ts children()} +1033-1041 function children +1041-1042 punctuation ( +1042-1043 punctuation ) +1043-1044 punctuation } +1046-1054 tag +1046-1053 tag +1055-1060 svelte_expression {/if} +1055-1060 block {/if} +1055-1056 punctuation { +1056-1059 special_keyword /if +1059-1060 punctuation } +1062-1118 comment +1119-1154 svelte_expression {@html 'raw html'} +1119-1154 at_directive {@html 'raw html'} +1119-1120 punctuation { +1120-1125 at_keyword @html +1126-1154 lang_ts 'raw html'} +1126-1153 string 'raw html' +1153-1154 punctuation } +1156-1205 tag +1156-1162 tag +1207-1250 tag +1207-1212 tag +1253-1260 tag +1253-1259 tag +1262-1287 svelte_expression {@render my_snippet('p')} +1262-1287 at_directive {@render my_snippet('p')} +1262-1263 punctuation { +1263-1270 at_keyword @render +1271-1287 lang_ts my_snippet('p')} +1271-1281 function my_snippet +1281-1282 punctuation ( +1282-1285 string 'p' +1285-1286 punctuation ) +1286-1287 punctuation } +1289-1321 svelte_expression {#snippet my_snippet(p: string)} +1289-1321 block {#snippet my_snippet(p: string)} +1289-1290 punctuation { +1290-1298 special_keyword #snippet +1298-1320 lang_ts my_snippet(p: string) +1299-1309 function my_snippet +1309-1310 punctuation ( +1311-1312 operator : +1313-1319 builtin string +1319-1320 punctuation ) +1320-1321 punctuation } +1323-1355 tag +1358-1366 tag +1368-1378 svelte_expression {/snippet} +1368-1378 block {/snippet} +1368-1369 punctuation { +1369-1377 special_keyword /snippet +1377-1378 punctuation } +1380-1429 tag

+1380-1382 tag

+1436-1456 tag +1436-1441 tag +1460-1467 tag +1460-1466 tag +1468-1472 tag

+1468-1471 tag

+1474-1505 tag +1515-1523 tag +1526-1565 comment +1566-1579 svelte_expression {b.repeat(2)} +1566-1567 punctuation { +1567-1578 lang_ts b.repeat(2) +1568-1569 punctuation . +1569-1575 function repeat +1575-1576 punctuation ( +1576-1577 number 2 +1577-1578 punctuation ) +1578-1579 punctuation } +1580-1587 svelte_expression {bound} +1580-1581 punctuation { +1581-1586 lang_ts bound +1586-1587 punctuation } +1589-1595 tag
+1589-1592 tag
+1597-1603 tag
+1597-1600 tag
+1605-1641 tag access +1605-1609 tag +1643-1647 tag
    +1643-1646 tag
      +1649-1653 tag
    • +1649-1652 tag
    • +1664-1669 tag
    • +1664-1668 tag +1671-1675 tag
    • +1671-1674 tag
    • +1686-1691 tag
    • +1686-1690 tag +1692-1697 tag
    +1692-1696 tag
+1699-1742 comment +1743-1748 tag
+1743-1747 tag
+1750-1758 tag +1796-1804 tag +1807-1814 tag +1848-1855 tag +1857-1863 tag
+1857-1862 tag
+1865-1872 tag +2257-2264 tag diff --git a/src/fixtures/generated/ts/ts_complex.html b/src/fixtures/generated/ts/ts_complex.html new file mode 100644 index 00000000..4eb31e46 --- /dev/null +++ b/src/fixtures/generated/ts/ts_complex.html @@ -0,0 +1,152 @@ +const a = 1; + +const b: string = 'b'; + +const c = true; + +export type Some_Type = 1 | 'b' | true; + +declare const some_decorator: any; + +abstract class Base { + abstract abstract_method(): void; +} + +/* eslint-disable no-console */ + +@some_decorator +class D extends Base { + readonly d1: string = 'd'; + d2: number; + d3 = $state(null); + + @some_decorator + decorated = true; + + constructor(d2: number) { + super(); + this.d2 = d2; + } + + abstract_method(): void { + // implementation + } + + @some_decorator('example', {option: true}) + class_method(): string { + return `Hello, ${this.d1}`; + } + + instance_method = (): void => { + /* ... */ + let i = 0; + do { + i++; + } while (i < 3); + + for (const c2 of this.d1) { + if (c2 === 'd') continue; + if (!c2) break; + this.#private_method(a, c2); + } + + switch (this.d1) { + case 'a': + console.log('case a'); + break; + case 'b': + case 'c': + console.log('case b or c'); + break; + default: + console.log('default'); + } + + const obj: {has_d1?: boolean; is_d: boolean} = { + has_d1: 'd1' in this, + is_d: this instanceof D, + }; + delete obj.has_d1; + // foo + }; + + #private_method(a2: number, c2: any): void { + throw new Error(`${this.d1} + multiline + etc ${a2 + c2} + `); + } + + *generator(): Generator<number | Array<number>> { + yield 1; + yield* [2, 3]; + } + + async *async_generator(): AsyncGenerator<number> { + yield await Promise.resolve(4); + } + + protected async protected_method(): Promise<void> { + try { + await new Promise((resolve) => setTimeout(resolve, 100)); + if (Math.random() > 0.5) { + console.log(new Date()); + } else if (Math.random() > 0.2) { + console.log('else if branch'); + } else { + console.log('else branch'); + } + } catch (error: unknown) { + console.error(error); + } finally { + console.log('finally block'); + } + } +} + +// comment + +/* +other comment + +const comment = false; +*/ + +/** + * JSDoc comment + */ + +import {sample_langs, type Sample_Lang} from '../code_sample.js'; +import * as A from '../code_sample.js'; + +export {a, A, b, c, D}; + +sample_langs as unknown as Sample_Lang satisfies Sample_Lang; + +export interface Some_E<T = null> { + name: string; + age: number; + t?: T; +} + +const e: {name: string; age: number} = {name: 'A. H.', age: 100}; +const v = [['', e]] as const; +export const some_e: Map<string, Some_E> = new Map(v); + +export function add(x: number, y: number): number { + return x + y; +} + +export const plus = (a: any, b: any): any => a + b; + +// boundary test cases +export const str_with_keywords = 'const class function string'; +export const str_with_comment = '// this is not a comment'; +export const template_with_expr = `Value: ${1 + 2}`; + +// regex that looks like comment +export const regex = /\/\/.*/g; +export const complex_regex = /^(?:\/\*.*?\*\/|\/\/.*|[^/])+$/; + +// string in comment should not be highlighted as string +// const commented = "this string is in a comment"; diff --git a/src/fixtures/generated/ts/ts_complex.txt b/src/fixtures/generated/ts/ts_complex.txt new file mode 100644 index 00000000..c7d2722e --- /dev/null +++ b/src/fixtures/generated/ts/ts_complex.txt @@ -0,0 +1,666 @@ +=== STATS === +Sample length: 2885 characters +Total tokens: 632 + +Token types (27 unique): + punctuation: 233 + operator: 90 + keyword: 56 + special_keyword: 48 + builtin: 34 + function: 26 + string: 25 + capitalized_identifier: 20 + number: 14 + class_name: 11 + comment: 11 + interpolation_punctuation: 8 + constant: 7 + template_punctuation: 6 + type_annotation: 5 + :: 5 + type: 5 + boolean: 4 + interpolation: 4 + regex_delimiter: 4 + decorator: 3 + at: 3 + template_string: 3 + function_variable: 2 + regex: 2 + regex_source: 2 + regex_flags: 1 + +=== TOKENS === + 0-5 keyword const + 8-9 operator = + 10-11 number 1 + 11-12 punctuation ; + 14-19 keyword const + 21-30 type_annotation : string + 21-22 : : + 22-30 type string + 23-29 builtin string + 30-31 operator = + 32-35 string 'b' + 35-36 punctuation ; + 38-43 keyword const + 46-47 operator = + 48-52 boolean true + 52-53 punctuation ; + 55-61 special_keyword export + 62-66 keyword type + 67-76 class_name Some_Type + 67-76 capitalized_identifier Some_Type + 77-78 operator = + 79-80 number 1 + 81-82 operator | + 83-86 string 'b' + 87-88 operator | + 89-93 boolean true + 93-94 punctuation ; + 96-103 keyword declare + 104-109 keyword const + 124-125 operator : + 126-129 builtin any + 129-130 punctuation ; + 132-140 keyword abstract + 141-146 keyword class + 147-151 class_name Base + 147-151 capitalized_identifier Base + 152-153 punctuation { + 155-163 keyword abstract + 164-179 function abstract_method + 179-180 punctuation ( + 180-181 punctuation ) + 181-182 operator : + 183-187 keyword void + 187-188 punctuation ; + 189-190 punctuation } + 192-223 comment /* eslint-disable no-console */ + 225-240 decorator @some_decorator + 225-226 at @ + 226-240 function some_decorator + 241-246 keyword class + 247-248 class_name D + 247-248 constant D + 249-256 keyword extends + 257-261 class_name Base + 257-261 capitalized_identifier Base + 262-263 punctuation { + 265-273 keyword readonly + 276-285 type_annotation : string + 276-277 : : + 277-285 type string + 278-284 builtin string + 285-286 operator = + 287-290 string 'd' + 290-291 punctuation ; + 295-296 operator : + 297-303 builtin number + 303-304 punctuation ; + 309-310 operator = + 311-317 function $state + 317-318 punctuation ( + 318-322 keyword null + 322-323 punctuation ) + 323-324 punctuation ; + 327-342 decorator @some_decorator + 327-328 at @ + 328-342 function some_decorator + 354-355 operator = + 356-360 boolean true + 360-361 punctuation ; + 364-375 function constructor + 375-376 punctuation ( + 378-379 operator : + 380-386 builtin number + 386-387 punctuation ) + 388-389 punctuation { + 392-397 keyword super + 397-398 punctuation ( + 398-399 punctuation ) + 399-400 punctuation ; + 403-407 keyword this + 407-408 punctuation . + 411-412 operator = + 415-416 punctuation ; + 418-419 punctuation } + 422-437 function abstract_method + 437-438 punctuation ( + 438-439 punctuation ) + 439-440 operator : + 441-445 keyword void + 446-447 punctuation { + 450-467 comment // implementation + 469-470 punctuation } + 473-488 decorator @some_decorator + 473-474 at @ + 474-488 function some_decorator + 488-489 punctuation ( + 489-498 string 'example' + 498-499 punctuation , + 500-501 punctuation { + 507-508 operator : + 509-513 boolean true + 513-514 punctuation } + 514-515 punctuation ) + 517-529 function class_method + 529-530 punctuation ( + 530-531 punctuation ) + 531-532 operator : + 533-539 builtin string + 540-541 punctuation { + 544-550 special_keyword return + 551-570 template_string `Hello, ${this.d1}` + 551-552 template_punctuation ` + 552-559 string Hello, + 559-569 interpolation ${this.d1} + 559-561 interpolation_punctuation ${ + 561-565 keyword this + 565-566 punctuation . + 568-569 interpolation_punctuation } + 569-570 template_punctuation ` + 570-571 punctuation ; + 573-574 punctuation } + 577-592 function_variable instance_method + 593-594 operator = + 595-596 punctuation ( + 596-597 punctuation ) + 597-604 type_annotation : void + 597-598 : : + 598-604 type void + 599-603 keyword void + 604-606 operator => + 607-608 punctuation { + 611-620 comment /* ... */ + 623-626 keyword let + 629-630 operator = + 631-632 number 0 + 632-633 punctuation ; + 636-638 special_keyword do + 639-640 punctuation { + 645-647 operator ++ + 647-648 punctuation ; + 651-652 punctuation } + 653-658 special_keyword while + 659-660 punctuation ( + 662-663 operator < + 664-665 number 3 + 665-666 punctuation ) + 666-667 punctuation ; + 671-674 special_keyword for + 675-676 punctuation ( + 676-681 keyword const + 685-687 keyword of + 688-692 keyword this + 692-693 punctuation . + 695-696 punctuation ) + 697-698 punctuation { + 702-704 special_keyword if + 705-706 punctuation ( + 709-712 operator === + 713-716 string 'd' + 716-717 punctuation ) + 718-726 special_keyword continue + 726-727 punctuation ; + 731-733 special_keyword if + 734-735 punctuation ( + 735-736 operator ! + 738-739 punctuation ) + 740-745 special_keyword break + 745-746 punctuation ; + 750-754 keyword this + 754-755 punctuation . + 755-770 function #private_method + 770-771 punctuation ( + 772-773 punctuation , + 776-777 punctuation ) + 777-778 punctuation ; + 781-782 punctuation } + 786-792 special_keyword switch + 793-794 punctuation ( + 794-798 keyword this + 798-799 punctuation . + 801-802 punctuation ) + 803-804 punctuation { + 808-812 special_keyword case + 813-816 string 'a' + 816-817 operator : + 822-829 builtin console + 829-830 punctuation . + 830-833 function log + 833-834 punctuation ( + 834-842 string 'case a' + 842-843 punctuation ) + 843-844 punctuation ; + 849-854 special_keyword break + 854-855 punctuation ; + 859-863 special_keyword case + 864-867 string 'b' + 867-868 operator : + 872-876 special_keyword case + 877-880 string 'c' + 880-881 operator : + 886-893 builtin console + 893-894 punctuation . + 894-897 function log + 897-898 punctuation ( + 898-911 string 'case b or c' + 911-912 punctuation ) + 912-913 punctuation ; + 918-923 special_keyword break + 923-924 punctuation ; + 928-935 special_keyword default + 935-936 operator : + 941-948 builtin console + 948-949 punctuation . + 949-952 function log + 952-953 punctuation ( + 953-962 string 'default' + 962-963 punctuation ) + 963-964 punctuation ; + 967-968 punctuation } + 972-977 keyword const + 981-982 operator : + 983-984 punctuation { + 990-991 operator ? + 991-992 operator : + 993-1000 builtin boolean +1000-1001 punctuation ; +1006-1007 operator : +1008-1015 builtin boolean +1015-1016 punctuation } +1017-1018 operator = +1019-1020 punctuation { +1030-1031 operator : +1032-1036 string 'd1' +1037-1039 keyword in +1040-1044 keyword this +1044-1045 punctuation , +1053-1054 operator : +1055-1059 keyword this +1060-1070 keyword instanceof +1071-1072 class_name D +1071-1072 constant D +1072-1073 punctuation , +1076-1077 punctuation } +1077-1078 punctuation ; +1081-1087 keyword delete +1091-1092 punctuation . +1098-1099 punctuation ; +1102-1108 comment // foo +1110-1111 punctuation } +1111-1112 punctuation ; +1115-1130 function #private_method +1130-1131 punctuation ( +1133-1134 operator : +1135-1141 builtin number +1141-1142 punctuation , +1145-1146 operator : +1147-1150 builtin any +1150-1151 punctuation ) +1151-1152 operator : +1153-1157 keyword void +1158-1159 punctuation { +1162-1167 special_keyword throw +1168-1171 keyword new +1172-1177 class_name Error +1172-1177 capitalized_identifier Error +1177-1178 punctuation ( +1178-1224 template_string `${this.d1}\n\t\t\tmultiline\n\t\t\tetc ${a2 + c2}\n\t\t` +1178-1179 template_punctuation ` +1179-1189 interpolation ${this.d1} +1179-1181 interpolation_punctuation ${ +1181-1185 keyword this +1185-1186 punctuation . +1188-1189 interpolation_punctuation } +1189-1210 string \n\t\t\tmultiline\n\t\t\tetc +1210-1220 interpolation ${a2 + c2} +1210-1212 interpolation_punctuation ${ +1215-1216 operator + +1219-1220 interpolation_punctuation } +1220-1223 string \n\t\t +1223-1224 template_punctuation ` +1224-1225 punctuation ) +1225-1226 punctuation ; +1228-1229 punctuation } +1232-1233 operator * +1233-1242 function generator +1242-1243 punctuation ( +1243-1244 punctuation ) +1244-1245 operator : +1246-1255 capitalized_identifier Generator +1255-1256 operator < +1256-1262 builtin number +1263-1264 operator | +1265-1270 capitalized_identifier Array +1270-1271 operator < +1271-1277 builtin number +1277-1279 operator >> +1280-1281 punctuation { +1284-1289 special_keyword yield +1290-1291 number 1 +1291-1292 punctuation ; +1295-1300 special_keyword yield +1300-1301 operator * +1302-1303 punctuation [ +1303-1304 number 2 +1304-1305 punctuation , +1306-1307 number 3 +1307-1308 punctuation ] +1308-1309 punctuation ; +1311-1312 punctuation } +1315-1320 keyword async +1321-1322 operator * +1322-1337 function async_generator +1337-1338 punctuation ( +1338-1339 punctuation ) +1339-1340 operator : +1341-1355 capitalized_identifier AsyncGenerator +1355-1356 operator < +1356-1362 builtin number +1362-1363 operator > +1364-1365 punctuation { +1368-1373 special_keyword yield +1374-1379 special_keyword await +1380-1387 capitalized_identifier Promise +1387-1388 punctuation . +1388-1395 function resolve +1395-1396 punctuation ( +1396-1397 number 4 +1397-1398 punctuation ) +1398-1399 punctuation ; +1401-1402 punctuation } +1405-1414 keyword protected +1415-1420 keyword async +1421-1437 function protected_method +1437-1438 punctuation ( +1438-1439 punctuation ) +1439-1440 operator : +1441-1448 capitalized_identifier Promise +1448-1449 operator < +1449-1453 keyword void +1453-1454 operator > +1455-1456 punctuation { +1459-1462 special_keyword try +1463-1464 punctuation { +1468-1473 special_keyword await +1474-1477 keyword new +1478-1485 class_name Promise +1478-1485 capitalized_identifier Promise +1485-1486 punctuation ( +1486-1487 punctuation ( +1494-1495 punctuation ) +1496-1498 operator => +1499-1509 function setTimeout +1509-1510 punctuation ( +1517-1518 punctuation , +1519-1522 number 100 +1522-1523 punctuation ) +1523-1524 punctuation ) +1524-1525 punctuation ; +1529-1531 special_keyword if +1532-1533 punctuation ( +1533-1537 capitalized_identifier Math +1537-1538 punctuation . +1538-1544 function random +1544-1545 punctuation ( +1545-1546 punctuation ) +1547-1548 operator > +1549-1552 number 0.5 +1552-1553 punctuation ) +1554-1555 punctuation { +1560-1567 builtin console +1567-1568 punctuation . +1568-1571 function log +1571-1572 punctuation ( +1572-1575 keyword new +1576-1580 class_name Date +1576-1580 capitalized_identifier Date +1580-1581 punctuation ( +1581-1582 punctuation ) +1582-1583 punctuation ) +1583-1584 punctuation ; +1588-1589 punctuation } +1590-1594 special_keyword else +1595-1597 special_keyword if +1598-1599 punctuation ( +1599-1603 capitalized_identifier Math +1603-1604 punctuation . +1604-1610 function random +1610-1611 punctuation ( +1611-1612 punctuation ) +1613-1614 operator > +1615-1618 number 0.2 +1618-1619 punctuation ) +1620-1621 punctuation { +1626-1633 builtin console +1633-1634 punctuation . +1634-1637 function log +1637-1638 punctuation ( +1638-1654 string 'else if branch' +1654-1655 punctuation ) +1655-1656 punctuation ; +1660-1661 punctuation } +1662-1666 special_keyword else +1667-1668 punctuation { +1673-1680 builtin console +1680-1681 punctuation . +1681-1684 function log +1684-1685 punctuation ( +1685-1698 string 'else branch' +1698-1699 punctuation ) +1699-1700 punctuation ; +1704-1705 punctuation } +1708-1709 punctuation } +1710-1715 special_keyword catch +1716-1717 punctuation ( +1722-1723 operator : +1724-1731 builtin unknown +1731-1732 punctuation ) +1733-1734 punctuation { +1738-1745 builtin console +1745-1746 punctuation . +1746-1751 function error +1751-1752 punctuation ( +1757-1758 punctuation ) +1758-1759 punctuation ; +1762-1763 punctuation } +1764-1771 special_keyword finally +1772-1773 punctuation { +1777-1784 builtin console +1784-1785 punctuation . +1785-1788 function log +1788-1789 punctuation ( +1789-1804 string 'finally block' +1804-1805 punctuation ) +1805-1806 punctuation ; +1809-1810 punctuation } +1812-1813 punctuation } +1814-1815 punctuation } +1817-1827 comment // comment +1829-1872 comment /*\nother comment\n\nconst comment = false;\n*/ +1874-1898 comment /**\n * JSDoc comment\n */ +1900-1906 special_keyword import +1907-1908 punctuation { +1920-1921 punctuation , +1922-1926 keyword type +1927-1938 class_name Sample_Lang +1927-1938 capitalized_identifier Sample_Lang +1938-1939 punctuation } +1940-1944 special_keyword from +1945-1964 string '../code_sample.js' +1964-1965 punctuation ; +1966-1972 special_keyword import +1973-1974 operator * +1975-1977 special_keyword as +1978-1979 constant A +1980-1984 special_keyword from +1985-2004 string '../code_sample.js' +2004-2005 punctuation ; +2007-2013 special_keyword export +2014-2015 punctuation { +2016-2017 punctuation , +2018-2019 constant A +2019-2020 punctuation , +2022-2023 punctuation , +2025-2026 punctuation , +2027-2028 constant D +2028-2029 punctuation } +2029-2030 punctuation ; +2045-2047 special_keyword as +2048-2055 builtin unknown +2056-2058 special_keyword as +2059-2070 capitalized_identifier Sample_Lang +2071-2080 keyword satisfies +2081-2092 capitalized_identifier Sample_Lang +2092-2093 punctuation ; +2095-2101 special_keyword export +2102-2111 keyword interface +2112-2128 class_name Some_E +2112-2118 capitalized_identifier Some_E +2118-2119 operator < +2119-2120 constant T +2121-2122 operator = +2123-2127 keyword null +2127-2128 operator > +2129-2130 punctuation { +2136-2137 operator : +2138-2144 builtin string +2144-2145 punctuation ; +2150-2151 operator : +2152-2158 builtin number +2158-2159 punctuation ; +2162-2163 operator ? +2163-2164 operator : +2165-2166 constant T +2166-2167 punctuation ; +2168-2169 punctuation } +2171-2176 keyword const +2178-2179 operator : +2180-2181 punctuation { +2185-2186 operator : +2187-2193 builtin string +2193-2194 punctuation ; +2198-2199 operator : +2200-2206 builtin number +2206-2207 punctuation } +2208-2209 operator = +2210-2211 punctuation { +2215-2216 operator : +2217-2224 string 'A. H.' +2224-2225 punctuation , +2229-2230 operator : +2231-2234 number 100 +2234-2235 punctuation } +2235-2236 punctuation ; +2237-2242 keyword const +2245-2246 operator = +2247-2248 punctuation [ +2248-2249 punctuation [ +2249-2251 string '' +2251-2252 punctuation , +2254-2255 punctuation ] +2255-2256 punctuation ] +2257-2259 special_keyword as +2260-2265 keyword const +2265-2266 punctuation ; +2267-2273 special_keyword export +2274-2279 keyword const +2286-2308 type_annotation : Map +2286-2287 : : +2287-2308 type Map +2288-2291 capitalized_identifier Map +2291-2292 operator < +2292-2298 builtin string +2298-2299 punctuation , +2300-2306 capitalized_identifier Some_E +2306-2307 operator > +2308-2309 operator = +2310-2313 keyword new +2314-2317 class_name Map +2314-2317 capitalized_identifier Map +2317-2318 punctuation ( +2319-2320 punctuation ) +2320-2321 punctuation ; +2323-2329 special_keyword export +2330-2338 keyword function +2339-2342 function add +2342-2343 punctuation ( +2344-2345 operator : +2346-2352 builtin number +2352-2353 punctuation , +2355-2356 operator : +2357-2363 builtin number +2363-2364 punctuation ) +2364-2365 operator : +2366-2372 builtin number +2373-2374 punctuation { +2376-2382 special_keyword return +2385-2386 operator + +2388-2389 punctuation ; +2390-2391 punctuation } +2393-2399 special_keyword export +2400-2405 keyword const +2406-2410 function_variable plus +2411-2412 operator = +2413-2414 punctuation ( +2415-2416 operator : +2417-2420 builtin any +2420-2421 punctuation , +2423-2424 operator : +2425-2428 builtin any +2428-2429 punctuation ) +2429-2435 type_annotation : any +2429-2430 : : +2430-2435 type any +2431-2434 builtin any +2435-2437 operator => +2440-2441 operator + +2443-2444 punctuation ; +2446-2468 comment // boundary test cases +2469-2475 special_keyword export +2476-2481 keyword const +2500-2501 operator = +2502-2531 string 'const class function string' +2531-2532 punctuation ; +2533-2539 special_keyword export +2540-2545 keyword const +2563-2564 operator = +2565-2591 string '// this is not a comment' +2591-2592 punctuation ; +2593-2599 special_keyword export +2600-2605 keyword const +2625-2626 operator = +2627-2644 template_string `Value: ${1 + 2}` +2627-2628 template_punctuation ` +2628-2635 string Value: +2635-2643 interpolation ${1 + 2} +2635-2637 interpolation_punctuation ${ +2637-2638 number 1 +2639-2640 operator + +2641-2642 number 2 +2642-2643 interpolation_punctuation } +2643-2644 template_punctuation ` +2644-2645 punctuation ; +2647-2679 comment // regex that looks like comment +2680-2686 special_keyword export +2687-2692 keyword const +2699-2700 operator = +2701-2710 regex /\/\/.*/g +2701-2702 regex_delimiter / +2702-2708 regex_source \/\/.* +2708-2709 regex_delimiter / +2709-2710 regex_flags g +2710-2711 punctuation ; +2712-2718 special_keyword export +2719-2724 keyword const +2739-2740 operator = +2741-2773 regex /^(?:\/\*.*?\*\/|\/\/.*|[^/])+$/ +2741-2742 regex_delimiter / +2742-2772 regex_source ^(?:\/\*.*?\*\/|\/\/.*|[^/])+$ +2772-2773 regex_delimiter / +2773-2774 punctuation ; +2776-2832 comment // string in comment should not be highlighted as string +2833-2884 comment // const commented = "this string is in a comment"; diff --git a/src/fixtures/helpers.ts b/src/fixtures/helpers.ts index 3cccd320..b43d5dd5 100644 --- a/src/fixtures/helpers.ts +++ b/src/fixtures/helpers.ts @@ -50,8 +50,12 @@ export const discover_samples = (): Array => { /** * Get the fixture path for a given language and variant */ -export const get_fixture_path = (lang: string, variant: string, ext: 'json' | 'txt'): string => { - return join('fixtures/generated', lang, `${lang}_${variant}.${ext}`); +export const get_fixture_path = ( + lang: string, + variant: string, + ext: 'json' | 'txt' | 'html', +): string => { + return join('src/fixtures/generated', lang, `${lang}_${variant}.${ext}`); }; /** @@ -151,7 +155,22 @@ export const process_sample = (sample: Sample_Spec): Generated_Output => { export const generate_debug_text = (output: Generated_Output): string => { const {sample, tokens} = output; - let debug = '=== TOKENS ===\n'; + let debug = '=== STATS ===\n'; + debug += `Sample length: ${sample.content.length} characters\n`; + debug += `Total tokens: ${tokens.length}\n`; + + // Count token types + const tokenTypes: Record = {}; + for (const token of tokens) { + const {type} = token; + tokenTypes[type] = (tokenTypes[type] || 0) + 1; + } + debug += `\nToken types (${Object.keys(tokenTypes).length} unique):\n`; + for (const [type, count] of Object.entries(tokenTypes).sort((a, b) => b[1] - a[1])) { + debug += ` ${type}: ${count}\n`; + } + + debug += '\n=== TOKENS ===\n'; // Show all tokens, no elision for (const t of tokens) { const text = sample.content @@ -165,21 +184,5 @@ export const generate_debug_text = (output: Generated_Output): string => { debug += `${position.padEnd(10)} ${t.type.padEnd(25)} ${text}\n`; } - // Add token statistics - debug += '\n=== STATS ===\n'; - debug += `Total tokens: ${tokens.length}\n`; - debug += `Sample length: ${sample.content.length} characters\n`; - - // Count token types - const tokenTypes: Record = {}; - for (const token of tokens) { - const {type} = token; - tokenTypes[type] = (tokenTypes[type] || 0) + 1; - } - debug += '\nToken types:\n'; - for (const [type, count] of Object.entries(tokenTypes).sort((a, b) => b[1] - a[1])) { - debug += ` ${type}: ${count}\n`; - } - return debug; }; diff --git a/src/fixtures/update.task.ts b/src/fixtures/update.task.ts index 5f904f6d..a3b3cd1a 100644 --- a/src/fixtures/update.task.ts +++ b/src/fixtures/update.task.ts @@ -1,7 +1,6 @@ import type {Task} from '@ryanatkn/gro'; import {writeFileSync, mkdirSync, rmSync, existsSync} from 'node:fs'; -import {join} from 'node:path'; -import {format_file} from '@ryanatkn/gro/format_file.js'; +import {join, resolve} from 'node:path'; import { discover_samples, process_sample, @@ -20,9 +19,11 @@ export const task: Task = { // Get unique languages to clean directories const languages = new Set(samples.map((s) => s.lang)); + const generated_fixtures_dir = resolve('src/fixtures/generated'); + // Remove existing language directories for (const lang of languages) { - const dir = join('fixtures/generated', lang); + const dir = join(generated_fixtures_dir, lang); if (existsSync(dir)) { rmSync(dir, {recursive: true, force: true}); console.log(`Removed existing directory: ${dir}`); // eslint-disable-line no-console @@ -37,15 +38,13 @@ export const task: Task = { const output = process_sample(sample); // Ensure directory exists - const dir = join('fixtures/generated', sample.lang); + const dir = join(generated_fixtures_dir, sample.lang); mkdirSync(dir, {recursive: true}); - // Write JSON file - const json_path = get_fixture_path(sample.lang, sample.variant, 'json'); - const json_content = JSON.stringify(output); - const formatted_json = await format_file(json_content, {filepath: json_path}); // eslint-disable-line no-await-in-loop - writeFileSync(json_path, formatted_json); - console.log(` → ${json_path}`); // eslint-disable-line no-console + // Write HTML file (no formatting needed, already formatted) + const html_path = get_fixture_path(sample.lang, sample.variant, 'html'); + writeFileSync(html_path, output.html); + console.log(` → ${html_path}`); // eslint-disable-line no-console // Generate and write debug text file const debug_text = generate_debug_text(output); diff --git a/src/lib/Code.svelte b/src/lib/Code.svelte index 8f5cf6e6..69043c5d 100644 --- a/src/lib/Code.svelte +++ b/src/lib/Code.svelte @@ -112,6 +112,7 @@ /* not a comment */'; + /* prettier-ignore */ + content: " /* not a comment */"; } .attr[title='Click: here'] { - cursor: pointer; -} - -.background /* comment*/ { background-image: url('data:image/svg+xml...'); } `, @@ -88,44 +84,109 @@ div > p { lang: 'ts', content: `const a = 1; -const b = 'b'; +const b: string = 'b'; const c = true; export type Some_Type = 1 | 'b' | true; -class D { - d1: string = 'd'; +declare const some_decorator: any; + +abstract class Base { + abstract abstract_method(): void; +} + +/* eslint-disable no-console */ + +@some_decorator +class D extends Base { + readonly d1: string = 'd'; d2: number; - d3 = $state(false); + d3 = $state(null); + + @some_decorator + decorated = true; constructor(d2: number) { + super(); this.d2 = d2; } + abstract_method(): void { + // implementation + } + + @some_decorator('example', {option: true}) class_method(): string { return \`Hello, \${this.d1}\`; } instance_method = (): void => { /* ... */ - this.#private_method(); + let i = 0; + do { + i++; + } while (i < 3); + + for (const c2 of this.d1) { + if (c2 === 'd') continue; + if (!c2) break; + this.#private_method(a, c2); + } + + switch (this.d1) { + case 'a': + console.log('case a'); + break; + case 'b': + case 'c': + console.log('case b or c'); + break; + default: + console.log('default'); + } + + const obj: {has_d1?: boolean; is_d: boolean} = { + has_d1: 'd1' in this, + is_d: this instanceof D, + }; + delete obj.has_d1; // foo }; - #private_method() { - throw new Error(\`\${this.d1} + #private_method(a2: number, c2: any): void { + throw new Error(\`\${this.d1} multiline - etc + etc \${a2 + c2} \`); } - protected protected_method(): void { - console.log(new Date()); // eslint-disable-line no-console + *generator(): Generator> { + yield 1; + yield* [2, 3]; } -} -export {a, b, c, D}; + async *async_generator(): AsyncGenerator { + yield await Promise.resolve(4); + } + + protected async protected_method(): Promise { + try { + await new Promise((resolve) => setTimeout(resolve, 100)); + if (Math.random() > 0.5) { + console.log(new Date()); + } else if (Math.random() > 0.2) { + console.log('else if branch'); + } else { + console.log('else branch'); + } + } catch (error: unknown) { + console.error(error); + } finally { + console.log('finally block'); + } + } +} // comment @@ -139,12 +200,22 @@ const comment = false; * JSDoc comment */ -export interface Some_E { +import {sample_langs, type Sample_Lang} from '../code_sample.js'; +import * as A from '../code_sample.js'; + +export {a, A, b, c, D}; + +sample_langs as unknown as Sample_Lang satisfies Sample_Lang; + +export interface Some_E { name: string; age: number; + t?: T; } -export const some_e: Some_E = {name: 'A. H.', age: 100}; +const e: {name: string; age: number} = {name: 'A. H.', age: 100}; +const v = [['', e]] as const; +export const some_e: Map = new Map(v); export function add(x: number, y: number): number { return x + y; @@ -182,7 +253,7 @@ export const complex_regex = /^(?:\\/\\*.*?\\*\\/|\\/\\/.*|[^/])+$/;
-
+
access @@ -191,7 +262,15 @@ export const complex_regex = /^(?:\\/\\*.*?\\*\\/|\\/\\/.*|[^/])+$/;
  • list item 2
  • - @@ -220,108 +299,73 @@ export const complex_regex = /^(?:\\/\\*.*?\\*\\/|\\/\\/.*|[^/])+$/; thing, bound = $bindable(true), children, + onclick, }: { thing: Record; bound?: boolean; children: Snippet; + onclick?: () => void; } = $props(); - const thing_keys = $derived(Object.keys(thing)); + const thing_keys = $derived(Object.entries(thing)); - const a = 1; + const a = 1 as number; const b = 'b'; let c: boolean = $state(true); - export type Some_Type = 1 | 'b' | true; - - class D { - d1: string = 'd'; - d2: number; - d3 = $state(false); - - constructor(d2: number) { - this.d2 = d2; - } - - class_method(): string { - return \`Hello, \${this.d1}\`; - } + const f = (p: any) => p; - instance_method = () => { - /* ... */ - this.#private_method(); - // foo + const attachment = (_p1: string, _p2: number) => { + return (el: HTMLElement) => { + element_ref !== el; }; + }; - #private_method() { - throw new Error(\`\${this.d1} etc\`); - } - - protected protected_method() { - console.log(new Date(123)); // eslint-disable-line no-console - } - } - - // comment - - /* - other comment - - const comment = false; - */ - - /** - * JSDoc comment - */ - - export interface Some_E { - name: string; - age: number; - } - - export const some_e: Some_E = {name: 'A. H.', age: 100}; - - export function add(x: number, y: number): number { - return x + y; - } - - export const plus = (a: any, b: any): any => a + b; + let value = $state(thing['']); + let element_ref: HTMLElement; -

    hello {HELLO}!

    +

    hello {HELLO}!

    -{#each thing_keys as key (key)} - {@const value = thing[key]} - {value} +{#each thing_keys as [k, { t, u }] (f(k))} + {@const v = Math.round(t[k + f(u)])} + {f(v)} {/each} -{#if c} - +{#if f(c)} + +{:else if f(a > 0)} + bigger {:else} - (c = !c)}> + (c = f(!c))}> {@render children()} {/if} - + +{@html 'raw html'} -
    -

    hello world!

    -
    + + +... + +{@render my_snippet('p')} -

    +{#snippet my_snippet(p: string)} + +{/snippet} + +

    some text

    - -{a} -{b} + +{b.repeat(2)} {bound} -{D}
    @@ -334,6 +378,18 @@ export const complex_regex = /^(?:\\/\\*.*?\\*\\/|\\/\\/.*|[^/])+$/;
  • list item 2
  • + +
    + + +
    + diff --git a/src/lib/samples/sample_complex.css b/src/lib/samples/sample_complex.css index 6c19b1a2..7feee2d8 100644 --- a/src/lib/samples/sample_complex.css +++ b/src/lib/samples/sample_complex.css @@ -31,19 +31,15 @@ div > p { @media (max-width: 600px) { body { - background-color: lightblue; + background-color: light-dark(lightblue, darkblue); } } -/* patterns in strings are not falsely detected */ .content::before { - content: ' /* not a comment */'; + /* prettier-ignore */ + content: " /* not a comment */"; } .attr[title='Click: here'] { - cursor: pointer; -} - -.background /* comment*/ { background-image: url('data:image/svg+xml...'); } diff --git a/src/lib/samples/sample_complex.html b/src/lib/samples/sample_complex.html index c79c452d..316483d3 100644 --- a/src/lib/samples/sample_complex.html +++ b/src/lib/samples/sample_complex.html @@ -12,7 +12,7 @@
    -
    +
    access @@ -21,7 +21,15 @@
  • list item 2
  • - diff --git a/src/lib/samples/sample_complex.svelte b/src/lib/samples/sample_complex.svelte index b2b5692c..405a33b0 100644 --- a/src/lib/samples/sample_complex.svelte +++ b/src/lib/samples/sample_complex.svelte @@ -11,108 +11,73 @@ thing, bound = $bindable(true), children, + onclick, }: { thing: Record; bound?: boolean; children: Snippet; + onclick?: () => void; } = $props(); - const thing_keys = $derived(Object.keys(thing)); + const thing_keys = $derived(Object.entries(thing)); - const a = 1; + const a = 1 as number; const b = 'b'; let c: boolean = $state(true); - export type Some_Type = 1 | 'b' | true; + const f = (p: any) => p; - class D { - d1: string = 'd'; - d2: number; - d3 = $state(false); - - constructor(d2: number) { - this.d2 = d2; - } - - class_method(): string { - return `Hello, ${this.d1}`; - } - - instance_method = () => { - /* ... */ - this.#private_method(); - // foo + const attachment = (_p1: string, _p2: number) => { + return (el: HTMLElement) => { + element_ref !== el; }; + }; - #private_method() { - throw new Error(`${this.d1} etc`); - } - - protected protected_method() { - console.log(new Date(123)); // eslint-disable-line no-console - } - } - - // comment - - /* - other comment - - const comment = false; - */ - - /** - * JSDoc comment - */ - - export interface Some_E { - name: string; - age: number; - } - - export const some_e: Some_E = {name: 'A. H.', age: 100}; - - export function add(x: number, y: number): number { - return x + y; - } - - export const plus = (a: any, b: any): any => a + b; + let value = $state(thing['']); + let element_ref: HTMLElement; -

    hello {HELLO}!

    +

    hello {HELLO}!

    -{#each thing_keys as key (key)} - {@const value = thing[key]} - {value} +{#each thing_keys as [k, { t, u }] (f(k))} + {@const v = Math.round(t[k + f(u)])} + {f(v)} {/each} -{#if c} - +{#if f(c)} + +{:else if f(a > 0)} + bigger {:else} - (c = !c)}> + (c = f(!c))}> {@render children()} {/if} - + +{@html 'raw html'} -
    -

    hello world!

    -
    + + +... -

    +{@render my_snippet('p')} + +{#snippet my_snippet(p: string)} + +{/snippet} + +

    some text

    - -{a} -{b} + +{b.repeat(2)} {bound} -{D}
    @@ -125,6 +90,18 @@
  • list item 2
  • + +
    + + +
    + diff --git a/src/lib/samples/sample_complex.ts b/src/lib/samples/sample_complex.ts index a8175af3..c5cbea59 100644 --- a/src/lib/samples/sample_complex.ts +++ b/src/lib/samples/sample_complex.ts @@ -1,43 +1,108 @@ const a = 1; -const b = 'b'; +const b: string = 'b'; const c = true; export type Some_Type = 1 | 'b' | true; -class D { - d1: string = 'd'; +declare const some_decorator: any; + +abstract class Base { + abstract abstract_method(): void; +} + +/* eslint-disable no-console */ + +@some_decorator +class D extends Base { + readonly d1: string = 'd'; d2: number; - d3 = $state(false); + d3 = $state(null); + + @some_decorator + decorated = true; constructor(d2: number) { + super(); this.d2 = d2; } + abstract_method(): void { + // implementation + } + + @some_decorator('example', {option: true}) class_method(): string { return `Hello, ${this.d1}`; } instance_method = (): void => { /* ... */ - this.#private_method(); + let i = 0; + do { + i++; + } while (i < 3); + + for (const c2 of this.d1) { + if (c2 === 'd') continue; + if (!c2) break; + this.#private_method(a, c2); + } + + switch (this.d1) { + case 'a': + console.log('case a'); + break; + case 'b': + case 'c': + console.log('case b or c'); + break; + default: + console.log('default'); + } + + const obj: {has_d1?: boolean; is_d: boolean} = { + has_d1: 'd1' in this, + is_d: this instanceof D, + }; + delete obj.has_d1; // foo }; - #private_method() { - throw new Error(`${this.d1} + #private_method(a2: number, c2: any): void { + throw new Error(`${this.d1} multiline - etc + etc ${a2 + c2} `); } - protected protected_method(): void { - console.log(new Date()); // eslint-disable-line no-console + *generator(): Generator> { + yield 1; + yield* [2, 3]; } -} -export {a, b, c, D}; + async *async_generator(): AsyncGenerator { + yield await Promise.resolve(4); + } + + protected async protected_method(): Promise { + try { + await new Promise((resolve) => setTimeout(resolve, 100)); + if (Math.random() > 0.5) { + console.log(new Date()); + } else if (Math.random() > 0.2) { + console.log('else if branch'); + } else { + console.log('else branch'); + } + } catch (error: unknown) { + console.error(error); + } finally { + console.log('finally block'); + } + } +} // comment @@ -51,12 +116,22 @@ const comment = false; * JSDoc comment */ -export interface Some_E { +import {sample_langs, type Sample_Lang} from '../code_sample.js'; +import * as A from '../code_sample.js'; + +export {a, A, b, c, D}; + +sample_langs as unknown as Sample_Lang satisfies Sample_Lang; + +export interface Some_E { name: string; age: number; + t?: T; } -export const some_e: Some_E = {name: 'A. H.', age: 100}; +const e: {name: string; age: number} = {name: 'A. H.', age: 100}; +const v = [['', e]] as const; +export const some_e: Map = new Map(v); export function add(x: number, y: number): number { return x + y; diff --git a/src/lib/theme.css b/src/lib/theme.css index d39ee2e1..8b4c59de 100644 --- a/src/lib/theme.css +++ b/src/lib/theme.css @@ -1,28 +1,3 @@ -/* Comments, punctuation, and processing */ -.token.comment, -::highlight(comment), -.token.attr_equals, -::highlight(attr_equals), -.token.processing_instruction, -::highlight(processing_instruction), -.token.doctype, -::highlight(doctype), -.token.cdata, -::highlight(cdata), -.token.punctuation, -::highlight(punctuation) { - color: var(--text_color_5); -} - -/* Namespaces */ -.token.namespace, -::highlight(namespace) { - color: var(--color_d_5); -} - -/* Properties, tags, and constants */ -.token.property, -::highlight(property), .token.tag, ::highlight(tag), .token.constant, @@ -30,29 +5,20 @@ .token.symbol, ::highlight(symbol), .token.deleted, -::highlight(deleted) { - color: var(--color_a_5); -} - -/* Numbers */ -.token.number, -::highlight(number) { - color: var(--color_e_5); -} - -/* Booleans */ +::highlight(deleted), +.token.keyword, +::highlight(keyword), +.token.null, +::highlight(null), .token.boolean, -::highlight(boolean) { - color: var(--color_i_5); +::highlight(boolean), +.token.interpolation_punctuation, +::highlight(interpolation_punctuation) { + color: var(--color_a_5); } -/* Strings, selectors, and values */ -.token.selector, -::highlight(selector), -.token.string, -::highlight(string), -.token.attr_value, -::highlight(attr_value), +.token.comment, +::highlight(comment), .token.char, ::highlight(char), .token.builtin, @@ -62,27 +28,30 @@ color: var(--color_b_5); } -/* Keywords and at-rules */ -.token.atrule, -::highlight(atrule), -.token.attr_name, -::highlight(attr_name), -.token.keyword, -::highlight(keyword), -.token.null, -::highlight(null) { - color: var(--color_f_5); +.token.builtin, +::highlight(builtin), +.token.class_name, +::highlight(class_name), +.token.number, +::highlight(number) { + color: var(--color_j_5); } -/* Functions and classes */ -.token.function, -::highlight(function), -.token.class_name, -::highlight(class_name) { - color: var(--color_d_5); +.token.attr_value, +::highlight(attr_value), +.token.attr_quote, +::highlight(attr_quote), +.token.string, +::highlight(string), +.token.template_punctuation, +::highlight(template_punctuation) { + color: var(--color_h_5); } -/* Regular expressions and variables */ +.token.selector, +::highlight(selector), +.token.function, +::highlight(function), .token.regex, ::highlight(regex), .token.important, @@ -92,155 +61,57 @@ color: var(--color_e_5); } -/* Font weight modifiers */ -.token.important, -::highlight(important), -.token.bold, -::highlight(bold) { - font-weight: bold; -} - -/* Font style modifiers */ -.token.italic, -::highlight(italic) { - font-style: italic; -} - -/* TODO some of these should probably be used, others are unnecessary */ - -/* TypeScript/JS specific */ -/* .token.template_string, -::highlight(template_string) { - color: var(--color_b_5); -} - -.token.template_punctuation, -::highlight(template_punctuation) { - color: var(--color_a_5); +.token.atrule, +::highlight(atrule) { + color: var(--color_f_5); } -.token.parameter, -::highlight(parameter) { - color: var(--color_a_5); -} */ - -/* Regex-specific tokens */ -/* .token.regex_delimiter, -::highlight(regex_delimiter) { - color: var(--text_color_5); +.token.attr_name, +::highlight(attr_name), +.token.property, +::highlight(property), +.token.decorator, +::highlight(decorator), +.token.decorator_name, +::highlight(decorator_name) { + color: var(--color_i_5); } -.token.regex_flags, -::highlight(regex_flags) { - color: var(--color_e_5); +.token.special_keyword, +::highlight(special_keyword), +.token.namespace, +::highlight(namespace), +.token.rule, +::highlight(rule) { + color: var(--color_g_5); } -.token.regex_source, -::highlight(regex_source) { - color: var(--color_e_5); +.token.at_keyword, +::highlight(at_keyword) { + color: var(--color_d_5); } -.token.operator, -::highlight(operator) { +.token.attr_equals, +::highlight(attr_equals), +.token.processing_instruction, +::highlight(processing_instruction), +.token.doctype, +::highlight(doctype), +.token.cdata, +::highlight(cdata), +.token.punctuation, +::highlight(punctuation) { color: var(--text_color_5); -} */ - -/* CSS specific */ -/* .token.color, -::highlight(color), -.token.hexcode, -::highlight(hexcode) { - color: var(--color_e_5); -} - -.token.unit, -::highlight(unit) { - color: var(--color_e_5); -} - -.token.url, -::highlight(url) { - color: var(--color_b_5); -} */ - -/* HTML/XML specific */ -/* .token.entity, -::highlight(entity) { - color: var(--color_e_5); -} - -.token.attr, -::highlight(attr) { - color: var(--color_f_5); -} */ - -/* JSON specific */ -/* .token.property_string, -::highlight(property_string) { - color: var(--color_a_5); } -.token.function_variable, -::highlight(function_variable) { - color: var(--color_d_5); -} */ - -/* Additional tokens */ -/* .token.interpolation_punctuation, -::highlight(interpolation_punctuation) { - color: var(--color_a_5); +.token.important, +::highlight(important), +.token.bold, +::highlight(bold) { + font-weight: bold; } -.token.at, -::highlight(at) { - color: var(--color_f_5); +.token.italic, +::highlight(italic) { + font-style: italic; } - -.token.rule, -::highlight(rule) { - color: var(--color_f_5); -} */ - -/* - * TODO: Additional tokens found in current grammars that need .token rules: - * - * These tokens exist in the grammars but aren't in theme.css. They should be added - * with appropriate colors: - * - * 11. Template strings (JS) - * - .token.template_string / ::highlight(template_string) - same as string: var(--color_b_5) - * - .token.template_punctuation / ::highlight(template_punctuation) - same as string: var(--color_b_5) - * - * 12. Regex components (JS) - * - .token.regex_delimiter / ::highlight(regex_delimiter) - same as punctuation: var(--text_color_5) - * - .token.regex_flags / ::highlight(regex_flags) - same as regex: var(--color_e_5) - * - .token.regex_source / ::highlight(regex_source) - same as regex: var(--color_e_5) - * - * 13. Operators - * - .token.operator / ::highlight(operator) - same as punctuation: var(--text_color_5) - * - * 14. Function variable (JS) - * - .token.function_variable / ::highlight(function_variable) - same as function: var(--color_d_5) - * - * 15. Interpolation punctuation (JS template strings) - * - .token.interpolation_punctuation / ::highlight(interpolation_punctuation) - same as punctuation: var(--text_color_5) - * - * 16. At symbol (TS decorators) - * - .token.at / ::highlight(at) - same as operator/punctuation: var(--text_color_5) - * - * 17. CSS rule - * - .token.rule / ::highlight(rule) - same as atrule: var(--color_f_5) - * - * 18. URL (CSS) - * - .token.url / ::highlight(url) - same as string: var(--color_b_5) - * - * 19. Entity (HTML) - * - .token.entity / ::highlight(entity) - same as number: var(--color_e_5) - * - * Tokens to NOT include: - * - parameter (deleted in grammar_ts) - * - color, hexcode, unit (not in grammars) - * - attr (we use attr_name, attr_value, attr_equals) - * - property_string (doesn't exist) - * - interpolation (container only, no styling needed) - */ diff --git a/src/lib/theme_variables.css b/src/lib/theme_variables.css index 2dece325..96d92de4 100644 --- a/src/lib/theme_variables.css +++ b/src/lib/theme_variables.css @@ -16,4 +16,5 @@ Import this alongside theme.css if you're not using Moss. --color_g_5: light-dark(#e03e81, #ea7ba9); --color_h_5: light-dark(#f24e0d, #f67c4c); --color_i_5: light-dark(#19b3b3, #79ecec); + --color_j_5: light-dark(#2e9e82, #61d1b5); } diff --git a/src/routes/package.ts b/src/routes/package.ts index f7036fbe..d20ded91 100644 --- a/src/routes/package.ts +++ b/src/routes/package.ts @@ -64,7 +64,10 @@ export const package_json: Package_Json = { printWidth: 100, singleQuote: true, bracketSpacing: false, - overrides: [{files: 'package.json', options: {useTabs: false}}], + overrides: [ + {files: 'package.json', options: {useTabs: false}}, + {files: 'src/fixtures/generated/**', options: {requirePragma: true}}, + ], }, sideEffects: ['**/*.css'], files: ['dist', 'src/lib/**/*.ts', '!src/lib/**/*.test.*', '!dist/**/*.test.*'], @@ -95,7 +98,10 @@ export const src_json: Src_Json = { }, './grammar_clike.js': { path: 'grammar_clike.ts', - declarations: [{name: 'add_grammar_clike', kind: 'function'}], + declarations: [ + {name: 'class_keywords', kind: 'variable'}, + {name: 'add_grammar_clike', kind: 'function'}, + ], }, './grammar_css.js': { path: 'grammar_css.ts', @@ -136,6 +142,17 @@ export const src_json: Src_Json = { {name: 'Highlight_Manager', kind: 'class'}, ], }, + './highlight_priorities.gen.js': { + path: 'highlight_priorities.gen.ts', + declarations: [{name: 'gen', kind: 'variable'}], + }, + './highlight_priorities.js': { + path: 'highlight_priorities.ts', + declarations: [ + {name: 'Highlight_Token_Name', kind: 'type'}, + {name: 'highlight_priorities', kind: 'variable'}, + ], + }, './syntax_styler_global.js': { path: 'syntax_styler_global.ts', declarations: [{name: 'syntax_styler_global', kind: 'variable'}], diff --git a/src/routes/samples/+page.svelte b/src/routes/samples/+page.svelte index 406eec06..9be855f8 100644 --- a/src/routes/samples/+page.svelte +++ b/src/routes/samples/+page.svelte @@ -13,11 +13,11 @@

    {sample.lang}

    -

    HTML

    +

    {sample.lang} html strings

    -

    Highlight API

    +

    {sample.lang} highlighted ranges