Merge pull request #253 from patricklx/fix-byte-issues

Author: gitKrystan

src/parse/index.ts

@@ -1,37 +1,42 @@
 import { traverse } from '@babel/core';
 import type {
   BlockStatement,
+  File,
   Node,
   ObjectExpression,
   StaticBlock,
 } from '@babel/types';
-import type { Parsed as RawGlimmerTemplate } from 'content-tag';
 import { Preprocessor } from 'content-tag';
 import type { Parser } from 'prettier';
 import { parsers as babelParsers } from 'prettier/plugins/babel.js';
 
 import { PRINTER_NAME } from '../config.js';
 import type { Options } from '../options.js';
 import { assert } from '../utils/index.js';
-import { preprocessTemplateRange } from './preprocess.js';
+import {
+  byteToCharIndex,
+  preprocessTemplateRange,
+  type Template,
+} from './preprocess.js';
 
 const typescript = babelParsers['babel-ts'] as Parser<Node | undefined>;
 const p = new Preprocessor();
 
 /** Converts a node into a GlimmerTemplate node */
 function convertNode(
   node: BlockStatement | ObjectExpression | StaticBlock,
-  rawTemplate: RawGlimmerTemplate,
+  rawTemplate: Template,
 ): void {
+  node.innerComments = [];
   node.extra = Object.assign(node.extra ?? {}, {
     isGlimmerTemplate: true as const,
     template: rawTemplate,
   });
 }
 
 /** Traverses the AST and replaces the transformed template parts with other AST */
-function convertAst(ast: Node, rawTemplates: RawGlimmerTemplate[]): void {
-  const unprocessedTemplates = [...rawTemplates];
+function convertAst(ast: File, templates: Template[]): void {
+  const unprocessedTemplates = [...templates];
 
   traverse(ast, {
     enter(path) {
@@ -47,11 +52,10 @@ function convertAst(ast: Node, rawTemplates: RawGlimmerTemplate[]): void {
 
         const templateIndex = unprocessedTemplates.findIndex(
           (t) =>
-            (t.range.start === start && t.range.end === end) ||
+            (t.utf16Range.start === start && t.utf16Range.end === end) ||
             (node.type === 'ObjectExpression' &&
-              node.extra?.['parenthesized'] === true &&
-              t.range.start === start - 1 &&
-              t.range.end === end + 1),
+              t.utf16Range.start === start - 1 &&
+              t.utf16Range.end === end + 1),
         );
         if (templateIndex > -1) {
           const rawTemplate = unprocessedTemplates.splice(templateIndex, 1)[0];
@@ -60,6 +64,12 @@ function convertAst(ast: Node, rawTemplates: RawGlimmerTemplate[]): void {
               'expected raw template because splice index came from findIndex',
             );
           }
+          const index =
+            node.innerComments?.[0] &&
+            ast.comments?.indexOf(node.innerComments[0]);
+          if (ast.comments && index !== undefined && index >= 0) {
+            ast.comments.splice(index, 1);
+          }
           convertNode(node, rawTemplate);
         } else {
           return null;
@@ -87,15 +97,25 @@ function preprocess(
   fileName: string,
 ): {
   code: string;
-  rawTemplates: RawGlimmerTemplate[];
+  templates: Template[];
 } {
   const rawTemplates = p.parse(code, fileName);
+  const templates: Template[] = rawTemplates.map((r) => ({
+    type: r.type,
+    range: r.range,
+    contentRange: r.contentRange,
+    contents: r.contents,
+    utf16Range: {
+      start: byteToCharIndex(code, r.range.start),
+      end: byteToCharIndex(code, r.range.end),
+    },
+  }));
 
-  for (const rawTemplate of rawTemplates) {
-    code = preprocessTemplateRange(rawTemplate, code);
+  for (const template of templates) {
+    code = preprocessTemplateRange(template, code);
   }
 
-  return { rawTemplates, code };
+  return { templates, code };
 }
 
 export const parser: Parser<Node | undefined> = {
@@ -106,7 +126,7 @@ export const parser: Parser<Node | undefined> = {
     const preprocessed = preprocess(code, options.filepath);
     const ast = await typescript.parse(preprocessed.code, options);
     assert('expected ast', ast);
-    convertAst(ast, preprocessed.rawTemplates);
+    convertAst(ast as File, preprocessed.templates);
     return ast;
   },
 };

src/parse/preprocess.ts

@@ -1,104 +1,85 @@
-import type { Parsed as RawGlimmerTemplate } from 'content-tag';
-
-const EMPTY_SPACE = ' ';
+export interface Template {
+  contents: string;
+  type: string;
+  range: {
+    start: number;
+    end: number;
+  };
+  utf16Range: {
+    start: number;
+    end: number;
+  };
+}
 
-/**
- * Given a string (`original`), replaces the bytes in the given `range` with
- * equivalent bytes of empty space (' ') surrounded by the given prefix and
- * suffix. The total byte length will not change.
- *
- * Returns the resulting string.
- */
-function replaceByteRange(
-  originalBuffer: Buffer,
-  range: { start: number; end: number },
-  options: { prefix: string; suffix: string },
-): string {
-  const prefixBuffer = Buffer.from(options.prefix);
-  const suffixBuffer = Buffer.from(options.suffix);
+const BufferMap: Map<string, Buffer> = new Map();
 
-  // Validate range
-  if (
-    range.start < 0 ||
-    range.end > originalBuffer.length ||
-    range.start > range.end ||
-    prefixBuffer.length + suffixBuffer.length > range.end - range.start
-  ) {
-    throw new Error(
-      `Invalid byte range:\n\tstart=${range.start}\n\tend=${
-        range.end
-      }\n\tprefix=${options.prefix}\n\tsuffix=${
-        options.suffix
-      }\n\tstring=\n\t${originalBuffer.toString()}`,
-    );
+function getBuffer(s: string): Buffer {
+  let buf = BufferMap.get(s);
+  if (!buf) {
+    buf = Buffer.from(s);
+    BufferMap.set(s, buf);
   }
+  return buf;
+}
 
-  // Adjust the space length to account for the prefix and suffix lengths
-  const totalReplacementLength = range.end - range.start;
-  const spaceLength =
-    totalReplacementLength - prefixBuffer.length - suffixBuffer.length;
-
-  // Create a buffer for the replacement
-  const spaceBuffer = Buffer.alloc(spaceLength, EMPTY_SPACE);
-
-  // Concatenate prefix, space, and suffix buffers
-  const replacementBuffer = Buffer.concat([
-    prefixBuffer,
-    spaceBuffer,
-    suffixBuffer,
-  ]);
-
-  // Create buffers for before and after the range using subarray
-  const beforeRange = originalBuffer.subarray(0, range.start);
-  const afterRange = originalBuffer.subarray(range.end);
+/** Slice string using byte range */
+export function sliceByteRange(s: string, a: number, b?: number): string {
+  const buf = getBuffer(s);
+  return buf.subarray(a, b).toString();
+}
 
-  // Concatenate all parts and convert back to a string
-  const result = Buffer.concat([beforeRange, replacementBuffer, afterRange]);
+/** Converts byte index to js char index (utf16) */
+export function byteToCharIndex(s: string, byteOffset: number): number {
+  const buf = getBuffer(s);
+  return buf.subarray(0, byteOffset).toString().length;
+}
 
-  if (result.length !== originalBuffer.length) {
-    throw new Error(
-      `Result length (${result.length}) does not match original length (${originalBuffer.length})`,
-    );
-  }
+/** Calculate byte length */
+export function byteLength(s: string): number {
+  return getBuffer(s).length;
+}
 
-  return result.toString('utf8');
+function replaceRange(
+  s: string,
+  start: number,
+  end: number,
+  substitute: string,
+): string {
+  return sliceByteRange(s, 0, start) + substitute + sliceByteRange(s, end);
 }
 
 /**
  * Replace the template with a parsable placeholder that takes up the same
  * range.
  */
 export function preprocessTemplateRange(
-  rawTemplate: RawGlimmerTemplate,
+  template: Template,
   code: string,
 ): string {
-  const codeBuffer = Buffer.from(code);
-
   let prefix: string;
   let suffix: string;
 
-  if (rawTemplate.type === 'class-member') {
+  if (template.type === 'class-member') {
     // Replace with StaticBlock
-    prefix = 'static{';
-    suffix = '}';
+    prefix = 'static{/*';
+    suffix = '*/}';
   } else {
     // Replace with BlockStatement or ObjectExpression
-    prefix = '{';
-    suffix = '}';
+    prefix = '{/*';
+    suffix = '*/}';
 
-    const nextToken = codeBuffer
-      .subarray(rawTemplate.range.end)
-      .toString()
-      .match(/\S+/);
+    const nextToken = code.slice(template.range.end).toString().match(/\S+/);
     if (nextToken && nextToken[0] === 'as') {
       // Replace with parenthesized ObjectExpression
       prefix = '(' + prefix;
       suffix = suffix + ')';
     }
   }
 
-  return replaceByteRange(codeBuffer, rawTemplate.range, {
-    prefix,
-    suffix,
-  });
+  const content = template.contents.replaceAll('/', '\\/');
+  const tplLength = template.range.end - template.range.start;
+  const spaces =
+    tplLength - byteLength(content) - prefix.length - suffix.length;
+  const total = prefix + content + ' '.repeat(spaces) + suffix;
+  return replaceRange(code, template.range.start, template.range.end, total);
 }

tests/cases/gts/issue-191-d.gts

@@ -0,0 +1,41 @@
+import Component from '@glimmer/component';
+import { on } from '@ember/modifier';
+
+import { getSnippetElement, toClipboard, withExtraStyles } from './copy-utils';
+import Menu from './menu';
+
+/**
+ * This component is injected via the markdown rendering
+ */
+export default class CopyMenu extends Component {
+    copyAsText = (event: Event) => {
+        let code = getSnippetElement(event);
+
+        navigator.clipboard.writeText(code.innerText);
+    };
+
+    copyAsImage = async (event: Event) => {
+        let code = getSnippetElement(event);
+
+        await withExtraStyles(code, () => toClipboard(code));
+    };
+
+    <template>
+      <Menu data-test-copy-menu>
+        <:trigger as |t|>
+          <t.Default class="absolute top-3 right-4 z-10" data-test-copy-menu>
+            📋
+          </t.Default>
+        </:trigger>
+
+        <:options as |Item|>
+          <Item {{on "click" this.copyAsText}}>
+            Copy as text
+          </Item>
+          <Item {{on "click" this.copyAsImage}}>
+            Copy as image
+          </Item>
+        </:options>
+      </Menu>
+    </template>
+}

tests/cases/gts/issue-191-e.gts

@@ -0,0 +1,16 @@
+import Component from '@glimmer/component';
+
+/**
+ * This component contains a multi-byte character
+ */
+export default class MultiByteCharComponent extends Component {
+    get rows() {
+        console.log('abc다윤6')
+        return []
+    }
+    <template>
+  {{#each this.rows as |row|}}
+                  {{row.id}}
+  {{/each}}
+    </template>
+}