Skip to content

Commit 3c85ffb

Browse files
mrlubosmrlubos
authored
Merge pull request #3828 from SukkaW/improve-speed-2
feat/perf: more speed boost + logging improvements
2 parents 460e71c + 6e5249f commit 3c85ffb

13 files changed

Lines changed: 449 additions & 143 deletions

File tree

‎.changeset/gentle-rivers-remain.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hey-api/openapi-ts": patch
3+
---
4+
5+
**cli**: print file count and generator speed

‎packages/codegen-core/src/symbols/registry.ts‎

Lines changed: 38 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@ import type { ISymbolMeta } from '../extensions';
22
import { Symbol } from './symbol';
33
import type { ISymbolIdentifier, ISymbolIn, ISymbolRegistry } from './types';
44

5-
type IndexEntry = [string, unknown];
5+
type IndexEntry = readonly [key: string, value: unknown, serialized: string];
66
type IndexKeySpace = ReadonlyArray<IndexEntry>;
77
type QueryCacheKey = string;
88
type SymbolId = number;
99

1010
export class SymbolRegistry implements ISymbolRegistry {
11-
/** Forward index: cache key → serialized entries it depends on. */
12-
private _cacheDependencies: Map<QueryCacheKey, ReadonlyArray<string>> = new Map();
11+
/** Forward index: cache key → index key space it was registered with. */
12+
private _cacheDependencies: Map<QueryCacheKey, IndexKeySpace> = new Map();
1313
/** Reverse index: serialized index entry → set of cache keys that depend on it. */
14-
private _dependencyToCache: Map<QueryCacheKey, Set<QueryCacheKey>> = new Map();
14+
private _dependencyToCache: Map<string, Set<QueryCacheKey>> = new Map();
1515
private _id: SymbolId = 0;
1616
private _indices: Map<IndexEntry[0], Map<IndexEntry[1], Set<SymbolId>>> = new Map();
1717
private _queryCache: Map<QueryCacheKey, ReadonlyArray<Symbol>> = new Map();
@@ -87,7 +87,7 @@ export class SymbolRegistry implements ISymbolRegistry {
8787
if (value && typeof value === 'object' && !Array.isArray(value)) {
8888
entries.push(...this.buildIndexKeySpace(value as ISymbolMeta, path));
8989
} else {
90-
entries.push([path, value]);
90+
entries.push([path, value, `${path}:${JSON.stringify(value)}`]);
9191
}
9292
}
9393
return entries;
@@ -99,7 +99,7 @@ export class SymbolRegistry implements ISymbolRegistry {
9999
*/
100100
private cacheKeyFromKeySpace(indexKeySpace: IndexKeySpace): QueryCacheKey {
101101
return indexKeySpace
102-
.map((indexEntry) => this.serializeIndexEntry(indexEntry))
102+
.map((indexEntry) => indexEntry[2])
103103
.sort() // ensure order-insensitivity
104104
.join('|');
105105
}
@@ -116,7 +116,7 @@ export class SymbolRegistry implements ISymbolRegistry {
116116

117117
private invalidateCache(indexKeySpace: IndexKeySpace): void {
118118
for (const indexEntry of indexKeySpace) {
119-
const serialized = this.serializeIndexEntry(indexEntry);
119+
const serialized = indexEntry[2];
120120
const cacheKeys = this._dependencyToCache.get(serialized);
121121
if (!cacheKeys) continue;
122122
for (const cacheKey of cacheKeys) {
@@ -126,8 +126,8 @@ export class SymbolRegistry implements ISymbolRegistry {
126126
const deps = this._cacheDependencies.get(cacheKey);
127127
if (deps) {
128128
for (const dep of deps) {
129-
if (dep !== serialized) {
130-
this._dependencyToCache.get(dep)?.delete(cacheKey);
129+
if (dep[2] !== serialized) {
130+
this._dependencyToCache.get(dep[2])?.delete(cacheKey);
131131
}
132132
}
133133
this._cacheDependencies.delete(cacheKey);
@@ -138,7 +138,7 @@ export class SymbolRegistry implements ISymbolRegistry {
138138
}
139139

140140
private isSubset(sub: IndexKeySpace, sup: IndexKeySpace): boolean {
141-
const supMap = new Map(sup);
141+
const supMap = new Map(sup.map((e) => [e[0], e[1]] as const));
142142
for (const [key, value] of sub) {
143143
if (!supMap.has(key) || supMap.get(key) !== value) {
144144
return false;
@@ -157,22 +157,40 @@ export class SymbolRegistry implements ISymbolRegistry {
157157
}
158158
const sets: Array<Set<SymbolId>> = [];
159159
let missed = false;
160+
161+
// Build index sets and register cache dependencies inline within a single pass.
160162
for (const indexEntry of indexKeySpace) {
161-
const values = this._indices.get(indexEntry[0]);
162-
if (!values) {
163-
missed = true;
164-
break;
163+
const serialized = indexEntry[2];
164+
let cacheKeys: Set<string>;
165+
166+
if (this._dependencyToCache.has(serialized)) {
167+
cacheKeys = this._dependencyToCache.get(serialized)!;
168+
} else {
169+
cacheKeys = new Set();
170+
this._dependencyToCache.set(serialized, cacheKeys);
165171
}
166-
const set = values.get(indexEntry[1]);
167-
if (!set) {
168-
missed = true;
169-
break;
172+
173+
cacheKeys.add(cacheKey);
174+
175+
if (!missed) {
176+
const values = this._indices.get(indexEntry[0]);
177+
if (!values) {
178+
missed = true;
179+
continue;
180+
}
181+
const set = values.get(indexEntry[1]);
182+
if (!set) {
183+
missed = true;
184+
continue;
185+
}
186+
sets.push(set);
170187
}
171-
sets.push(set);
172188
}
189+
190+
this._cacheDependencies.set(cacheKey, indexKeySpace);
191+
173192
if (missed || !sets.length) {
174193
this._queryCache.set(cacheKey, []);
175-
this.registerCacheDependencies(cacheKey, indexKeySpace);
176194
return [];
177195
}
178196

@@ -193,29 +211,9 @@ export class SymbolRegistry implements ISymbolRegistry {
193211

194212
const symbols = Array.from(result, (symbolId) => this._values.get(symbolId)!);
195213
this._queryCache.set(cacheKey, symbols);
196-
this.registerCacheDependencies(cacheKey, indexKeySpace);
197214
return symbols;
198215
}
199216

200-
/**
201-
* Registers reverse-index entries so that when any of the given index entries
202-
* are invalidated, the cache key is evicted in O(1) per entry.
203-
*/
204-
private registerCacheDependencies(cacheKey: QueryCacheKey, indexKeySpace: IndexKeySpace): void {
205-
const serializedEntries: Array<string> = [];
206-
for (const indexEntry of indexKeySpace) {
207-
const serialized = this.serializeIndexEntry(indexEntry);
208-
serializedEntries.push(serialized);
209-
let cacheKeys = this._dependencyToCache.get(serialized);
210-
if (!cacheKeys) {
211-
cacheKeys = new Set();
212-
this._dependencyToCache.set(serialized, cacheKeys);
213-
}
214-
cacheKeys.add(cacheKey);
215-
}
216-
this._cacheDependencies.set(cacheKey, serializedEntries);
217-
}
218-
219217
private replaceStubs(symbol: Symbol, indexKeySpace: IndexKeySpace): void {
220218
for (const stubId of this._stubs.values()) {
221219
const stub = this._values.get(stubId);
@@ -230,8 +228,4 @@ export class SymbolRegistry implements ISymbolRegistry {
230228
}
231229
}
232230
}
233-
234-
private serializeIndexEntry(indexEntry: IndexEntry): string {
235-
return `${indexEntry[0]}:${JSON.stringify(indexEntry[1])}`;
236-
}
237231
}

‎packages/json-schema-ref-parser/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,10 @@
5151
"dependencies": {
5252
"@jsdevtools/ono": "7.1.3",
5353
"@types/json-schema": "7.0.15",
54-
"yaml": "2.8.3"
54+
"js-yaml": "4.1.1"
5555
},
5656
"devDependencies": {
57+
"@types/js-yaml": "4.0.9",
5758
"typescript": "6.0.2"
5859
},
5960
"engines": {

‎packages/json-schema-ref-parser/src/parsers/yaml.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { parse } from 'yaml';
1+
import { load } from 'js-yaml';
22

33
import type { FileInfo, JSONSchema, Plugin } from '../types';
44
import { ParserError } from '../util/errors';
@@ -15,7 +15,7 @@ export const yamlParser: Plugin = {
1515
}
1616

1717
try {
18-
return parse(data) as JSONSchema;
18+
return load(data) as JSONSchema;
1919
} catch (error: any) {
2020
throw new ParserError(error?.message || 'Parser Error', file.url);
2121
}

‎packages/openapi-python/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
"@hey-api/shared": "workspace:*",
6565
"@hey-api/spec-types": "workspace:*",
6666
"@hey-api/types": "workspace:*",
67+
"@lukeed/ms": "2.0.2",
6768
"ansi-colors": "4.1.3",
6869
"color-support": "1.1.3",
6970
"commander": "14.0.3"

‎packages/openapi-python/src/createClient.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
patchOpenApiSpec,
1616
postprocessOutput,
1717
} from '@hey-api/shared';
18+
import { format as ms } from '@lukeed/ms';
1819
import colors from 'ansi-colors';
1920

2021
import { postProcessors } from './config/output/postprocess';
@@ -44,6 +45,7 @@ export async function createClient({
4445
headers: new Headers(),
4546
}));
4647

48+
const jobStart = Date.now();
4749
const inputPaths = config.input.map((input) => compileInputPath(input));
4850

4951
// on first run, print the message as soon as possible
@@ -166,9 +168,11 @@ export async function createClient({
166168
eventParser.timeEnd();
167169

168170
const eventGenerator = logger.timeEvent('generator');
169-
await generateOutput(context);
171+
const { fileCount } = await generateOutput(context);
170172
eventGenerator.timeEnd();
171173

174+
const totalMs = Date.now() - jobStart;
175+
172176
const eventPostprocess = logger.timeEvent('postprocess');
173177
if (!config.dryRun) {
174178
const jobPrefix = colors.gray(`[Job ${jobIndex + 1}] `);
@@ -179,7 +183,7 @@ export async function createClient({
179183
? `./${path.relative(process.env.INIT_CWD, config.output.path)}`
180184
: config.output.path;
181185
console.log(
182-
`${jobPrefix}${colors.green('✅ Done!')} Your output is in ${colors.cyanBright(outputPath)}`,
186+
`${jobPrefix}${colors.green('✅ Done!')} Your output is in ${colors.cyanBright(outputPath)} ${colors.gray(`(${fileCount} ${fileCount === 1 ? 'file' : 'files'} in ${ms(totalMs)})`)}`,
183187
);
184188
}
185189
}

‎packages/openapi-python/src/generate/output.ts‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import fs from 'node:fs';
1+
import fsPromises from 'node:fs/promises';
22
import path from 'node:path';
33

44
import type { Context } from '@hey-api/shared';
@@ -8,12 +8,17 @@ import { getTypedConfig } from '../config/utils';
88
import { getClientPlugin } from '../plugins/@hey-api/client-core/utils';
99
import { generateClientBundle } from './client';
1010

11-
export async function generateOutput(context: Context): Promise<void> {
11+
export async function generateOutput(context: Context): Promise<{ fileCount: number }> {
1212
const outputPath = path.resolve(context.config.output.path);
1313

1414
if (context.config.output.clean) {
15-
if (fs.existsSync(outputPath)) {
16-
fs.rmSync(outputPath, { force: true, recursive: true });
15+
if (
16+
await fsPromises
17+
.access(outputPath)
18+
.then(() => true)
19+
.catch(() => false)
20+
) {
21+
await fsPromises.rm(outputPath, { force: true, recursive: true });
1722
}
1823
}
1924

@@ -43,32 +48,42 @@ export async function generateOutput(context: Context): Promise<void> {
4348
await intent.run(ctx);
4449
}
4550

51+
let fileCount = 0;
52+
const writes: Promise<void>[] = [];
4653
for (const file of context.gen.render()) {
4754
const filePath = path.resolve(outputPath, file.path);
4855
const dir = path.dirname(filePath);
4956
if (!context.config.dryRun) {
50-
fs.mkdirSync(dir, { recursive: true });
51-
fs.writeFileSync(filePath, file.content, { encoding: 'utf8' });
57+
writes.push(
58+
fsPromises
59+
.mkdir(dir, { recursive: true })
60+
.then(() => fsPromises.writeFile(filePath, file.content, { encoding: 'utf8' })),
61+
);
5262
}
63+
fileCount++;
5364
}
65+
await Promise.all(writes);
5466

5567
const { source } = context.config.output;
5668
if (source.enabled) {
5769
const sourcePath = source.path === null ? undefined : path.resolve(outputPath, source.path);
5870
if (!context.config.dryRun && sourcePath && sourcePath !== outputPath) {
59-
fs.mkdirSync(sourcePath, { recursive: true });
71+
await fsPromises.mkdir(sourcePath, { recursive: true });
6072
}
6173
const serialized = await source.serialize(context.spec);
6274
// TODO: handle yaml (convert before writing)
6375
if (!context.config.dryRun && sourcePath) {
64-
fs.writeFileSync(
76+
await fsPromises.writeFile(
6577
path.resolve(sourcePath, `${source.fileName}.${source.extension}`),
6678
serialized,
6779
{ encoding: 'utf8' },
6880
);
81+
fileCount++;
6982
}
7083
if (source.callback) {
7184
await source.callback(serialized);
7285
}
7386
}
87+
88+
return { fileCount };
7489
}

‎packages/openapi-ts/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
"@hey-api/shared": "workspace:*",
7474
"@hey-api/spec-types": "workspace:*",
7575
"@hey-api/types": "workspace:*",
76+
"@lukeed/ms": "2.0.2",
7677
"ansi-colors": "4.1.3",
7778
"color-support": "1.1.3",
7879
"commander": "14.0.3",

‎packages/openapi-ts/src/createClient.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
patchOpenApiSpec,
1616
postprocessOutput,
1717
} from '@hey-api/shared';
18+
import { format as ms } from '@lukeed/ms';
1819
import colors from 'ansi-colors';
1920

2021
import { postProcessors } from './config/output/postprocess';
@@ -44,6 +45,7 @@ export async function createClient({
4445
headers: new Headers(),
4546
}));
4647

48+
const jobStart = Date.now();
4749
const inputPaths = config.input.map((input) => compileInputPath(input));
4850

4951
// on first run, print the message as soon as possible
@@ -166,9 +168,11 @@ export async function createClient({
166168
eventParser.timeEnd();
167169

168170
const eventGenerator = logger.timeEvent('generator');
169-
await generateOutput(context);
171+
const { fileCount } = await generateOutput(context);
170172
eventGenerator.timeEnd();
171173

174+
const totalMs = Date.now() - jobStart;
175+
172176
const eventPostprocess = logger.timeEvent('postprocess');
173177
if (!config.dryRun) {
174178
const jobPrefix = colors.gray(`[Job ${jobIndex + 1}] `);
@@ -179,7 +183,7 @@ export async function createClient({
179183
? `./${path.relative(process.env.INIT_CWD, config.output.path)}`
180184
: config.output.path;
181185
console.log(
182-
`${jobPrefix}${colors.green('✅ Done!')} Your output is in ${colors.cyanBright(outputPath)}`,
186+
`${jobPrefix}${colors.green('✅ Done!')} Your output is in ${colors.cyanBright(outputPath)} ${colors.gray(`(${fileCount} ${fileCount === 1 ? 'file' : 'files'} in ${ms(totalMs)})`)}`,
183187
);
184188
}
185189
}

0 commit comments

Comments
 (0)