diff --git a/CHANGELOG.md b/CHANGELOG.md index 83c084d..1ebacb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,20 @@ --- +## v0.1.3 + +- 🐛 Normalize Windows path separators when building JavaScript sourcemap `minified_url` values +- 🐛 Accept Windows-style absolute `--minified-path-prefix` values such as `\dist` +- 🐛 Avoid reporting backend payload validation errors as invalid `FLASHCAT_API_KEY` errors + +## v0.1.2 + +- 🐛 Prepare CLI release package + +## v0.1.1 + +- ✨ Add WeChat miniprogram sourcemap zip upload command + ## v0.1.0 - ✨ Add sourcemaps upload command diff --git a/package.json b/package.json index 4b3c976..3eb939b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@flashcatcloud/flashcat-cli", - "version": "0.1.0", + "version": "0.1.3", "description": "CLI tool for Flashcat Cloud", "main": "dist/index.js", "bin": { diff --git a/src/commands/flutter-symbols/__tests__/upload.test.ts b/src/commands/flutter-symbols/__tests__/upload.test.ts new file mode 100644 index 0000000..2825c8b --- /dev/null +++ b/src/commands/flutter-symbols/__tests__/upload.test.ts @@ -0,0 +1,159 @@ +import fs from 'fs' +import os from 'os' +import path from 'path' + +import {Cli} from 'clipanion/lib/advanced' + +import {UploadStatus, upload} from '../../../helpers/upload' +import {version} from '../../../helpers/version' +import {UploadCommand} from '../upload' + +jest.mock('../../../helpers/upload', () => { + const actual = jest.requireActual('../../../helpers/upload') + + return { + ...actual, + upload: jest.fn(), + } +}) + +describe('flutter-symbols upload', () => { + const mockedUpload = upload as jest.MockedFunction + + beforeEach(() => { + mockedUpload.mockReset() + process.env = {FLASHCAT_API_KEY: 'PLACEHOLDER'} + }) + + test('uploads each app.-.symbols with parsed metadata', async () => { + const dir = createSymbolsFixture(['app.android-arm64.symbols', 'app.ios-arm64.symbols']) + const uploadImplementation = jest.fn().mockResolvedValue(UploadStatus.Success) + mockedUpload.mockReturnValue(uploadImplementation) + + const {code} = await runCLI([ + 'flutter-symbols', + 'upload', + dir, + '--service', + 'my-app', + '--release-version', + '1.2.3', + ]) + + expect(code).toBe(0) + expect(uploadImplementation).toHaveBeenCalledTimes(2) + + const events = uploadImplementation.mock.calls.map(([payload]) => { + const event = payload.content.get('event') + if (event?.type !== 'string') { + throw new Error('event should be a string multipart value') + } + expect(event.options).toEqual({contentType: 'application/json'}) + + const file = payload.content.get('flutter_symbol_file') + expect(file?.type).toBe('file') + + return JSON.parse(event.value) + }) + + const byArch = Object.fromEntries(events.map((e) => [`${e.platform}-${e.arch}`, e])) + expect(byArch['android-arm64']).toEqual({ + type: 'flutter_symbol_file', + service: 'my-app', + version: '1.2.3', + flavor: 'release', + platform: 'android', + arch: 'arm64', + cli_version: version, + }) + expect(byArch['ios-arm64'].platform).toBe('ios') + + const [, options] = uploadImplementation.mock.calls[0] + expect(options).toMatchObject({retries: 5, useGzip: true}) + }) + + test('dry-run reports files without uploading', async () => { + const dir = createSymbolsFixture(['app.android-arm.symbols']) + const uploadImplementation = jest.fn().mockResolvedValue(UploadStatus.Success) + mockedUpload.mockReturnValue(uploadImplementation) + + const {code, context} = await runCLI([ + 'flutter-symbols', + 'upload', + dir, + '--service', + 'my-app', + '--release-version', + '1.2.3', + '--dry-run', + ]) + + expect(code).toBe(0) + expect(uploadImplementation).not.toHaveBeenCalled() + expect(context.stdout.toString()).toContain('[DRYRUN]') + expect(context.stdout.toString()).toContain('platform=android') + }) + + test('ignores non-symbol files and errors when none found', async () => { + const dir = createSymbolsFixture(['not-a-symbol.txt', 'app.web-x64.symbols']) + + const {code, context} = await runCLI([ + 'flutter-symbols', + 'upload', + dir, + '--service', + 'my-app', + '--release-version', + '1.2.3', + ]) + + expect(code).toBe(1) + expect(context.stderr.toString()).toContain('No app.-.symbols files found') + expect(mockedUpload).not.toHaveBeenCalled() + }) + + test('requires --service and --release-version', async () => { + const dir = createSymbolsFixture(['app.android-arm64.symbols']) + + const {code} = await runCLI(['flutter-symbols', 'upload', dir]) + expect(code).toBe(1) + }) +}) + +const runCLI = async (args: string[]) => { + const cli = new Cli() + cli.register(UploadCommand) + const context = createMockContext() as any + const code = await cli.run(args, context) + + return {code, context} +} + +const createSymbolsFixture = (names: string[]) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flashcat-flutter-test-')) + for (const name of names) { + fs.writeFileSync(path.join(dir, name), 'symbol-content') + } + + return dir +} + +const createMockContext = () => { + let stdout = '' + let stderr = '' + + return { + stderr: { + toString: () => stderr, + write: (input: string) => { + stderr += input + }, + }, + stdout: { + toString: () => stdout, + write: (input: string) => { + stdout += input + }, + }, + } +} diff --git a/src/commands/flutter-symbols/cli.ts b/src/commands/flutter-symbols/cli.ts new file mode 100644 index 0000000..1a0522e --- /dev/null +++ b/src/commands/flutter-symbols/cli.ts @@ -0,0 +1,3 @@ +import {UploadCommand} from './upload' + +module.exports = [UploadCommand] diff --git a/src/commands/flutter-symbols/upload.ts b/src/commands/flutter-symbols/upload.ts new file mode 100644 index 0000000..ec9ff40 --- /dev/null +++ b/src/commands/flutter-symbols/upload.ts @@ -0,0 +1,213 @@ +import path from 'path' + +import chalk from 'chalk' +import {Command, Option} from 'clipanion' +import {glob} from 'glob' + +import {ApiKeyValidator, newApiKeyValidator} from '../../helpers/apikey' +import {getBaseSourcemapIntakeUrl} from '../../helpers/base-intake-url' +import {doWithMaxConcurrency} from '../../helpers/concurrency' +import {InvalidConfigurationError} from '../../helpers/errors' +import {RequestBuilder} from '../../helpers/interfaces' +import {MultipartPayload, MultipartValue, UploadStatus, upload} from '../../helpers/upload' +import {buildPath, getRequestBuilder} from '../../helpers/utils' +import {version} from '../../helpers/version' + +// A Flutter symbol file is named app.-.symbols, produced by +// `flutter build --obfuscate --split-debug-info=` (one per built architecture). +const SYMBOL_FILE_RE = /^app\.(android|ios)-(arm|arm64|x64)\.symbols$/ + +interface FlutterSymbolFile { + path: string + platform: string + arch: string +} + +export class UploadCommand extends Command { + public static paths = [['flutter-symbols', 'upload']] + + public static usage = Command.Usage({ + category: 'RUM', + description: 'Upload Flutter (Dart AOT) debug symbols to Flashcat for crash de-obfuscation.', + details: ` + Point this at the --split-debug-info directory of an obfuscated release build. + It uploads every app.-.symbols file found there; the server extracts + each file's build-id and symbolicates matching obfuscated stacks at query time. + + Build with: + flutter build apk --obfuscate --split-debug-info= + flutter build ipa --obfuscate --split-debug-info= + `, + examples: [ + [ + 'Upload Flutter symbols for a release', + 'flashcat-cli flutter-symbols upload ./debug-symbols --service my-app --release-version 1.2.3', + ], + ], + }) + + private basePath = Option.String({required: true}) + private service = Option.String('--service') + private releaseVersion = Option.String('--release-version') + private flavor = Option.String('--flavor', 'release') + private maxConcurrency = Option.String('--max-concurrency', '20') + private dryRun = Option.Boolean('--dry-run', false) + private quiet = Option.Boolean('--quiet', false) + + private cliVersion = version + + private config = { + apiKey: process.env.FLASHCAT_API_KEY, + flashcatSite: process.env.FLASHCAT_SITE || 'flashcat.cloud', + } + + public async execute() { + if (!this.service) { + this.context.stderr.write('Missing --service\n') + + return 1 + } + if (!this.releaseVersion) { + this.context.stderr.write('Missing --release-version\n') + + return 1 + } + + const files = this.findSymbolFiles() + if (files.length === 0) { + this.context.stderr.write( + `No app.-.symbols files found under ${path.resolve(this.basePath)}. ` + + `Build with --obfuscate --split-debug-info= and point at that dir.\n` + ) + + return 1 + } + + if (this.dryRun) { + for (const f of files) { + this.context.stdout.write( + `[DRYRUN] would upload ${f.path} (platform=${f.platform}, arch=${f.arch}, service=${this.service}, version=${this.releaseVersion})\n` + ) + } + + return 0 + } + + const apiKeyValidator = newApiKeyValidator({ + apiKey: this.config.apiKey, + flashcatSite: this.config.flashcatSite, + }) + const requestBuilder = this.getRequestBuilder() + + try { + const results = await doWithMaxConcurrency(Math.max(1, parseInt(this.maxConcurrency, 10) || 20), files, (f) => + this.uploadSymbolFile(requestBuilder, apiKeyValidator)(f) + ) + + if (results.some((r) => r !== UploadStatus.Success)) { + return 1 + } + + if (!this.quiet) { + this.context.stdout.write(`Uploaded ${files.length} Flutter symbol file(s).\n`) + } + + return 0 + } catch (error) { + if (error instanceof InvalidConfigurationError) { + this.context.stderr.write(`${error.message}\n`) + + return 1 + } + + throw error + } + } + + private findSymbolFiles(): FlutterSymbolFile[] { + const matches = glob.sync(buildPath(this.basePath, '**/app.*.symbols')) + + return matches.reduce((acc, filePath) => { + const m = SYMBOL_FILE_RE.exec(path.basename(filePath)) + if (m) { + acc.push({path: filePath, platform: m[1], arch: m[2]}) + } + + return acc + }, []) + } + + private asMultipartPayload(file: FlutterSymbolFile): MultipartPayload { + const eventMeta: Record = { + type: 'flutter_symbol_file', + service: this.service!, + version: this.releaseVersion!, + flavor: this.flavor, + platform: file.platform, + arch: file.arch, + cli_version: this.cliVersion, + } + + const content = new Map([ + [ + 'event', + { + type: 'string', + options: {contentType: 'application/json'}, + value: JSON.stringify(eventMeta), + }, + ], + [ + 'flutter_symbol_file', + { + type: 'file', + path: file.path, + options: {filename: path.basename(file.path)}, + }, + ], + ]) + + return {content} + } + + private getRequestBuilder(): RequestBuilder { + if (!this.config.apiKey) { + throw new InvalidConfigurationError(`Missing ${chalk.bold('FLASHCAT_API_KEY')} in your environment.`) + } + + return getRequestBuilder({ + apiKey: this.config.apiKey, + baseUrl: getBaseSourcemapIntakeUrl(this.config.flashcatSite), + headers: new Map([ + ['DD-EVP-ORIGIN', 'flashcat-cli_flutter'], + ['DD-EVP-ORIGIN-VERSION', this.cliVersion], + ]), + overrideUrl: '/sourcemap/upload', + }) + } + + private uploadSymbolFile( + requestBuilder: RequestBuilder, + apiKeyValidator: ApiKeyValidator + ): (file: FlutterSymbolFile) => Promise { + return async (file: FlutterSymbolFile) => + upload(requestBuilder)(this.asMultipartPayload(file), { + apiKeyValidator, + onError: (e) => { + this.context.stderr.write(`Upload failed for ${path.basename(file.path)}: ${e.message}\n`) + }, + onRetry: (e, attempts) => { + if (!this.quiet) { + this.context.stdout.write(`Retry ${attempts} for ${path.basename(file.path)}: ${e.message}\n`) + } + }, + onUpload: () => { + if (!this.quiet) { + this.context.stdout.write(`Uploading ${path.basename(file.path)} (${file.platform}/${file.arch})...\n`) + } + }, + retries: 5, + useGzip: true, + }) + } +} diff --git a/src/commands/sourcemaps/__tests__/upload.test.ts b/src/commands/sourcemaps/__tests__/upload.test.ts index fd1ee5d..239e702 100644 --- a/src/commands/sourcemaps/__tests__/upload.test.ts +++ b/src/commands/sourcemaps/__tests__/upload.test.ts @@ -43,6 +43,30 @@ describe('upload', () => { }) }) + describe('getMinifiedURL: minified file path uses Windows separators', () => { + test('should return URL paths with forward slashes', () => { + const command = new UploadCommand() + command['basePath'] = 'dist' + command['minifiedPathPrefix'] = '/dist' + expect(command['getMinifiedURLAndRelativePath']('dist\\js\\common.min.js')).toStrictEqual([ + '/dist/js/common.min.js', + '/js/common.min.js', + ]) + }) + }) + + describe('getMinifiedURL: minifiedPathPrefix uses Windows separators', () => { + test('should return URL paths with forward slashes', () => { + const command = new UploadCommand() + command['basePath'] = 'dist' + command['minifiedPathPrefix'] = '\\dist' + expect(command['getMinifiedURLAndRelativePath']('dist\\js\\common.min.js')).toStrictEqual([ + '/dist/js/common.min.js', + '/js/common.min.js', + ]) + }) + }) + describe('isMinifiedPathPrefixValid: full URL', () => { test('should return true', () => { const command = new UploadCommand() @@ -70,6 +94,15 @@ describe('upload', () => { }) }) + describe('isMinifiedPathPrefixValid: leading Windows separator', () => { + test('should return true', () => { + const command = new UploadCommand() + command['minifiedPathPrefix'] = '\\js' + + expect(command['isMinifiedPathPrefixValid']()).toBe(true) + }) + }) + describe('isMinifiedPathPrefixValid: no leading slash', () => { test('should return false', () => { const command = new UploadCommand() diff --git a/src/commands/sourcemaps/upload.ts b/src/commands/sourcemaps/upload.ts index 7c73901..ce75d95 100644 --- a/src/commands/sourcemaps/upload.ts +++ b/src/commands/sourcemaps/upload.ts @@ -97,6 +97,7 @@ export class UploadCommand extends Command { return 1 } + this.minifiedPathPrefix = this.getNormalizedMinifiedPathPrefix() // Normalizing the basePath to resolve .. and . // Always using the posix version to avoid \ on Windows. @@ -174,9 +175,11 @@ export class UploadCommand extends Command { } private getMinifiedURLAndRelativePath(minifiedFilePath: string): [string, string] { - const relativePath = minifiedFilePath.replace(this.basePath, '') + const normalizedMinifiedFilePath = minifiedFilePath.replace(/\\/g, '/') + const normalizedBasePath = this.basePath.replace(/\\/g, '/') + const relativePath = normalizedMinifiedFilePath.replace(normalizedBasePath, '') - return [buildPath(this.minifiedPathPrefix!, relativePath), relativePath] + return [buildPath(this.getNormalizedMinifiedPathPrefix(), relativePath), relativePath] } private getPayloadsToUpload = async (useGit: boolean): Promise => { @@ -240,20 +243,25 @@ export class UploadCommand extends Command { private isMinifiedPathPrefixValid(): boolean { let host + const minifiedPathPrefix = this.getNormalizedMinifiedPathPrefix() try { - const objUrl = new URL(this.minifiedPathPrefix!) + const objUrl = new URL(minifiedPathPrefix) host = objUrl.host } catch { // Do nothing. } - if (!host && !this.minifiedPathPrefix!.startsWith('/')) { + if (!host && !minifiedPathPrefix.startsWith('/')) { return false } return true } + private getNormalizedMinifiedPathPrefix(): string { + return this.minifiedPathPrefix!.replace(/\\/g, '/') + } + private upload( requestBuilder: RequestBuilder, apiKeyValidator: ApiKeyValidator diff --git a/src/helpers/__tests__/apikey.test.ts b/src/helpers/__tests__/apikey.test.ts new file mode 100644 index 0000000..62387e3 --- /dev/null +++ b/src/helpers/__tests__/apikey.test.ts @@ -0,0 +1,34 @@ +import {AxiosError} from 'axios' + +import {InvalidConfigurationError} from '../errors' +import {newApiKeyValidator} from '../apikey' + +describe('ApiKeyValidator', () => { + test('does not report invalid parameter responses as invalid API keys', async () => { + const validator = newApiKeyValidator({apiKey: 'api-key', flashcatSite: 'flashcat.cloud'}) + const error = { + response: { + status: 400, + data: { + error: { + code: 'InvalidParameter', + message: 'Invalid Minified URL: contains dangerous path characters', + }, + }, + }, + } as AxiosError + + await expect(validator.verifyApiKey(error)).resolves.toBeUndefined() + }) + + test('reports unauthorized responses as invalid API keys', async () => { + const validator = newApiKeyValidator({apiKey: 'api-key', flashcatSite: 'flashcat.cloud'}) + const error = { + response: { + status: 401, + }, + } as AxiosError + + await expect(validator.verifyApiKey(error)).rejects.toBeInstanceOf(InvalidConfigurationError) + }) +}) diff --git a/src/helpers/apikey.ts b/src/helpers/apikey.ts index 35c28d4..92555b4 100644 --- a/src/helpers/apikey.ts +++ b/src/helpers/apikey.ts @@ -41,7 +41,7 @@ class ApiKeyValidatorImplem { if (error.response === undefined) { return; } - if (error.response.status === 403 || error.response.status === 400) { + if (error.response.status === 401 || error.response.status === 403) { throw new InvalidConfigurationError( `${chalk.red.bold( "FLASHCAT_API_KEY"