vip app deploy - Flag large archive files included in BYOR deploy

__tests__/bin/vip-app-deploy.js

@@ -3,6 +3,7 @@ import * as exit from '../../src/lib/cli/exit';
 import { uploadImportFileToS3 } from '../../src/lib/client-file-uploader';
 import {
 	validateFile,
+	validateLargeArchiveFiles,
 	promptToContinue,
 	validateCustomDeployKey,
 } from '../../src/lib/custom-deploy/custom-deploy';
@@ -18,6 +19,7 @@ jest.mock( '../../src/lib/client-file-uploader', () => ( {
 
 jest.mock( '../../src/lib/custom-deploy/custom-deploy', () => ( {
 	validateFile: jest.fn(),
+	validateLargeArchiveFiles: jest.fn(),
 	renameFile: jest.fn(),
 	promptToContinue: jest.fn().mockResolvedValue( true ),
 	validateCustomDeployKey: jest.fn().mockResolvedValue( {
@@ -96,16 +98,29 @@ describe( 'vip-app-deploy', () => {
 	} );
 
 	describe( 'appDeployCmd', () => {
+		beforeEach( () => {
+			jest.clearAllMocks();
+		} );
+
 		it( 'should call expected functions', async () => {
 			await appDeployCmd( args, opts );
 
 			expect( validateCustomDeployKey ).toHaveBeenCalledTimes( 1 );
 
 			expect( validateFile ).toHaveBeenCalledTimes( 1 );
 
+			expect( validateLargeArchiveFiles ).toHaveBeenCalledTimes( 1 );
+
 			expect( promptToContinue ).not.toHaveBeenCalled();
 
 			expect( uploadImportFileToS3 ).toHaveBeenCalledTimes( 1 );
 		} );
+
+		it( 'skips large file verification when requested', async () => {
+			await appDeployCmd( args, { ...opts, skipLargeFileValidation: true } );
+
+			expect( validateFile ).toHaveBeenCalledTimes( 1 );
+			expect( validateLargeArchiveFiles ).not.toHaveBeenCalled();
+		} );
 	} );
 } );

__tests__/lib/custom-deploy/custom-deploy.js

@@ -0,0 +1,80 @@
+import * as exit from '../../../src/lib/cli/exit';
+import { validateLargeArchiveFiles } from '../../../src/lib/custom-deploy/custom-deploy';
+import { trackEventWithEnv } from '../../../src/lib/tracker';
+import { findLargeArchiveFilesInDeployArchive } from '../../../src/lib/validations/custom-deploy';
+
+jest.mock( '../../../src/lib/validations/custom-deploy', () => ( {
+	...jest.requireActual( '../../../src/lib/validations/custom-deploy' ),
+	findLargeArchiveFilesInDeployArchive: jest.fn(),
+} ) );
+
+jest.mock( '../../../src/lib/tracker', () => ( {
+	trackEventWithEnv: jest.fn().mockResolvedValue( [] ),
+} ) );
+
+const exitSpy = jest.spyOn( exit, 'withError' );
+jest.spyOn( process, 'exit' ).mockImplementation( () => {} );
+jest.spyOn( console, 'error' ).mockImplementation( () => {} );
+jest.spyOn( console, 'log' ).mockImplementation( () => {} );
+
+describe( 'custom deploy large archive file validation', () => {
+	beforeEach( () => {
+		jest.clearAllMocks();
+	} );
+
+	it( 'passes when the deploy archive has no large archive files', async () => {
+		findLargeArchiveFilesInDeployArchive.mockResolvedValue( [] );
+
+		await validateLargeArchiveFiles( 123, 456, {
+			fileName: '/vip/skeleton.zip',
+			basename: 'skeleton.zip',
+		} );
+
+		expect( exitSpy ).not.toHaveBeenCalled();
+		expect( trackEventWithEnv ).not.toHaveBeenCalled();
+	} );
+
+	it( 'exits when the deploy archive has archive files over 20 MB', async () => {
+		findLargeArchiveFilesInDeployArchive.mockResolvedValue( [
+			{
+				path: 'mysite/plugins/big-plugin.tar.gz',
+				size: 21 * 1024 * 1024,
+			},
+		] );
+
+		await validateLargeArchiveFiles( 123, 456, {
+			fileName: '/vip/skeleton.zip',
+			basename: 'skeleton.zip',
+		} );
+
+		expect( trackEventWithEnv ).toHaveBeenCalledWith( 123, 456, 'deploy_app_command_error', {
+			error_type: 'large-archive-files',
+			large_archive_files: [ 'mysite/plugins/big-plugin.tar.gz' ],
+		} );
+		expect( exitSpy.mock.calls[ 0 ][ 0 ] ).toContain(
+			'Deploy archive contains archive file(s) larger than 20.0 MB in the deploy archive'
+		);
+		expect( exitSpy.mock.calls[ 0 ][ 0 ] ).toContain(
+			'mysite/plugins/big-plugin.tar.gz (21.0 MB)'
+		);
+		expect( exitSpy.mock.calls[ 0 ][ 0 ] ).toContain( '--skip-large-file-validation' );
+	} );
+
+	it( 'exits with skip instructions when large file verification fails', async () => {
+		findLargeArchiveFilesInDeployArchive.mockRejectedValue( new Error( 'invalid archive' ) );
+
+		await validateLargeArchiveFiles( 123, 456, {
+			fileName: '/vip/skeleton.zip',
+			basename: 'skeleton.zip',
+		} );
+
+		expect( trackEventWithEnv ).toHaveBeenCalledWith( 123, 456, 'deploy_app_command_error', {
+			error_type: 'large-archive-file-verify-failed',
+			verify_error: 'invalid archive',
+		} );
+		expect( exitSpy.mock.calls[ 0 ][ 0 ] ).toContain(
+			'Unable to verify large archive files in the deploy archive: invalid archive'
+		);
+		expect( exitSpy.mock.calls[ 0 ][ 0 ] ).toContain( '--skip-large-file-validation' );
+	} );
+} );

src/bin/vip-app-deploy.ts

@@ -23,7 +23,11 @@ import {
 	WithId,
 	UploadArguments,
 } from '../lib/client-file-uploader';
-import { validateCustomDeployKey, validateFile } from '../lib/custom-deploy/custom-deploy';
+import {
+	validateCustomDeployKey,
+	validateFile,
+	validateLargeArchiveFiles,
+} from '../lib/custom-deploy/custom-deploy';
 import { trackEventWithEnv } from '../lib/tracker';
 
 const START_DEPLOY_MUTATION = gql`
@@ -95,6 +99,10 @@ export async function appDeployCmd( arg: string[] = [], opts: Record< string, un
 	debug( 'Validating file...' );
 	await validateFile( appId, envId, fileMeta );
 
+	if ( ! opts.skipLargeFileValidation ) {
+		await validateLargeArchiveFiles( appId, envId, fileMeta );
+	}
+
 	await track( 'deploy_app_command_execute' );
 
 	// Upload file as different name to avoid overwriting existing same named files
@@ -269,6 +277,11 @@ const examples = [
 		description:
 			'Skip the confirmation prompt for the Custom Deployment of the archived file named "file.tar.gz" to the environment.',
 	},
+	{
+		usage:
+			'WPVIP_DEPLOY_TOKEN=1234 vip @example-app.develop app deploy file.zip --skip-large-file-validation',
+		description: 'Skip checking the deploy archive for repository archive files larger than 20 MB.',
+	},
 ];
 
 void command( {
@@ -279,6 +292,11 @@ void command( {
 	.examples( examples )
 	.option( 'message', 'Add a description of a deployment.' )
 	.option( 'skip-confirmation', 'Skip the confirmation prompt.' )
+	.option(
+		'skip-large-file-validation',
+		'Skip checking for repository archive files over 20 MB.',
+		false
+	)
 	.option( 'force', 'Skip confirmation prompt (deprecated)' )
 	.option(
 		'app',

src/lib/custom-deploy/custom-deploy.ts

@@ -1,3 +1,4 @@
+import chalk from 'chalk';
 import fs from 'fs';
 import gql from 'graphql-tag';
 
@@ -6,10 +7,18 @@ import * as exit from '../../lib/cli/exit';
 import { checkFileAccess, getFileSize, isFile, FileMeta } from '../../lib/client-file-uploader';
 import { GB_IN_BYTES } from '../../lib/constants/file-size';
 import { trackEventWithEnv } from '../../lib/tracker';
-import { validateDeployFileExt, validateFilename } from '../../lib/validations/custom-deploy';
+import {
+	findLargeArchiveFilesInDeployArchive,
+	LARGE_ARCHIVE_FILE_SIZE_LIMIT,
+	validateDeployFileExt,
+	validateFilename,
+} from '../../lib/validations/custom-deploy';
+
+import type { LargeArchiveFile } from '../../lib/validations/custom-deploy';
 
 const DEPLOY_MAX_FILE_SIZE = 4 * GB_IN_BYTES;
 const WPVIP_DEPLOY_TOKEN = process.env.WPVIP_DEPLOY_TOKEN;
+const SKIP_LARGE_FILE_VALIDATION_FLAG = '--skip-large-file-validation';
 
 type CustomDeployInfo = {
 	success: boolean;
@@ -25,6 +34,22 @@ type ValidateMutationPayload = {
 	validateCustomDeployAccess: CustomDeployInfo;
 };
 
+function formatFileSize( bytes: number ): string {
+	return `${ ( bytes / ( 1024 * 1024 ) ).toFixed( 1 ) } MB`;
+}
+
+function getLargeArchiveFilesMessage( largeArchiveFiles: LargeArchiveFile[] ): string {
+	const fileList = largeArchiveFiles
+		.map( file => `- ${ file.path } (${ formatFileSize( file.size ) })` )
+		.join( '\n' );
+
+	return `Deploy archive contains archive file(s) larger than ${ formatFileSize(
+		LARGE_ARCHIVE_FILE_SIZE_LIMIT
+	) } in the deploy archive:\n${ fileList }\nRemove these files from the repository, or rerun with ${ chalk.bold(
+		SKIP_LARGE_FILE_VALIDATION_FLAG
+	) } to skip this check.`;
+}
+
 export async function validateCustomDeployKey(
 	app: string | number,
 	env: string | number
@@ -127,3 +152,43 @@ export async function validateFile( appId: number, envId: number, fileMeta: File
 		);
 	}
 }
+
+/**
+ * @param {number} appId
+ * @param {number} envId
+ * @param {FileMeta} fileMeta
+ */
+export async function validateLargeArchiveFiles(
+	appId: number,
+	envId: number,
+	fileMeta: FileMeta
+) {
+	const track = trackEventWithEnv.bind( null, appId, envId );
+	let largeArchiveFiles: LargeArchiveFile[];
+
+	try {
+		largeArchiveFiles = await findLargeArchiveFilesInDeployArchive( fileMeta.fileName );
+	} catch ( error ) {
+		await track( 'deploy_app_command_error', {
+			error_type: 'large-archive-file-verify-failed',
+			verify_error: ( error as Error ).message,
+		} );
+
+		return exit.withError(
+			`Unable to verify large archive files in the deploy archive: ${
+				( error as Error ).message
+			}. Rerun with ${ SKIP_LARGE_FILE_VALIDATION_FLAG } to skip this check.`
+		);
+	}
+
+	if ( ! largeArchiveFiles.length ) {
+		return;
+	}
+
+	await track( 'deploy_app_command_error', {
+		error_type: 'large-archive-files',
+		large_archive_files: largeArchiveFiles.map( file => file.path ),
+	} );
+
+	return exit.withError( getLargeArchiveFilesMessage( largeArchiveFiles ) );
+}

src/lib/validations/custom-deploy.ts

@@ -4,11 +4,18 @@ import path from 'path';
 import * as tar from 'tar';
 
 import * as exit from '../../lib/cli/exit';
+import { MB_IN_BYTES } from '../../lib/constants/file-size';
 
 interface TarEntry {
 	path: string;
 	type: string;
 	mode: number | undefined;
+	size?: number;
+}
+
+export interface LargeArchiveFile {
+	path: string;
+	size: number;
 }
 
 const errorMessages = {
@@ -21,6 +28,93 @@ const errorMessages = {
 };
 const symlinkIgnorePattern = /\/node_modules\/[^/]+\/\.bin\//;
 const macosxDir = '__MACOSX';
+export const LARGE_ARCHIVE_FILE_SIZE_LIMIT = 20 * MB_IN_BYTES;
+
+function getDeployFileExt( filename: string ): string {
+	const lower = filename.toLowerCase();
+
+	if ( lower.endsWith( '.tar.gz' ) ) {
+		return '.tar.gz';
+	}
+
+	if ( lower.endsWith( '.tgz' ) ) {
+		return '.tgz';
+	}
+
+	if ( lower.endsWith( '.zip' ) ) {
+		return '.zip';
+	}
+
+	return path.extname( lower );
+}
+
+function isDeployArchiveFile( filename: string ): boolean {
+	return [ '.zip', '.tar.gz', '.tgz' ].includes( getDeployFileExt( filename ) );
+}
+
+async function findLargeArchiveFilesInZipArchive(
+	filePath: string
+): Promise< LargeArchiveFile[] > {
+	const zipFile = new StreamZip.async( { file: filePath } );
+
+	try {
+		const zipEntries = await zipFile.entries();
+
+		return Object.values( zipEntries )
+			.filter(
+				entry =>
+					! entry.isDirectory &&
+					isDeployArchiveFile( entry.name ) &&
+					entry.size > LARGE_ARCHIVE_FILE_SIZE_LIMIT
+			)
+			.map( entry => ( {
+				path: entry.name,
+				size: entry.size,
+			} ) );
+	} finally {
+		await zipFile.close();
+	}
+}
+
+async function findLargeArchiveFilesInTarArchive(
+	filePath: string
+): Promise< LargeArchiveFile[] > {
+	const largeArchiveFiles: LargeArchiveFile[] = [];
+
+	await tar.list( {
+		file: filePath,
+		onReadEntry: entry => {
+			if (
+				entry.type === 'File' &&
+				isDeployArchiveFile( entry.path ) &&
+				entry.size > LARGE_ARCHIVE_FILE_SIZE_LIMIT
+			) {
+				largeArchiveFiles.push( {
+					path: entry.path,
+					size: entry.size,
+				} );
+			}
+		},
+	} );
+
+	return largeArchiveFiles;
+}
+
+export async function findLargeArchiveFilesInDeployArchive(
+	filePath: string
+): Promise< LargeArchiveFile[] > {
+	const ext = getDeployFileExt( filePath );
+
+	if ( ext === '.zip' ) {
+		return findLargeArchiveFilesInZipArchive( filePath );
+	}
+
+	if ( ext === '.tar.gz' || ext === '.tgz' ) {
+		return findLargeArchiveFilesInTarArchive( filePath );
+	}
+
+	return [];
+}
 
 /**
  * Check if a file has a valid extension
@@ -29,11 +123,7 @@ const macosxDir = '__MACOSX';
  * @returns {boolean} True if the extension is valid
  */
 export function validateDeployFileExt( filename: string ): void {
-	let ext = path.extname( filename ).toLowerCase();
-
-	if ( ext === '.gz' && path.extname( path.basename( filename, ext ) ) === '.tar' ) {
-		ext = '.tar.gz';
-	}
+	const ext = getDeployFileExt( filename );
 
 	if ( ! [ '.zip', '.tar.gz', '.tgz' ].includes( ext ) ) {
 		exit.withError( errorMessages.invalidExt );