diff --git a/__tests__/devenv-e2e/014-database-defaults.spec.js b/__tests__/devenv-e2e/014-database-defaults.spec.js new file mode 100644 index 000000000..c09d31a3a --- /dev/null +++ b/__tests__/devenv-e2e/014-database-defaults.spec.js @@ -0,0 +1,195 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import Docker from 'dockerode'; +import nock from 'nock'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { CliTest } from './helpers/cli-test'; +import { vipDevEnvExec, vipDevEnvStart } from './helpers/commands'; +import { killProjectContainers } from './helpers/docker-utils'; +import { + createAndStartEnvironment, + destroyEnvironment, + getProjectSlug, + prepareEnvironment, +} from './helpers/utils'; + +jest.setTimeout( 600 * 1000 ).retryTimes( 1, { logErrorsBeforeRetry: true } ); + +// Stock server defaults that the template deliberately overrides. Values below are +// asserted as "not stock" floors rather than exact matches, so future tuning of the +// template (e.g. shrinking the buffer pool) does not break this test; only *losing* +// an override (falling back to the stock default) fails it. +const STOCK_BUFFER_POOL_SIZE = 134217728; // 128M +const STOCK_REDO_LOG_CAPACITY = 104857600; // ~100M + +const OLD_VERSION = '2.3.3'; +const OLD_DB_COMMAND = + 'docker-entrypoint.sh mysqld --sql-mode=ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION --max_allowed_packet=67M --mysql-native-password=ON'; + +describe( 'dev-env database performance defaults', () => { + /** @type {CliTest} */ + let cliTest; + /** @type {NodeJS.ProcessEnv} */ + let env; + /** @type {string} */ + let tmpPath; + /** @type {Docker} */ + let docker; + /** @type {string} */ + let slug; + + const dbQuery = async query => { + const result = await cliTest.spawn( + [ + process.argv[ 0 ], + vipDevEnvExec, + '--slug', + slug, + '--quiet', + '--', + 'wp', + 'db', + 'query', + query, + '--skip-column-names', + ], + { env }, + true + ); + expect( result.rc ).toBe( 0 ); + return result.stdout.trim(); + }; + + const assertTunedDatabaseDefaults = async () => { + const row = await dbQuery( + 'SELECT @@log_bin, @@innodb_buffer_pool_size, @@innodb_redo_log_capacity, @@innodb_flush_log_at_trx_commit' + ); + const [ logBin, bufferPoolSize, redoLogCapacity, flushMode ] = row.split( /\s+/ ).map( Number ); + + // Binary logging must be off: nothing consumes binlogs locally and they + // double the write volume of large imports. + expect( logBin ).toBe( 0 ); + // Must be configured above the stock defaults; exact values are a tuning choice. + expect( bufferPoolSize ).toBeGreaterThan( STOCK_BUFFER_POOL_SIZE ); + expect( redoLogCapacity ).toBeGreaterThan( STOCK_REDO_LOG_CAPACITY ); + // Anything but fsync-per-commit (1) preserves the bulk-write intent. + expect( flushMode ).not.toBe( 1 ); + }; + + beforeAll( async () => { + nock.cleanAll(); + nock.enableNetConnect(); + + cliTest = new CliTest(); + + tmpPath = await mkdtemp( path.join( os.tmpdir(), 'vip-dev-env-' ) ); + process.env.XDG_DATA_HOME = tmpPath; + + env = prepareEnvironment( tmpPath ); + docker = new Docker(); + + slug = getProjectSlug(); + await createAndStartEnvironment( cliTest, slug, env ); + } ); + + afterAll( async () => { + try { + await destroyEnvironment( cliTest, slug, env ); + } finally { + await killProjectContainers( docker, slug ); + } + } ); + + afterAll( () => rm( tmpPath, { recursive: true, force: true } ) ); + afterAll( () => nock.restore() ); + + // eslint-disable-next-line jest/expect-expect -- assertions live in assertTunedDatabaseDefaults + it( 'should run the database with tuned (non-stock) defaults', async () => { + await assertTunedDatabaseDefaults(); + } ); + + it( 'should apply the tuned defaults to an environment created by an older CLI', async () => { + // Simulate an environment created before the defaults changed: stamp the + // pre-tuning version and restore the old database command line. + const instanceDataPath = path.join( + tmpPath, + 'vip', + 'dev-environment', + slug, + 'instance_data.json' + ); + const instanceData = JSON.parse( await readFile( instanceDataPath, 'utf8' ) ); + expect( instanceData.version ).not.toBe( OLD_VERSION ); + instanceData.version = OLD_VERSION; + await writeFile( instanceDataPath, JSON.stringify( instanceData, null, 2 ) ); + + const landoFilePath = path.join( tmpPath, 'vip', 'dev-environment', slug, '.lando.yml' ); + const landoFile = await readFile( landoFilePath, 'utf8' ); + const oldLandoFile = landoFile.replace( + /command: docker-entrypoint\.sh mysqld[^\n]*/, + `command: ${ OLD_DB_COMMAND }` + ); + expect( oldLandoFile ).not.toBe( landoFile ); + await writeFile( landoFilePath, oldLandoFile ); + + // Plant data that must survive the upgrade rebuild. + const marker = await cliTest.spawn( + [ + process.argv[ 0 ], + vipDevEnvExec, + '--slug', + slug, + '--quiet', + '--', + 'wp', + 'option', + 'add', + 'upgrade_e2e_marker', + 'survived', + ], + { env }, + true + ); + expect( marker.rc ).toBe( 0 ); + + // Starting with the current CLI must detect the old version, re-render the + // template, and rebuild the environment without prompting. + const result = await cliTest.spawn( + [ process.argv[ 0 ], vipDevEnvStart, '--slug', slug, '-w' ], + { env }, + true + ); + expect( result.rc ).toBe( 0 ); + expect( result.stdout ).toContain( `Current local environment version is: ${ OLD_VERSION }` ); + expect( result.stdout ).toContain( 'Local environment version updated to:' ); + expect( result.stdout ).toMatch( /STATUS\s+UP/u ); + + const updatedInstanceData = JSON.parse( await readFile( instanceDataPath, 'utf8' ) ); + expect( updatedInstanceData.version ).not.toBe( OLD_VERSION ); + + // The tuned defaults apply to the pre-existing data directory... + await assertTunedDatabaseDefaults(); + + // ...and the data survived the transition. + const markerValue = await cliTest.spawn( + [ + process.argv[ 0 ], + vipDevEnvExec, + '--slug', + slug, + '--quiet', + '--', + 'wp', + 'option', + 'get', + 'upgrade_e2e_marker', + ], + { env }, + true + ); + expect( markerValue.rc ).toBe( 0 ); + expect( markerValue.stdout.trim() ).toBe( 'survived' ); + } ); +} ); diff --git a/assets/dev-env.lando.template.yml.ejs b/assets/dev-env.lando.template.yml.ejs index 2b3824f1c..a3fde4ddd 100644 --- a/assets/dev-env.lando.template.yml.ejs +++ b/assets/dev-env.lando.template.yml.ejs @@ -103,12 +103,19 @@ services: database: type: compose services: + # Bulk-import friendly settings (local dev only, durability is not a concern): + # - skip-log-bin (MySQL): binary logging is on by default in MySQL 8 and doubles the + # write volume of large SQL imports for no benefit locally. MariaDB has it off by default. + # - innodb-buffer-pool-size: the 128M default forces constant flushing on multi-GB imports. + # - innodb-redo-log-capacity (MySQL) / innodb-log-file-size (MariaDB): the ~100M default + # causes "Redo log writer is waiting for a new redo log file" stalls during imports. + # - innodb-flush-log-at-trx-commit=2: no fsync per commit; at most ~1s of writes lost on crash. <% if ( mariadb ) { %> image: mariadb:<%= mariadb %> - command: docker-entrypoint.sh mysqld --sql-mode=ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION --max_allowed_packet=67M + command: docker-entrypoint.sh mysqld --sql-mode=ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION --max_allowed_packet=67M --innodb-buffer-pool-size=1G --innodb-log-file-size=1G --innodb-flush-log-at-trx-commit=2 <% } else { %> image: mysql:8.4 - command: docker-entrypoint.sh mysqld --sql-mode=ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION --max_allowed_packet=67M --mysql-native-password=ON + command: docker-entrypoint.sh mysqld --sql-mode=ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION --max_allowed_packet=67M --mysql-native-password=ON --skip-log-bin --innodb-buffer-pool-size=1G --innodb-redo-log-capacity=1G --innodb-flush-log-at-trx-commit=2 <% } %> ports: - ":3306" diff --git a/src/lib/constants/dev-environment.ts b/src/lib/constants/dev-environment.ts index f99c18b0c..e75dfa932 100644 --- a/src/lib/constants/dev-environment.ts +++ b/src/lib/constants/dev-environment.ts @@ -52,4 +52,4 @@ export const DEV_ENVIRONMENT_DEFAULTS = { phpVersion: Object.keys( DEV_ENVIRONMENT_PHP_VERSIONS )[ 0 ], } as const; -export const DEV_ENVIRONMENT_VERSION = '2.3.3'; +export const DEV_ENVIRONMENT_VERSION = '2.3.4';