From f26e5514338a745d4e900efb8576361f7d142404 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Wed, 18 Nov 2026 08:06:19 +0530 Subject: [PATCH 1/2] fix(migration): serialise migrations with a lock Two ee processes could run the migration pipeline at the same time, for example the midnight `ee site ssl-renew --all` cron and a user command right after a phar swap, or two concurrent nightly commands. Both stopped and recreated the same containers. During `ee cli update`, the downloaded phar migrates while the old phar and the old DB version are still in place, so a command started meanwhile skipped migrating and ran in the middle of the upgrade. maybe_trigger_migration() now runs under an flock() on EE_ROOT_DIR/db/migration.lock (root-only, close-on-exec, released by the kernel when the process exits). Commands take it shared to read the DB version, so they wait while a migration runs but never wait for each other, and take it exclusive to migrate, re-reading the version after the wait so a migration that another process finished meanwhile isn't repeated. The wait is bounded (3600 seconds, EE_MIGRATION_LOCK_TIMEOUT overrides it) and announced on STDERR after two seconds with the holder's PID; on timeout the command stops with a clear error. A command that waited and finds the DB on a newer version than its own phar asks to be run again instead of suggesting an update. The `ee cli update` child takes the lock through `cli info`; the parent first waits visibly for a running migration, since the child's output is captured. `cli`, `config` and `help` don't migrate and take no lock. If the lock file can't be opened or locked, ee warns and continues unlocked. A command that didn't run because of a migration, either because it gave up waiting or because EasyEngine was updated while it waited, exits with 75 (EX_TEMPFAIL, MigrationLock::RETRY_EXIT_CODE) instead of 1, so callers such as cron or the Dashboard can tell "try again" from a failure. `ee cli update` checks the new phar's exit status for it (its `cli info` only warns when EasyEngine was updated meanwhile, so 75 there always means it gave up waiting): instead of reporting the download as broken, it says another ee process is migrating, keeps the installed phar (nothing has been migrated or replaced at that point), asks to run the update again later and exits 75 itself. --- php/EE/MigrationLock.php | 148 +++++++++++++++++++++++++++++++ php/EE/Runner.php | 85 ++++++++++++++---- php/commands/src/CLI_Command.php | 9 ++ 3 files changed, 226 insertions(+), 16 deletions(-) create mode 100644 php/EE/MigrationLock.php diff --git a/php/EE/MigrationLock.php b/php/EE/MigrationLock.php new file mode 100644 index 000000000..75f94406e --- /dev/null +++ b/php/EE/MigrationLock.php @@ -0,0 +1,148 @@ +path = null === $path ? EE_ROOT_DIR . '/db/migration.lock' : $path; + if ( null === $timeout ) { + $env = getenv( 'EE_MIGRATION_LOCK_TIMEOUT' ); + $timeout = ( false !== $env && is_numeric( $env ) ) ? (int) $env : self::DEFAULT_TIMEOUT; + } + $this->timeout = max( 0, (int) $timeout ); + } + + /** + * Takes the lock, waiting up to the timeout for other processes. Errors out with RETRY_EXIT_CODE when the wait times out. + * + * @param bool $exclusive True to migrate, false to only wait for a running migration. + * + * @return bool Whether this process had to wait. + */ + public function acquire( $exclusive ) { + if ( ! $this->open() ) { + return false; + } + + // Linux drops a held lock before converting it anyway; do it explicitly so the wait below is the only one. + flock( $this->handle, LOCK_UN ); + + $operation = $exclusive ? LOCK_EX : LOCK_SH; + $start = microtime( true ); + $delay = 100000; + $waited = false; + $announced = false; + + while ( ! flock( $this->handle, $operation | LOCK_NB, $would_block ) ) { + if ( ! $would_block ) { + EE::warning( sprintf( 'Cannot lock %s, so migrations are not protected against concurrent ee processes.', $this->path ) ); + $this->unavailable = true; + return $waited; + } + $waited = true; + $elapsed = microtime( true ) - $start; + if ( $elapsed >= $this->timeout ) { + EE::error( sprintf( 'Another ee process is still running EasyEngine migrations (%s). Gave up waiting after %d seconds. Check its progress in %s, then run this command again.', $this->get_holder(), $this->timeout, EE_ROOT_DIR . '/logs/ee.log' ), self::RETRY_EXIT_CODE ); + } + if ( ! $announced && $elapsed >= self::NOTICE_AFTER ) { + // STDERR, so `--format=json` output read by scripts stays parseable. + EE::warning( sprintf( 'Another ee process is running EasyEngine migrations (%s). Waiting up to %d seconds for it to finish...', $this->get_holder(), $this->timeout ) ); + $announced = true; + } + usleep( $delay ); + $delay = min( 500000, $delay * 2 ); + } + + if ( $exclusive ) { + ftruncate( $this->handle, 0 ); + rewind( $this->handle ); + fwrite( $this->handle, sprintf( 'PID %d since %s', getmypid(), gmdate( 'Y-m-d H:i:s' ) . ' UTC' ) ); + fflush( $this->handle ); + } + + return $waited; + } + + /** + * Releases the lock. Safe to call more than once. + */ + public function release() { + if ( $this->handle ) { + flock( $this->handle, LOCK_UN ); + fclose( $this->handle ); + $this->handle = null; + } + } + + /** + * @return bool + */ + private function open() { + if ( $this->unavailable ) { + return false; + } + if ( $this->handle ) { + return true; + } + + // 'e' (O_CLOEXEC): docker and other children must not keep the lock after ee dies. + $umask = umask( 0077 ); + $this->handle = @fopen( $this->path, 'c+e' ); + umask( $umask ); + + if ( ! $this->handle ) { + $this->handle = null; + $this->unavailable = true; + EE::warning( sprintf( 'Cannot open %s, so migrations are not protected against concurrent ee processes.', $this->path ) ); + return false; + } + // A readable lock would let any local user stall migrations with a shared lock. + if ( fileperms( $this->path ) & 0077 ) { + @chmod( $this->path, 0600 ); + } + + return true; + } + + /** + * @return string The holder recorded by the last exclusive lock, for messages. + */ + private function get_holder() { + $holder = trim( (string) @file_get_contents( $this->path ) ); + + return '' === $holder ? 'holder unknown' : $holder; + } +} diff --git a/php/EE/Runner.php b/php/EE/Runner.php index bb34a859a..d7a0700e3 100644 --- a/php/EE/Runner.php +++ b/php/EE/Runner.php @@ -1019,13 +1019,64 @@ private function auto_check_update() { /** * Triggers migration if current phar version > version in ee_option table. * Also, trigger migrations if phar version >= version in ee_option table but nightly version differ. + * + * Runs under the migration lock, so a command started while another process migrates waits for it. */ private function maybe_trigger_migration() { - $db_version = Option::get( 'version' ); - $current_version = EE_VERSION; + $lock = new MigrationLock(); + $waited = $lock->acquire( false ); + + try { + $db_version = Option::get( 'version' ); + $action = $this->get_migration_action( $db_version ); + + if ( 'setup' === $action || 'migrate' === $action ) { + $waited = $lock->acquire( true ) || $waited; + // Another process may have migrated while this one waited. + $db_version = Option::get( 'version' ); + $action = $this->get_migration_action( $db_version ); + } + $this->run_migration_action( $action, $db_version, $waited ); + } finally { + $lock->release(); + } + } + + /** + * @param string|false $db_version Version stored in the DB. + * + * @return string 'setup', 'outdated', 'migrate' or 'none'. + */ + private function get_migration_action( $db_version ) { if ( ! $db_version ) { + return 'setup'; + } + + $base_db_version = preg_replace( '/-nightly.*$/', '', $db_version ); + $base_current_version = preg_replace( '/-nightly.*$/', '', EE_VERSION ); + + if ( Comparator::lessThan( $base_current_version, $base_db_version ) ) { + return 'outdated'; + } + if ( $db_version !== EE_VERSION || false !== strpos( EE_VERSION, 'nightly' ) ) { + return 'migrate'; + } + + return 'none'; + } + + /** + * @param string $action From get_migration_action(). + * @param string|false $db_version Version stored in the DB. + * @param bool $waited Whether this process waited for another one's migration. + */ + private function run_migration_action( $action, $db_version, $waited ) { + + $current_version = EE_VERSION; + + if ( 'setup' === $action ) { $this->trigger_migration( $current_version ); EE::log( 'ee successfully setup.' ); if ( IS_DARWIN ) { @@ -1041,26 +1092,28 @@ private function maybe_trigger_migration() { return; } - $base_db_version = preg_replace( '/-nightly.*$/', '', $db_version ); - $base_current_version = preg_replace( '/-nightly.*$/', '', EE_VERSION ); - - if ( Comparator::lessThan( $base_current_version, $base_db_version ) ) { + if ( 'outdated' === $action ) { - $ee_update_command = IS_DARWIN ? 'brew upgrade easyengine' : 'ee cli update --stable --yes'; - $ee_update_msg = sprintf( - 'It seems you\'re not running latest version. Update EasyEngine using `%s`.', - $ee_update_command - ); + if ( $waited ) { + // Typically `ee cli update` replaced this phar while the command waited for its migration. + $ee_update_msg = sprintf( 'EasyEngine was updated to %s while this command waited for the migration. Please run it again.', $db_version ); + } else { + $ee_update_command = IS_DARWIN ? 'brew upgrade easyengine' : 'ee cli update --stable --yes'; + $ee_update_msg = sprintf( + 'It seems you\'re not running latest version. Update EasyEngine using `%s`.', + $ee_update_command + ); + } if ( ! empty( $this->arguments ) && 'cli' === $this->arguments[0] ) { EE::warning( $ee_update_msg ); } else { - EE::error( $ee_update_msg ); + EE::error( $ee_update_msg, $waited ? MigrationLock::RETRY_EXIT_CODE : true ); + } + } elseif ( 'migrate' === $action ) { + if ( $db_version !== $current_version ) { + EE::log( 'Executing migrations. This might take some time.' ); } - } elseif ( $db_version !== $current_version ) { - EE::log( 'Executing migrations. This might take some time.' ); - $this->trigger_migration( $current_version ); - } elseif ( false !== strpos( $current_version, 'nightly' ) ) { $this->trigger_migration( $current_version ); } } diff --git a/php/commands/src/CLI_Command.php b/php/commands/src/CLI_Command.php index a7a2af1a5..8f3f32d25 100644 --- a/php/commands/src/CLI_Command.php +++ b/php/commands/src/CLI_Command.php @@ -334,12 +334,21 @@ public function update( $_, $assoc_args ) { EE::log( 'Updating EasyEngine to new version. This might take some time.' ); + // The new phar migrates under the migration lock with its output captured, so wait visibly for a running migration first. + $migration_lock = new EE\MigrationLock(); + $migration_lock->acquire( false ); + $migration_lock->release(); + $php_binary = Utils\get_php_binary(); $process = EE\Process::create( "{$php_binary} $temp cli info", null, null ); $result = $process->run(); if ( 0 !== $result->return_code || false === stripos( $result->stdout, 'EE version' ) ) { $multi_line = explode( PHP_EOL, $result->stderr ); EE::error_multi_line( $multi_line ); + if ( EE\MigrationLock::RETRY_EXIT_CODE === $result->return_code ) { + // Only a lock give-up: `cli info` just warns if ee was updated meanwhile. The new phar gave up before migrating anything, so keep the installed one. + EE::error( sprintf( 'Another ee process is running EasyEngine migrations, so EasyEngine was not updated and is still at %s. Try running ee cli update again once it has finished.', EE_VERSION ), EE\MigrationLock::RETRY_EXIT_CODE ); + } EE::error( 'The downloaded PHAR is broken, try running ee cli update again.' ); } EE::log( 'New version works. Proceeding to replace.' ); From 890dbe7488f28fecae060d30ada352d80e158456 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Wed, 18 Nov 2026 14:27:35 +0530 Subject: [PATCH 2/2] fix(migration): run without the migration lock when flock() is disabled PHP 8 removes a function listed in disable_functions instead of only warning when it is called, so on a host that disables flock() every command except `cli` stopped with "Call to undefined function flock()". The migration lock now checks that flock() exists first; when it doesn't, it warns once that migrations are not protected against concurrent ee processes and the command runs without the lock, as it already did when the lock file can't be opened. --- php/EE/MigrationLock.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/php/EE/MigrationLock.php b/php/EE/MigrationLock.php index 75f94406e..38dac9fd9 100644 --- a/php/EE/MigrationLock.php +++ b/php/EE/MigrationLock.php @@ -117,6 +117,12 @@ private function open() { if ( $this->handle ) { return true; } + // PHP 8 removes a function listed in disable_functions, so calling it would be fatal. + if ( ! function_exists( 'flock' ) ) { + $this->unavailable = true; + EE::warning( 'PHP function flock() is unavailable, so migrations are not protected against concurrent ee processes.' ); + return false; + } // 'e' (O_CLOEXEC): docker and other children must not keep the lock after ee dies. $umask = umask( 0077 );