diff --git a/php/EE/MigrationLock.php b/php/EE/MigrationLock.php new file mode 100644 index 000000000..38dac9fd9 --- /dev/null +++ b/php/EE/MigrationLock.php @@ -0,0 +1,154 @@ +path = null === $path ? EE_ROOT_DIR . '/db/migration.lock' : $path; + if ( null === $timeout ) { + $env = getenv( 'EE_MIGRATION_LOCK_TIMEOUT' ); + $timeout = ( false !== $env && is_numeric( $env ) ) ? (int) $env : self::DEFAULT_TIMEOUT; + } + $this->timeout = max( 0, (int) $timeout ); + } + + /** + * Takes the lock, waiting up to the timeout for other processes. Errors out with RETRY_EXIT_CODE when the wait times out. + * + * @param bool $exclusive True to migrate, false to only wait for a running migration. + * + * @return bool Whether this process had to wait. + */ + public function acquire( $exclusive ) { + if ( ! $this->open() ) { + return false; + } + + // Linux drops a held lock before converting it anyway; do it explicitly so the wait below is the only one. + flock( $this->handle, LOCK_UN ); + + $operation = $exclusive ? LOCK_EX : LOCK_SH; + $start = microtime( true ); + $delay = 100000; + $waited = false; + $announced = false; + + while ( ! flock( $this->handle, $operation | LOCK_NB, $would_block ) ) { + if ( ! $would_block ) { + EE::warning( sprintf( 'Cannot lock %s, so migrations are not protected against concurrent ee processes.', $this->path ) ); + $this->unavailable = true; + return $waited; + } + $waited = true; + $elapsed = microtime( true ) - $start; + if ( $elapsed >= $this->timeout ) { + EE::error( sprintf( 'Another ee process is still running EasyEngine migrations (%s). Gave up waiting after %d seconds. Check its progress in %s, then run this command again.', $this->get_holder(), $this->timeout, EE_ROOT_DIR . '/logs/ee.log' ), self::RETRY_EXIT_CODE ); + } + if ( ! $announced && $elapsed >= self::NOTICE_AFTER ) { + // STDERR, so `--format=json` output read by scripts stays parseable. + EE::warning( sprintf( 'Another ee process is running EasyEngine migrations (%s). Waiting up to %d seconds for it to finish...', $this->get_holder(), $this->timeout ) ); + $announced = true; + } + usleep( $delay ); + $delay = min( 500000, $delay * 2 ); + } + + if ( $exclusive ) { + ftruncate( $this->handle, 0 ); + rewind( $this->handle ); + fwrite( $this->handle, sprintf( 'PID %d since %s', getmypid(), gmdate( 'Y-m-d H:i:s' ) . ' UTC' ) ); + fflush( $this->handle ); + } + + return $waited; + } + + /** + * Releases the lock. Safe to call more than once. + */ + public function release() { + if ( $this->handle ) { + flock( $this->handle, LOCK_UN ); + fclose( $this->handle ); + $this->handle = null; + } + } + + /** + * @return bool + */ + private function open() { + if ( $this->unavailable ) { + return false; + } + if ( $this->handle ) { + return true; + } + // PHP 8 removes a function listed in disable_functions, so calling it would be fatal. + if ( ! function_exists( 'flock' ) ) { + $this->unavailable = true; + EE::warning( 'PHP function flock() is unavailable, so migrations are not protected against concurrent ee processes.' ); + return false; + } + + // 'e' (O_CLOEXEC): docker and other children must not keep the lock after ee dies. + $umask = umask( 0077 ); + $this->handle = @fopen( $this->path, 'c+e' ); + umask( $umask ); + + if ( ! $this->handle ) { + $this->handle = null; + $this->unavailable = true; + EE::warning( sprintf( 'Cannot open %s, so migrations are not protected against concurrent ee processes.', $this->path ) ); + return false; + } + // A readable lock would let any local user stall migrations with a shared lock. + if ( fileperms( $this->path ) & 0077 ) { + @chmod( $this->path, 0600 ); + } + + return true; + } + + /** + * @return string The holder recorded by the last exclusive lock, for messages. + */ + private function get_holder() { + $holder = trim( (string) @file_get_contents( $this->path ) ); + + return '' === $holder ? 'holder unknown' : $holder; + } +} diff --git a/php/EE/Runner.php b/php/EE/Runner.php index bb34a859a..d7a0700e3 100644 --- a/php/EE/Runner.php +++ b/php/EE/Runner.php @@ -1019,13 +1019,64 @@ private function auto_check_update() { /** * Triggers migration if current phar version > version in ee_option table. * Also, trigger migrations if phar version >= version in ee_option table but nightly version differ. + * + * Runs under the migration lock, so a command started while another process migrates waits for it. */ private function maybe_trigger_migration() { - $db_version = Option::get( 'version' ); - $current_version = EE_VERSION; + $lock = new MigrationLock(); + $waited = $lock->acquire( false ); + + try { + $db_version = Option::get( 'version' ); + $action = $this->get_migration_action( $db_version ); + + if ( 'setup' === $action || 'migrate' === $action ) { + $waited = $lock->acquire( true ) || $waited; + // Another process may have migrated while this one waited. + $db_version = Option::get( 'version' ); + $action = $this->get_migration_action( $db_version ); + } + $this->run_migration_action( $action, $db_version, $waited ); + } finally { + $lock->release(); + } + } + + /** + * @param string|false $db_version Version stored in the DB. + * + * @return string 'setup', 'outdated', 'migrate' or 'none'. + */ + private function get_migration_action( $db_version ) { if ( ! $db_version ) { + return 'setup'; + } + + $base_db_version = preg_replace( '/-nightly.*$/', '', $db_version ); + $base_current_version = preg_replace( '/-nightly.*$/', '', EE_VERSION ); + + if ( Comparator::lessThan( $base_current_version, $base_db_version ) ) { + return 'outdated'; + } + if ( $db_version !== EE_VERSION || false !== strpos( EE_VERSION, 'nightly' ) ) { + return 'migrate'; + } + + return 'none'; + } + + /** + * @param string $action From get_migration_action(). + * @param string|false $db_version Version stored in the DB. + * @param bool $waited Whether this process waited for another one's migration. + */ + private function run_migration_action( $action, $db_version, $waited ) { + + $current_version = EE_VERSION; + + if ( 'setup' === $action ) { $this->trigger_migration( $current_version ); EE::log( 'ee successfully setup.' ); if ( IS_DARWIN ) { @@ -1041,26 +1092,28 @@ private function maybe_trigger_migration() { return; } - $base_db_version = preg_replace( '/-nightly.*$/', '', $db_version ); - $base_current_version = preg_replace( '/-nightly.*$/', '', EE_VERSION ); - - if ( Comparator::lessThan( $base_current_version, $base_db_version ) ) { + if ( 'outdated' === $action ) { - $ee_update_command = IS_DARWIN ? 'brew upgrade easyengine' : 'ee cli update --stable --yes'; - $ee_update_msg = sprintf( - 'It seems you\'re not running latest version. Update EasyEngine using `%s`.', - $ee_update_command - ); + if ( $waited ) { + // Typically `ee cli update` replaced this phar while the command waited for its migration. + $ee_update_msg = sprintf( 'EasyEngine was updated to %s while this command waited for the migration. Please run it again.', $db_version ); + } else { + $ee_update_command = IS_DARWIN ? 'brew upgrade easyengine' : 'ee cli update --stable --yes'; + $ee_update_msg = sprintf( + 'It seems you\'re not running latest version. Update EasyEngine using `%s`.', + $ee_update_command + ); + } if ( ! empty( $this->arguments ) && 'cli' === $this->arguments[0] ) { EE::warning( $ee_update_msg ); } else { - EE::error( $ee_update_msg ); + EE::error( $ee_update_msg, $waited ? MigrationLock::RETRY_EXIT_CODE : true ); + } + } elseif ( 'migrate' === $action ) { + if ( $db_version !== $current_version ) { + EE::log( 'Executing migrations. This might take some time.' ); } - } elseif ( $db_version !== $current_version ) { - EE::log( 'Executing migrations. This might take some time.' ); - $this->trigger_migration( $current_version ); - } elseif ( false !== strpos( $current_version, 'nightly' ) ) { $this->trigger_migration( $current_version ); } } diff --git a/php/commands/src/CLI_Command.php b/php/commands/src/CLI_Command.php index a7a2af1a5..8f3f32d25 100644 --- a/php/commands/src/CLI_Command.php +++ b/php/commands/src/CLI_Command.php @@ -334,12 +334,21 @@ public function update( $_, $assoc_args ) { EE::log( 'Updating EasyEngine to new version. This might take some time.' ); + // The new phar migrates under the migration lock with its output captured, so wait visibly for a running migration first. + $migration_lock = new EE\MigrationLock(); + $migration_lock->acquire( false ); + $migration_lock->release(); + $php_binary = Utils\get_php_binary(); $process = EE\Process::create( "{$php_binary} $temp cli info", null, null ); $result = $process->run(); if ( 0 !== $result->return_code || false === stripos( $result->stdout, 'EE version' ) ) { $multi_line = explode( PHP_EOL, $result->stderr ); EE::error_multi_line( $multi_line ); + if ( EE\MigrationLock::RETRY_EXIT_CODE === $result->return_code ) { + // Only a lock give-up: `cli info` just warns if ee was updated meanwhile. The new phar gave up before migrating anything, so keep the installed one. + EE::error( sprintf( 'Another ee process is running EasyEngine migrations, so EasyEngine was not updated and is still at %s. Try running ee cli update again once it has finished.', EE_VERSION ), EE\MigrationLock::RETRY_EXIT_CODE ); + } EE::error( 'The downloaded PHAR is broken, try running ee cli update again.' ); } EE::log( 'New version works. Proceeding to replace.' );