Repository navigation
fix(migration): serialise migrations with a lock #1965
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: develop
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| <?php | ||
|
|
||
| namespace EE; | ||
|
|
||
| use EE; | ||
|
|
||
| /** | ||
| * Serialises migrations across ee processes with flock(), which the kernel drops when the holder exits or crashes. | ||
| * | ||
| * Commands that may migrate take it shared to read the DB version, so they wait while a migration runs, and exclusive to migrate. | ||
| */ | ||
| class MigrationLock { | ||
|
|
||
| /** Seconds to wait for another process before giving up; `EE_MIGRATION_LOCK_TIMEOUT` overrides it. */ | ||
| const DEFAULT_TIMEOUT = 3600; | ||
|
|
||
| /** Seconds of waiting before telling the user, so brief version checks by other processes stay silent. */ | ||
| const NOTICE_AFTER = 2; | ||
|
|
||
| /** Exit code (sysexits EX_TEMPFAIL) when a command didn't run and should be retried: it gave up waiting, or ee was updated while it waited. `ee cli update` checks the new phar for it, so never change it. */ | ||
| const RETRY_EXIT_CODE = 75; | ||
|
|
||
| /** @var string */ | ||
| private $path; | ||
|
|
||
| /** @var int */ | ||
| private $timeout; | ||
|
|
||
| /** @var resource|null */ | ||
| private $handle = null; | ||
|
|
||
| /** @var bool Whether the lock could not be opened or isn't supported, so callers run unlocked. */ | ||
| private $unavailable = false; | ||
|
|
||
| /** | ||
| * @param string|null $path Lock file, null for the one all ee processes share. Never unlink it: a new inode would let a second process lock it too. | ||
| * @param int|null $timeout Seconds to wait, 0 to fail at once, null for the default. | ||
| */ | ||
| public function __construct( $path = null, $timeout = null ) { | ||
| $this->path = null === $path ? EE_ROOT_DIR . '/db/migration.lock' : $path; | ||
| if ( null === $timeout ) { | ||
| $env = getenv( 'EE_MIGRATION_LOCK_TIMEOUT' ); | ||
| $timeout = ( false !== $env && is_numeric( $env ) ) ? (int) $env : self::DEFAULT_TIMEOUT; | ||
| } | ||
| $this->timeout = max( 0, (int) $timeout ); | ||
| } | ||
|
|
||
| /** | ||
| * Takes the lock, waiting up to the timeout for other processes. Errors out with RETRY_EXIT_CODE when the wait times out. | ||
| * | ||
| * @param bool $exclusive True to migrate, false to only wait for a running migration. | ||
| * | ||
| * @return bool Whether this process had to wait. | ||
| */ | ||
| public function acquire( $exclusive ) { | ||
| if ( ! $this->open() ) { | ||
| return false; | ||
| } | ||
|
|
||
| // Linux drops a held lock before converting it anyway; do it explicitly so the wait below is the only one. | ||
| flock( $this->handle, LOCK_UN ); | ||
|
|
||
| $operation = $exclusive ? LOCK_EX : LOCK_SH; | ||
| $start = microtime( true ); | ||
| $delay = 100000; | ||
| $waited = false; | ||
| $announced = false; | ||
|
|
||
| while ( ! flock( $this->handle, $operation | LOCK_NB, $would_block ) ) { | ||
| if ( ! $would_block ) { | ||
| EE::warning( sprintf( 'Cannot lock %s, so migrations are not protected against concurrent ee processes.', $this->path ) ); | ||
| $this->unavailable = true; | ||
| return $waited; | ||
| } | ||
| $waited = true; | ||
| $elapsed = microtime( true ) - $start; | ||
| if ( $elapsed >= $this->timeout ) { | ||
| EE::error( sprintf( 'Another ee process is still running EasyEngine migrations (%s). Gave up waiting after %d seconds. Check its progress in %s, then run this command again.', $this->get_holder(), $this->timeout, EE_ROOT_DIR . '/logs/ee.log' ), self::RETRY_EXIT_CODE ); | ||
| } | ||
| if ( ! $announced && $elapsed >= self::NOTICE_AFTER ) { | ||
| // STDERR, so `--format=json` output read by scripts stays parseable. | ||
| EE::warning( sprintf( 'Another ee process is running EasyEngine migrations (%s). Waiting up to %d seconds for it to finish...', $this->get_holder(), $this->timeout ) ); | ||
| $announced = true; | ||
| } | ||
| usleep( $delay ); | ||
| $delay = min( 500000, $delay * 2 ); | ||
| } | ||
|
|
||
| if ( $exclusive ) { | ||
| ftruncate( $this->handle, 0 ); | ||
| rewind( $this->handle ); | ||
| fwrite( $this->handle, sprintf( 'PID %d since %s', getmypid(), gmdate( 'Y-m-d H:i:s' ) . ' UTC' ) ); | ||
| fflush( $this->handle ); | ||
| } | ||
|
|
||
| return $waited; | ||
| } | ||
|
|
||
| /** | ||
| * Releases the lock. Safe to call more than once. | ||
| */ | ||
| public function release() { | ||
| if ( $this->handle ) { | ||
| flock( $this->handle, LOCK_UN ); | ||
| fclose( $this->handle ); | ||
| $this->handle = null; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * @return bool | ||
| */ | ||
| private function open() { | ||
| if ( $this->unavailable ) { | ||
| return false; | ||
| } | ||
| if ( $this->handle ) { | ||
| return true; | ||
| } | ||
| // PHP 8 removes a function listed in disable_functions, so calling it would be fatal. | ||
| if ( ! function_exists( 'flock' ) ) { | ||
| $this->unavailable = true; | ||
| EE::warning( 'PHP function flock() is unavailable, so migrations are not protected against concurrent ee processes.' ); | ||
| return false; | ||
| } | ||
|
|
||
| // 'e' (O_CLOEXEC): docker and other children must not keep the lock after ee dies. | ||
| $umask = umask( 0077 ); | ||
| $this->handle = @fopen( $this->path, 'c+e' ); | ||
| umask( $umask ); | ||
|
|
||
| if ( ! $this->handle ) { | ||
| $this->handle = null; | ||
| $this->unavailable = true; | ||
| EE::warning( sprintf( 'Cannot open %s, so migrations are not protected against concurrent ee processes.', $this->path ) ); | ||
| return false; | ||
| } | ||
| // A readable lock would let any local user stall migrations with a shared lock. | ||
| if ( fileperms( $this->path ) & 0077 ) { | ||
| @chmod( $this->path, 0600 ); | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| /** | ||
| * @return string The holder recorded by the last exclusive lock, for messages. | ||
| */ | ||
| private function get_holder() { | ||
| $holder = trim( (string) @file_get_contents( $this->path ) ); | ||
|
|
||
| return '' === $holder ? 'holder unknown' : $holder; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1019,13 +1019,64 @@ private function auto_check_update() { | |
| /** | ||
| * Triggers migration if current phar version > version in ee_option table. | ||
| * Also, trigger migrations if phar version >= version in ee_option table but nightly version differ. | ||
| * | ||
| * Runs under the migration lock, so a command started while another process migrates waits for it. | ||
| */ | ||
| private function maybe_trigger_migration() { | ||
|
|
||
| $db_version = Option::get( 'version' ); | ||
| $current_version = EE_VERSION; | ||
| $lock = new MigrationLock(); | ||
| $waited = $lock->acquire( false ); | ||
|
|
||
| try { | ||
| $db_version = Option::get( 'version' ); | ||
| $action = $this->get_migration_action( $db_version ); | ||
|
|
||
| if ( 'setup' === $action || 'migrate' === $action ) { | ||
| $waited = $lock->acquire( true ) || $waited; | ||
| // Another process may have migrated while this one waited. | ||
| $db_version = Option::get( 'version' ); | ||
| $action = $this->get_migration_action( $db_version ); | ||
| } | ||
|
|
||
| $this->run_migration_action( $action, $db_version, $waited ); | ||
| } finally { | ||
| $lock->release(); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * @param string|false $db_version Version stored in the DB. | ||
| * | ||
| * @return string 'setup', 'outdated', 'migrate' or 'none'. | ||
| */ | ||
| private function get_migration_action( $db_version ) { | ||
| if ( ! $db_version ) { | ||
| return 'setup'; | ||
| } | ||
|
|
||
| $base_db_version = preg_replace( '/-nightly.*$/', '', $db_version ); | ||
| $base_current_version = preg_replace( '/-nightly.*$/', '', EE_VERSION ); | ||
|
|
||
| if ( Comparator::lessThan( $base_current_version, $base_db_version ) ) { | ||
| return 'outdated'; | ||
| } | ||
| if ( $db_version !== EE_VERSION || false !== strpos( EE_VERSION, 'nightly' ) ) { | ||
| return 'migrate'; | ||
|
Comment on lines
+1063
to
+1064
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Reject a superseded nightly before selecting If an old nightly command waits while a newer nightly with the same base version completes, the stripped versions compare equal. This condition then selects Detect that the waiting process's build was superseded before running migrations. Preserve intentional reruns of the same nightly build. Otherwise, the older build can also apply its older container-image versions. (raw.githubusercontent.com)
|
||
| } | ||
|
|
||
| return 'none'; | ||
| } | ||
|
|
||
| /** | ||
| * @param string $action From get_migration_action(). | ||
| * @param string|false $db_version Version stored in the DB. | ||
| * @param bool $waited Whether this process waited for another one's migration. | ||
| */ | ||
| private function run_migration_action( $action, $db_version, $waited ) { | ||
|
|
||
| $current_version = EE_VERSION; | ||
|
|
||
| if ( 'setup' === $action ) { | ||
| $this->trigger_migration( $current_version ); | ||
| EE::log( 'ee successfully setup.' ); | ||
| if ( IS_DARWIN ) { | ||
|
|
@@ -1041,26 +1092,28 @@ private function maybe_trigger_migration() { | |
| return; | ||
| } | ||
|
|
||
| $base_db_version = preg_replace( '/-nightly.*$/', '', $db_version ); | ||
| $base_current_version = preg_replace( '/-nightly.*$/', '', EE_VERSION ); | ||
|
|
||
| if ( Comparator::lessThan( $base_current_version, $base_db_version ) ) { | ||
| if ( 'outdated' === $action ) { | ||
|
|
||
| $ee_update_command = IS_DARWIN ? 'brew upgrade easyengine' : 'ee cli update --stable --yes'; | ||
| $ee_update_msg = sprintf( | ||
| 'It seems you\'re not running latest version. Update EasyEngine using `%s`.', | ||
| $ee_update_command | ||
| ); | ||
| if ( $waited ) { | ||
| // Typically `ee cli update` replaced this phar while the command waited for its migration. | ||
| $ee_update_msg = sprintf( 'EasyEngine was updated to %s while this command waited for the migration. Please run it again.', $db_version ); | ||
| } else { | ||
| $ee_update_command = IS_DARWIN ? 'brew upgrade easyengine' : 'ee cli update --stable --yes'; | ||
| $ee_update_msg = sprintf( | ||
| 'It seems you\'re not running latest version. Update EasyEngine using `%s`.', | ||
| $ee_update_command | ||
| ); | ||
| } | ||
|
|
||
| if ( ! empty( $this->arguments ) && 'cli' === $this->arguments[0] ) { | ||
| EE::warning( $ee_update_msg ); | ||
| } else { | ||
| EE::error( $ee_update_msg ); | ||
| EE::error( $ee_update_msg, $waited ? MigrationLock::RETRY_EXIT_CODE : true ); | ||
| } | ||
| } elseif ( 'migrate' === $action ) { | ||
| if ( $db_version !== $current_version ) { | ||
| EE::log( 'Executing migrations. This might take some time.' ); | ||
| } | ||
| } elseif ( $db_version !== $current_version ) { | ||
| EE::log( 'Executing migrations. This might take some time.' ); | ||
| $this->trigger_migration( $current_version ); | ||
| } elseif ( false !== strpos( $current_version, 'nightly' ) ) { | ||
| $this->trigger_migration( $current_version ); | ||
| } | ||
| } | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.