diff --git a/features/COVERAGE.md b/features/COVERAGE.md index c137c186..20343e1f 100644 --- a/features/COVERAGE.md +++ b/features/COVERAGE.md @@ -192,7 +192,7 @@ Legend: covered / partial / not covered (reason). Feature files are abbreviated: | restore latest, restore `--id`, unknown id | covered | backup: Restore the latest backup; Restore a specific backup by id; Restoring an unknown backup id fails | | disabled site | covered | backup: A disabled site cannot be backed up...; en-dis outline | | `--dash-auth` format and ed-api-url check | covered | backup: A malformed --dash-auth is rejected; --dash-auth needs ed-api-url in the config | -| `--dash-auth` success/failure callbacks, rollback of the uploaded backup | not covered | needs an EasyEngine Dashboard API endpoint (`ed-api-url`) | +| `--dash-auth` success/failure callbacks, uploaded backup kept when the success callback fails | not covered | needs an EasyEngine Dashboard API endpoint (`ed-api-url`) | | per-site lock blocks, lock left by a failed restore | covered | backup: A lock file of another backup blocks the backup (held with flock(1), so it holds with or without EasyEngine/site-command#480); not covered yet: a failed restore leaves the lock file and blocks later backups (its scenario comes back with the fix) | | stale download reuse | not covered yet | a restore by id reuses files an earlier restore left behind (its scenario comes back with the fix) | | retention (`no-of-backups`) | not covered yet | retention keeps one backup more than `no-of-backups` (its scenario comes back with the fix) | diff --git a/src/helper/Site_Backup_Restore.php b/src/helper/Site_Backup_Restore.php index ce01f102..ed89f001 100644 --- a/src/helper/Site_Backup_Restore.php +++ b/src/helper/Site_Backup_Restore.php @@ -43,7 +43,8 @@ class Site_Backup_Restore { private $dash_api_url; private $dash_backup_metadata; private $dash_backup_completed = false; - private $dash_new_backup_path; // Track new backup path for potential rollback + private $dash_callback_sent = false; // Guard: at most one terminal callback (success XOR failure) may be emitted + private $dash_new_backup_path; // Remote path of this run's upload, reported if the success callback fails // Error tracking for EasyEngine Dashboard failure callbacks private $dash_error_message = ''; @@ -159,8 +160,6 @@ public function backup( $args, $assoc_args = [] ) { $this->fs->remove( EE_BACKUP_DIR . '/' . $this->site_data['site_url'] . '.lock' ); - // Mark backup as completed and send success callback - $this->dash_backup_completed = true; if ( $this->dash_auth_enabled ) { $api_success = $this->send_dash_success_callback( $this->dash_api_url, @@ -169,13 +168,26 @@ public function backup( $args, $assoc_args = [] ) { $this->dash_backup_metadata ); - // Only cleanup old backups if API callback succeeded - // If API failed, rollback the newly uploaded backup if ( $api_success ) { + // Backup is now registered with the EasyEngine Dashboard; only then is it safe to + // mark complete (suppressing the shutdown failure callback) and prune. + $this->dash_backup_completed = true; $this->cleanup_old_backups(); } else { - $this->rollback_failed_backup(); + // Never delete an uploaded backup because only the callback failed. dash_backup_completed stays + // false, so the shutdown handler sends the single failure callback with this message. + $message = sprintf( + 'Backup uploaded to %s but the EasyEngine Dashboard success callback failed; the backup was kept.', + $this->dash_new_backup_path + ); + $this->capture_error( $message, self::ERROR_TYPE_NETWORK, 4004 ); + // Don't hold the lock through the failure callback's retries in the shutdown handler. + $this->release_global_backup_lock(); + EE::error( $message ); } + } else { + // Non-dash path: backup is done once the upload succeeds. + $this->dash_backup_completed = true; } // Release global backup lock (also released by shutdown handler as safety net) @@ -194,8 +206,9 @@ public function backup( $args, $assoc_args = [] ) { * explicitly captured during backup execution. */ public function dash_shutdown_handler() { - // Only send failure callback if dash auth was enabled and backup didn't complete - if ( $this->dash_auth_enabled && ! $this->dash_backup_completed ) { + // Only send a failure callback if dash auth was enabled, the backup didn't + // complete, and no terminal callback (success or failure) has been sent yet. + if ( $this->dash_auth_enabled && ! $this->dash_backup_completed && ! $this->dash_callback_sent ) { // If no error was captured yet, try to capture shutdown error if ( empty( $this->dash_error_message ) ) { @@ -1502,7 +1515,7 @@ private function rclone_upload( $path ) { $remote_path = $output->stdout; EE::success( 'Backup uploaded to remote storage. Remote path: ' . $remote_path ); - // Store the new backup path for potential rollback (only when using dash-auth) + // Kept for the error reported if the success callback fails (only when using dash-auth) if ( $this->dash_auth_enabled ) { $this->dash_new_backup_path = $this->get_remote_path(); } @@ -1629,32 +1642,6 @@ private function cleanup_old_backups() { } } - /** - * Rollback (delete) the newly uploaded backup when EasyEngine Dashboard API callback fails. - * This prevents orphaned backups in remote storage that aren't tracked by the EasyEngine Dashboard. - */ - private function rollback_failed_backup() { - if ( empty( $this->dash_new_backup_path ) ) { - EE::warning( 'Cannot rollback backup: backup path not found.' ); - - return; - } - - EE::warning( 'EasyEngine Dashboard API callback failed. Rolling back newly uploaded backup...' ); - EE::log( 'Deleting unregistered backup: ' . $this->dash_new_backup_path ); - - $result = EE::launch( sprintf( 'rclone purge %s', escapeshellarg( $this->dash_new_backup_path ) ) ); - - if ( $result->return_code ) { - EE::warning( sprintf( - 'Failed to delete backup from remote storage. Please manually delete: %s', - $this->dash_new_backup_path - ) ); - } else { - EE::success( 'Successfully removed unregistered backup from remote storage.' ); - } - } - private function restore_nginx_conf( $backup_dir ) { $backup_file = $backup_dir . '/conf.zip'; @@ -2037,7 +2024,15 @@ private function send_dash_success_callback( $ed_api_url, $backup_id, $verify_to EE::debug( 'Payload being sent: ' . json_encode( array_merge( $payload, [ 'verify' => '******' ] ) ) ); - return $this->send_dash_request( $endpoint, $payload ); + $success = $this->send_dash_request( $endpoint, $payload ); + + // A success that actually reached the EasyEngine Dashboard is the terminal callback; a failed + // attempt is not, so the failure path can still emit the failure callback. + if ( $success ) { + $this->dash_callback_sent = true; + } + + return $success; } /** @@ -2066,6 +2061,9 @@ private function send_dash_failure_callback( $ed_api_url, $backup_id, $verify_to 'error_code' => $payload['error_code'], ] ) ); + // Failure is terminal: mark before sending so a concurrent shutdown pass + // can never emit a second (duplicate) terminal callback. + $this->dash_callback_sent = true; $this->send_dash_request( $endpoint, $payload ); }