From b526907e8dcb1ad4e308abc75a437534e5b169b3 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 29 Jun 2026 22:03:46 +0530 Subject: [PATCH 1/6] fix(backup): make EasyDash backup callback flow consistent The --dash-auth backup path could report success even when no backup remained, and could leave EasyDash with neither a success nor a failure callback. - Defer dash_backup_completed = true until after a confirmed success callback so the shutdown handler still fires the failure callback when the success callback fails and the upload is rolled back. - On success-callback failure, capture_error + EE::error after rollback so the exit code and message reflect that no backup remains, instead of falling through to EE::success. - Add a dash_callback_sent guard set at the actual send sites and checked by the shutdown handler so exactly one terminal callback (success XOR failure) is ever emitted. - Surface a failed rollback purge via capture_error + EE::error so the orphaned-remote-backup state propagates to the exit code and failure callback rather than being swallowed by a warning. --- src/helper/Site_Backup_Restore.php | 51 ++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/src/helper/Site_Backup_Restore.php b/src/helper/Site_Backup_Restore.php index a287b639..6cd18dcc 100644 --- a/src/helper/Site_Backup_Restore.php +++ b/src/helper/Site_Backup_Restore.php @@ -41,6 +41,7 @@ class Site_Backup_Restore { private $dash_api_url; private $dash_backup_metadata; private $dash_backup_completed = false; + private $dash_callback_sent = false; // Guard: exactly one terminal callback (success XOR failure) may be emitted private $dash_new_backup_path; // Track new backup path for potential rollback // Error tracking for EasyDash failure callbacks @@ -154,8 +155,6 @@ public function backup( $args, $assoc_args = [] ) { $this->fs->remove( EE_BACKUP_DIR . '/' . $this->site_data['site_url'] . '.lock' ); - // Mark backup as completed and send success callback - $this->dash_backup_completed = true; if ( $this->dash_auth_enabled ) { $api_success = $this->send_dash_success_callback( $this->dash_api_url, @@ -164,13 +163,27 @@ public function backup( $args, $assoc_args = [] ) { $this->dash_backup_metadata ); - // Only cleanup old backups if API callback succeeded - // If API failed, rollback the newly uploaded backup if ( $api_success ) { + // Backup is now registered with EasyDash; only then is it safe to + // mark complete (suppressing the shutdown failure callback) and prune. + $this->dash_backup_completed = true; $this->cleanup_old_backups(); } else { + // Success callback failed: the upload is orphaned (EasyDash never + // recorded it), so roll it back and report failure. dash_backup_completed + // stays false so the shutdown handler emits the single failure callback; + // EE::error() makes the exit code/message reflect that no backup remains. + $this->capture_error( + 'Backup uploaded but EasyDash success callback failed; rolled back the orphaned upload.', + self::ERROR_TYPE_NETWORK, + 4002 + ); $this->rollback_failed_backup(); + EE::error( 'EasyDash success callback failed; the uploaded backup was rolled back. No backup was created.' ); } + } else { + // Non-dash path: backup is done once the upload succeeds. + $this->dash_backup_completed = true; } // Release global backup lock (also released by shutdown handler as safety net) @@ -189,8 +202,9 @@ public function backup( $args, $assoc_args = [] ) { * explicitly captured during backup execution. */ public function dash_shutdown_handler() { - // Only send failure callback if dash auth was enabled and backup didn't complete - if ( $this->dash_auth_enabled && ! $this->dash_backup_completed ) { + // Only send a failure callback if dash auth was enabled, the backup didn't + // complete, and no terminal callback (success or failure) has been sent yet. + if ( $this->dash_auth_enabled && ! $this->dash_backup_completed && ! $this->dash_callback_sent ) { // If no error was captured yet, try to capture shutdown error if ( empty( $this->dash_error_message ) ) { @@ -1635,10 +1649,16 @@ private function rollback_failed_backup() { $result = EE::launch( sprintf( 'rclone purge %s', escapeshellarg( $this->dash_new_backup_path ) ) ); if ( $result->return_code ) { - EE::warning( sprintf( - 'Failed to delete backup from remote storage. Please manually delete: %s', + // Rollback purge failed: an untracked backup is now orphaned on the + // remote. Surface it so the exit code and failure callback reflect this + // state instead of silently continuing past a warning. capture_error keeps + // the root cause if one was already recorded by the caller. + $message = sprintf( + 'Failed to delete orphaned backup from remote storage. Please manually delete: %s', $this->dash_new_backup_path - ) ); + ); + $this->capture_error( $message, self::ERROR_TYPE_FILESYSTEM, 4003 ); + EE::error( $message ); } else { EE::success( 'Successfully removed unregistered backup from remote storage.' ); } @@ -1727,7 +1747,15 @@ private function send_dash_success_callback( $ed_api_url, $backup_id, $verify_to EE::debug( 'Payload being sent: ' . json_encode( $payload ) ); - return $this->send_dash_request( $endpoint, $payload ); + $success = $this->send_dash_request( $endpoint, $payload ); + + // A success that actually reached EasyDash is the terminal callback; a failed + // attempt is not, so the failure path can still emit the failure callback. + if ( $success ) { + $this->dash_callback_sent = true; + } + + return $success; } /** @@ -1756,6 +1784,9 @@ private function send_dash_failure_callback( $ed_api_url, $backup_id, $verify_to 'error_code' => $payload['error_code'], ] ) ); + // Failure is terminal: mark before sending so a concurrent shutdown pass + // can never emit a second (duplicate) terminal callback. + $this->dash_callback_sent = true; $this->send_dash_request( $endpoint, $payload ); } From b88166fd896cca327d065e76b3d05dce83c8ebd4 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 29 Jun 2026 22:18:50 +0530 Subject: [PATCH 2/6] fix(backup): correct EasyDash error code and orphan-purge failure payload Review follow-up to the dash-callback consistency fix. - Give the callback-failure rollback its own error_code 4004; 4002 was already in use for ERROR_TYPE_DATABASE ("Database backup failed"), and error_code is shipped to EasyDash as a machine-readable field. - When the rollback rclone purge also fails, force-overwrite the captured dash error before re-capturing so the accurate FILESYSTEM 4003 "manually delete " message wins over the caller's first-wins optimistic "rolled back" message; otherwise EasyDash was told the inverse of reality (rolled back) and never received the manual path. - Demote the rollback-success notice from EE::success to EE::log since it always precedes a terminal EE::error. --- src/helper/Site_Backup_Restore.php | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/helper/Site_Backup_Restore.php b/src/helper/Site_Backup_Restore.php index 6cd18dcc..c36635e9 100644 --- a/src/helper/Site_Backup_Restore.php +++ b/src/helper/Site_Backup_Restore.php @@ -176,7 +176,7 @@ public function backup( $args, $assoc_args = [] ) { $this->capture_error( 'Backup uploaded but EasyDash success callback failed; rolled back the orphaned upload.', self::ERROR_TYPE_NETWORK, - 4002 + 4004 ); $this->rollback_failed_backup(); EE::error( 'EasyDash success callback failed; the uploaded backup was rolled back. No backup was created.' ); @@ -1649,18 +1649,21 @@ private function rollback_failed_backup() { $result = EE::launch( sprintf( 'rclone purge %s', escapeshellarg( $this->dash_new_backup_path ) ) ); if ( $result->return_code ) { - // Rollback purge failed: an untracked backup is now orphaned on the - // remote. Surface it so the exit code and failure callback reflect this - // state instead of silently continuing past a warning. capture_error keeps - // the root cause if one was already recorded by the caller. + // Rollback purge failed: the untracked backup genuinely survives on the + // remote, so the operator must delete it manually. Force-overwrite any + // optimistic "rolled back" error captured by the caller (capture_error is + // first-wins) so EasyDash receives this accurate path, not the inverse. $message = sprintf( 'Failed to delete orphaned backup from remote storage. Please manually delete: %s', $this->dash_new_backup_path ); + $this->dash_error_message = ''; $this->capture_error( $message, self::ERROR_TYPE_FILESYSTEM, 4003 ); EE::error( $message ); } else { - EE::success( 'Successfully removed unregistered backup from remote storage.' ); + // Demoted from EE::success: this always precedes a terminal EE::error, so a + // green success line would be misleading on a failing command. + EE::log( 'Successfully removed unregistered backup from remote storage.' ); } } From 092e4aaf489f1a3feeae6cd30300ba8acef3db5d Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Thu, 24 Sep 2026 10:39:24 +0000 Subject: [PATCH 3/6] fix(backup): release global lock before EasyDash callback-failure rollback The callback-failure path now exits via EE::error, so the explicit lock release is skipped and the global backup lock stays held through the shutdown failure callback, which can retry for up to 15 minutes when EasyDash is unreachable. Release it before the rollback so queued backups are not delayed further. --- src/helper/Site_Backup_Restore.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/helper/Site_Backup_Restore.php b/src/helper/Site_Backup_Restore.php index c36635e9..17de0e85 100644 --- a/src/helper/Site_Backup_Restore.php +++ b/src/helper/Site_Backup_Restore.php @@ -178,6 +178,8 @@ public function backup( $args, $assoc_args = [] ) { self::ERROR_TYPE_NETWORK, 4004 ); + // Don't hold the lock through the shutdown failure callback's retries. + $this->release_global_backup_lock(); $this->rollback_failed_backup(); EE::error( 'EasyDash success callback failed; the uploaded backup was rolled back. No backup was created.' ); } From 855a0bf5044ecb85888f051349521919936e2f42 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Wed, 4 Nov 2026 06:34:14 +0530 Subject: [PATCH 4/6] chore(backup): say EasyEngine Dashboard in the callback failure messages Matches the EasyDash rename in #525 for the strings and comments this PR adds. --- src/helper/Site_Backup_Restore.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/helper/Site_Backup_Restore.php b/src/helper/Site_Backup_Restore.php index ef7c4ec2..d72d333a 100644 --- a/src/helper/Site_Backup_Restore.php +++ b/src/helper/Site_Backup_Restore.php @@ -166,24 +166,24 @@ public function backup( $args, $assoc_args = [] ) { ); if ( $api_success ) { - // Backup is now registered with EasyDash; only then is it safe to + // Backup is now registered with the EasyEngine Dashboard; only then is it safe to // mark complete (suppressing the shutdown failure callback) and prune. $this->dash_backup_completed = true; $this->cleanup_old_backups(); } else { - // Success callback failed: the upload is orphaned (EasyDash never + // Success callback failed: the upload is orphaned (the EasyEngine Dashboard never // recorded it), so roll it back and report failure. dash_backup_completed // stays false so the shutdown handler emits the single failure callback; // EE::error() makes the exit code/message reflect that no backup remains. $this->capture_error( - 'Backup uploaded but EasyDash success callback failed; rolled back the orphaned upload.', + 'Backup uploaded but EasyEngine Dashboard success callback failed; rolled back the orphaned upload.', self::ERROR_TYPE_NETWORK, 4004 ); // Don't hold the lock through the shutdown failure callback's retries. $this->release_global_backup_lock(); $this->rollback_failed_backup(); - EE::error( 'EasyDash success callback failed; the uploaded backup was rolled back. No backup was created.' ); + EE::error( 'EasyEngine Dashboard success callback failed; the uploaded backup was rolled back. No backup was created.' ); } } else { // Non-dash path: backup is done once the upload succeeds. @@ -1662,7 +1662,7 @@ private function rollback_failed_backup() { // Rollback purge failed: the untracked backup genuinely survives on the // remote, so the operator must delete it manually. Force-overwrite any // optimistic "rolled back" error captured by the caller (capture_error is - // first-wins) so EasyDash receives this accurate path, not the inverse. + // first-wins) so the EasyEngine Dashboard receives this accurate path, not the inverse. $message = sprintf( 'Failed to delete orphaned backup from remote storage. Please manually delete: %s', $this->dash_new_backup_path @@ -1761,7 +1761,7 @@ private function send_dash_success_callback( $ed_api_url, $backup_id, $verify_to $success = $this->send_dash_request( $endpoint, $payload ); - // A success that actually reached EasyDash is the terminal callback; a failed + // A success that actually reached the EasyEngine Dashboard is the terminal callback; a failed // attempt is not, so the failure path can still emit the failure callback. if ( $success ) { $this->dash_callback_sent = true; From cb082b0b2bc6418c9396e6af4c567955d55702da Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Wed, 4 Nov 2026 07:13:08 +0530 Subject: [PATCH 5/6] fix(backup): hold the global backup lock until the orphaned upload is purged Releasing it before the purge let another backup prune against the orphan and drop a valid backup below the retention count. It's still released before the failure callback's retries, on both the purge success and the purge failure path. --- src/helper/Site_Backup_Restore.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/helper/Site_Backup_Restore.php b/src/helper/Site_Backup_Restore.php index d72d333a..19a7aa4b 100644 --- a/src/helper/Site_Backup_Restore.php +++ b/src/helper/Site_Backup_Restore.php @@ -180,9 +180,10 @@ public function backup( $args, $assoc_args = [] ) { self::ERROR_TYPE_NETWORK, 4004 ); - // Don't hold the lock through the shutdown failure callback's retries. - $this->release_global_backup_lock(); + // Hold the lock through the purge so no other backup prunes against the orphan, + // but not through the shutdown failure callback's retries. $this->rollback_failed_backup(); + $this->release_global_backup_lock(); EE::error( 'EasyEngine Dashboard success callback failed; the uploaded backup was rolled back. No backup was created.' ); } } else { @@ -1669,6 +1670,7 @@ private function rollback_failed_backup() { ); $this->dash_error_message = ''; $this->capture_error( $message, self::ERROR_TYPE_FILESYSTEM, 4003 ); + $this->release_global_backup_lock(); EE::error( $message ); } else { // Demoted from EE::success: this always precedes a terminal EE::error, so a From 5f4b0ffd132b6f90d9bcd058d84a4fdf4bee0080 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 16 Nov 2026 16:20:25 +0530 Subject: [PATCH 6/6] fix(backup): keep the uploaded backup when the EasyEngine Dashboard callback fails A failed success callback no longer purges the backup it just uploaded: the backup is kept, and the failure callback and the command error both name its remote path. --- features/COVERAGE.md | 2 +- src/helper/Site_Backup_Restore.php | 62 ++++++------------------------ 2 files changed, 12 insertions(+), 52 deletions(-) diff --git a/features/COVERAGE.md b/features/COVERAGE.md index 2f4ced49..20343e1f 100644 --- a/features/COVERAGE.md +++ b/features/COVERAGE.md @@ -192,7 +192,7 @@ Legend: covered / partial / not covered (reason). Feature files are abbreviated: | restore latest, restore `--id`, unknown id | covered | backup: Restore the latest backup; Restore a specific backup by id; Restoring an unknown backup id fails | | disabled site | covered | backup: A disabled site cannot be backed up...; en-dis outline | | `--dash-auth` format and ed-api-url check | covered | backup: A malformed --dash-auth is rejected; --dash-auth needs ed-api-url in the config | -| `--dash-auth` success/failure callbacks, rollback of the uploaded backup | not covered | needs an EasyDash API endpoint (`ed-api-url`) | +| `--dash-auth` success/failure callbacks, uploaded backup kept when the success callback fails | not covered | needs an EasyEngine Dashboard API endpoint (`ed-api-url`) | | per-site lock blocks, lock left by a failed restore | covered | backup: A lock file of another backup blocks the backup (held with flock(1), so it holds with or without EasyEngine/site-command#480); not covered yet: a failed restore leaves the lock file and blocks later backups (its scenario comes back with the fix) | | stale download reuse | not covered yet | a restore by id reuses files an earlier restore left behind (its scenario comes back with the fix) | | retention (`no-of-backups`) | not covered yet | retention keeps one backup more than `no-of-backups` (its scenario comes back with the fix) | diff --git a/src/helper/Site_Backup_Restore.php b/src/helper/Site_Backup_Restore.php index 19a7aa4b..f6c8645e 100644 --- a/src/helper/Site_Backup_Restore.php +++ b/src/helper/Site_Backup_Restore.php @@ -43,8 +43,8 @@ class Site_Backup_Restore { private $dash_api_url; private $dash_backup_metadata; private $dash_backup_completed = false; - private $dash_callback_sent = false; // Guard: exactly one terminal callback (success XOR failure) may be emitted - private $dash_new_backup_path; // Track new backup path for potential rollback + private $dash_callback_sent = false; // Guard: at most one terminal callback (success XOR failure) may be emitted + private $dash_new_backup_path; // Remote path of this run's upload, reported if the success callback fails // Error tracking for EasyDash failure callbacks private $dash_error_message = ''; @@ -171,20 +171,16 @@ public function backup( $args, $assoc_args = [] ) { $this->dash_backup_completed = true; $this->cleanup_old_backups(); } else { - // Success callback failed: the upload is orphaned (the EasyEngine Dashboard never - // recorded it), so roll it back and report failure. dash_backup_completed - // stays false so the shutdown handler emits the single failure callback; - // EE::error() makes the exit code/message reflect that no backup remains. - $this->capture_error( - 'Backup uploaded but EasyEngine Dashboard success callback failed; rolled back the orphaned upload.', - self::ERROR_TYPE_NETWORK, - 4004 + // Never delete an uploaded backup because only the callback failed. dash_backup_completed stays + // false, so the shutdown handler sends the single failure callback with this message. + $message = sprintf( + 'Backup uploaded to %s but the EasyEngine Dashboard success callback failed; the backup was kept.', + $this->dash_new_backup_path ); - // Hold the lock through the purge so no other backup prunes against the orphan, - // but not through the shutdown failure callback's retries. - $this->rollback_failed_backup(); + $this->capture_error( $message, self::ERROR_TYPE_NETWORK, 4004 ); + // Don't hold the lock through the failure callback's retries in the shutdown handler. $this->release_global_backup_lock(); - EE::error( 'EasyEngine Dashboard success callback failed; the uploaded backup was rolled back. No backup was created.' ); + EE::error( $message ); } } else { // Non-dash path: backup is done once the upload succeeds. @@ -1516,7 +1512,7 @@ private function rclone_upload( $path ) { $remote_path = $output->stdout; EE::success( 'Backup uploaded to remote storage. Remote path: ' . $remote_path ); - // Store the new backup path for potential rollback (only when using dash-auth) + // Kept for the error reported if the success callback fails (only when using dash-auth) if ( $this->dash_auth_enabled ) { $this->dash_new_backup_path = $this->get_remote_path(); } @@ -1643,42 +1639,6 @@ private function cleanup_old_backups() { } } - /** - * Rollback (delete) the newly uploaded backup when EasyDash API callback fails. - * This prevents orphaned backups in remote storage that aren't tracked by EasyDash. - */ - private function rollback_failed_backup() { - if ( empty( $this->dash_new_backup_path ) ) { - EE::warning( 'Cannot rollback backup: backup path not found.' ); - - return; - } - - EE::warning( 'EasyDash API callback failed. Rolling back newly uploaded backup...' ); - EE::log( 'Deleting unregistered backup: ' . $this->dash_new_backup_path ); - - $result = EE::launch( sprintf( 'rclone purge %s', escapeshellarg( $this->dash_new_backup_path ) ) ); - - if ( $result->return_code ) { - // Rollback purge failed: the untracked backup genuinely survives on the - // remote, so the operator must delete it manually. Force-overwrite any - // optimistic "rolled back" error captured by the caller (capture_error is - // first-wins) so the EasyEngine Dashboard receives this accurate path, not the inverse. - $message = sprintf( - 'Failed to delete orphaned backup from remote storage. Please manually delete: %s', - $this->dash_new_backup_path - ); - $this->dash_error_message = ''; - $this->capture_error( $message, self::ERROR_TYPE_FILESYSTEM, 4003 ); - $this->release_global_backup_lock(); - EE::error( $message ); - } else { - // Demoted from EE::success: this always precedes a terminal EE::error, so a - // green success line would be misleading on a failing command. - EE::log( 'Successfully removed unregistered backup from remote storage.' ); - } - } - private function restore_nginx_conf( $backup_dir ) { $backup_file = $backup_dir . '/conf.zip';