Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion features/COVERAGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ Legend: covered / partial / not covered (reason). Feature files are abbreviated:
| restore latest, restore `--id`, unknown id | covered | backup: Restore the latest backup; Restore a specific backup by id; Restoring an unknown backup id fails |
| disabled site | covered | backup: A disabled site cannot be backed up...; en-dis outline |
| `--dash-auth` format and ed-api-url check | covered | backup: A malformed --dash-auth is rejected; --dash-auth needs ed-api-url in the config |
| `--dash-auth` success/failure callbacks, rollback of the uploaded backup | not covered | needs an EasyEngine Dashboard API endpoint (`ed-api-url`) |
| `--dash-auth` success/failure callbacks, uploaded backup kept when the success callback fails | not covered | needs an EasyEngine Dashboard API endpoint (`ed-api-url`) |
| per-site lock blocks, lock left by a failed restore | covered | backup: A lock file of another backup blocks the backup (held with flock(1), so it holds with or without EasyEngine/site-command#480); not covered yet: a failed restore leaves the lock file and blocks later backups (its scenario comes back with the fix) |
| stale download reuse | not covered yet | a restore by id reuses files an earlier restore left behind (its scenario comes back with the fix) |
| retention (`no-of-backups`) | not covered yet | retention keeps one backup more than `no-of-backups` (its scenario comes back with the fix) |
Expand Down
70 changes: 34 additions & 36 deletions src/helper/Site_Backup_Restore.php
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ class Site_Backup_Restore {
private $dash_api_url;
private $dash_backup_metadata;
private $dash_backup_completed = false;
private $dash_new_backup_path; // Track new backup path for potential rollback
private $dash_callback_sent = false; // Guard: at most one terminal callback (success XOR failure) may be emitted
private $dash_new_backup_path; // Remote path of this run's upload, reported if the success callback fails

// Error tracking for EasyEngine Dashboard failure callbacks
private $dash_error_message = '';
Expand Down Expand Up @@ -159,8 +160,6 @@ public function backup( $args, $assoc_args = [] ) {

$this->fs->remove( EE_BACKUP_DIR . '/' . $this->site_data['site_url'] . '.lock' );

// Mark backup as completed and send success callback
$this->dash_backup_completed = true;
if ( $this->dash_auth_enabled ) {
$api_success = $this->send_dash_success_callback(
$this->dash_api_url,
Expand All @@ -169,13 +168,26 @@ public function backup( $args, $assoc_args = [] ) {
$this->dash_backup_metadata
);

// Only cleanup old backups if API callback succeeded
// If API failed, rollback the newly uploaded backup
if ( $api_success ) {
// Backup is now registered with the EasyEngine Dashboard; only then is it safe to
// mark complete (suppressing the shutdown failure callback) and prune.
$this->dash_backup_completed = true;
$this->cleanup_old_backups();
} else {
$this->rollback_failed_backup();
// Never delete an uploaded backup because only the callback failed. dash_backup_completed stays
// false, so the shutdown handler sends the single failure callback with this message.
$message = sprintf(
'Backup uploaded to %s but the EasyEngine Dashboard success callback failed; the backup was kept.',
$this->dash_new_backup_path
);
$this->capture_error( $message, self::ERROR_TYPE_NETWORK, 4004 );
// Don't hold the lock through the failure callback's retries in the shutdown handler.
$this->release_global_backup_lock();
EE::error( $message );
}
} else {
// Non-dash path: backup is done once the upload succeeds.
$this->dash_backup_completed = true;
}

// Release global backup lock (also released by shutdown handler as safety net)
Expand All @@ -194,8 +206,9 @@ public function backup( $args, $assoc_args = [] ) {
* explicitly captured during backup execution.
*/
public function dash_shutdown_handler() {
// Only send failure callback if dash auth was enabled and backup didn't complete
if ( $this->dash_auth_enabled && ! $this->dash_backup_completed ) {
// Only send a failure callback if dash auth was enabled, the backup didn't
// complete, and no terminal callback (success or failure) has been sent yet.
if ( $this->dash_auth_enabled && ! $this->dash_backup_completed && ! $this->dash_callback_sent ) {

// If no error was captured yet, try to capture shutdown error
if ( empty( $this->dash_error_message ) ) {
Expand Down Expand Up @@ -1502,7 +1515,7 @@ private function rclone_upload( $path ) {
$remote_path = $output->stdout;
EE::success( 'Backup uploaded to remote storage. Remote path: ' . $remote_path );

// Store the new backup path for potential rollback (only when using dash-auth)
// Kept for the error reported if the success callback fails (only when using dash-auth)
if ( $this->dash_auth_enabled ) {
$this->dash_new_backup_path = $this->get_remote_path();
}
Expand Down Expand Up @@ -1629,32 +1642,6 @@ private function cleanup_old_backups() {
}
}

/**
* Rollback (delete) the newly uploaded backup when EasyEngine Dashboard API callback fails.
* This prevents orphaned backups in remote storage that aren't tracked by the EasyEngine Dashboard.
*/
private function rollback_failed_backup() {
if ( empty( $this->dash_new_backup_path ) ) {
EE::warning( 'Cannot rollback backup: backup path not found.' );

return;
}

EE::warning( 'EasyEngine Dashboard API callback failed. Rolling back newly uploaded backup...' );
EE::log( 'Deleting unregistered backup: ' . $this->dash_new_backup_path );

$result = EE::launch( sprintf( 'rclone purge %s', escapeshellarg( $this->dash_new_backup_path ) ) );

if ( $result->return_code ) {
EE::warning( sprintf(
'Failed to delete backup from remote storage. Please manually delete: %s',
$this->dash_new_backup_path
) );
} else {
EE::success( 'Successfully removed unregistered backup from remote storage.' );
}
}

private function restore_nginx_conf( $backup_dir ) {
$backup_file = $backup_dir . '/conf.zip';

Expand Down Expand Up @@ -2037,7 +2024,15 @@ private function send_dash_success_callback( $ed_api_url, $backup_id, $verify_to

EE::debug( 'Payload being sent: ' . json_encode( array_merge( $payload, [ 'verify' => '******' ] ) ) );

return $this->send_dash_request( $endpoint, $payload );
$success = $this->send_dash_request( $endpoint, $payload );

// A success that actually reached the EasyEngine Dashboard is the terminal callback; a failed
// attempt is not, so the failure path can still emit the failure callback.
if ( $success ) {
$this->dash_callback_sent = true;
Comment thread
mrrobot47 marked this conversation as resolved.
Resolved
}

return $success;
}

/**
Expand Down Expand Up @@ -2066,6 +2061,9 @@ private function send_dash_failure_callback( $ed_api_url, $backup_id, $verify_to
'error_code' => $payload['error_code'],
] ) );

// Failure is terminal: mark before sending so a concurrent shutdown pass
// can never emit a second (duplicate) terminal callback.
$this->dash_callback_sent = true;
$this->send_dash_request( $endpoint, $payload );
}

Expand Down