From d01788ac940d6874322889283ba3ca113ab0e8be Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:53:16 +0000 Subject: [PATCH 1/5] Add S3 target with multipart upload, retention helpers and bucket access check Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01EreR27QcUGyx3TeBypytVW --- package-lock.json | 547 +++++++++++++++++++++++++++++++++++++++++ package.json | 2 + plans/README.md | 2 +- plans/faz-03-s3.md | 2 +- src/commands/backup.ts | 14 ++ src/targets/index.ts | 14 +- src/targets/local.ts | 10 +- src/targets/names.ts | 9 + src/targets/s3.ts | 294 ++++++++++++++++++++++ src/targets/types.ts | 2 + 10 files changed, 883 insertions(+), 13 deletions(-) create mode 100644 src/targets/names.ts create mode 100644 src/targets/s3.ts diff --git a/package-lock.json b/package-lock.json index a9b4673..696a892 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,8 @@ "name": "github-backup-cli", "version": "0.1.0", "dependencies": { + "@aws-sdk/client-s3": "^3.1152.0", + "@aws-sdk/lib-storage": "^3.1152.0", "commander": "^15.0.0", "zod": "^4.6.7" }, @@ -25,6 +27,334 @@ "node": ">=22" } }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1001.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1001.3.tgz", + "integrity": "sha512-CcAkVyEUKGngbEqJdVdSc53Iy52+XzVL5NIcTen2BlK8UByNC1DThVtf5VKSrNzyA+SqyTvs+3S+knVBbOUizQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1152.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1152.0.tgz", + "integrity": "sha512-oI1+ZaM6pSOwZazu4nAb7rL8ug7LSjg31tDoJqRB9MooRqqWCf/qvkIZqne1mRA3xK3slg6K0RNWXaJYHbWLvg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1001.3", + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/credential-provider-node": "^3.972.86", + "@aws-sdk/middleware-sdk-s3": "^3.972.79", + "@aws-sdk/signature-v4-multi-region": "^3.996.48", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.978.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.3.tgz", + "integrity": "sha512-nslpzUCTI9b2IQMWSd6mSAXldBexPc8rr0d3dXUutXYFLFaTdsHa2GD7sJdsn3PhyjK0O5K8ypd6zi+RXXPVlg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.6", + "@aws-sdk/xml-builder": "^3.972.41", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.35.2", + "@smithy/signature-v4": "^5.7.4", + "@smithy/types": "^4.19.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.74", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.74.tgz", + "integrity": "sha512-rsRJcW2iFUhCgclyD6OSbMEaPl8Eu0qSeRZb2uuCK1ViZ66hl5wOe/p6Vr6MO3b+ydyKgFeqza2p6tZThvILlw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.76.tgz", + "integrity": "sha512-pnKdh0Br7OJPj+7qKjzgTfDTFW0RhEMKXNDR2jv9CAM4oHuG/4z3hRh8KtutLfpmItuRtDbKBmwWkDqHwwAWXw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.19.tgz", + "integrity": "sha512-kcjvAxqQEV8yuwkX58RelNn3ne65o7JgRWf3Z/gVvbi7Xuw1qSDLJlatrLD76+40cmRxmNkLxBoyp/vAQChQ5g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/credential-provider-env": "^3.972.74", + "@aws-sdk/credential-provider-http": "^3.972.76", + "@aws-sdk/credential-provider-login": "^3.972.81", + "@aws-sdk/credential-provider-process": "^3.972.74", + "@aws-sdk/credential-provider-sso": "^3.973.18", + "@aws-sdk/credential-provider-web-identity": "^3.972.80", + "@aws-sdk/nested-clients": "^3.997.48", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/credential-provider-imds": "^4.5.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.81", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.81.tgz", + "integrity": "sha512-CeQHSbE2CnVdaHIJ/NayRaCecIER+Pm9faUPkmRATO54rCoMTEPSbVieGHzEwvm1dZM2HIs6qQZCznhayNTWJQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/nested-clients": "^3.997.48", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.86", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.86.tgz", + "integrity": "sha512-WNttSyr6e8innPtBSty//CdnqJcgZN6eAbtw3v/CR2IObz6MYZEo8eusdtuCHJ2ujwFqC2bSPHVyhXGHx8HruQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.74", + "@aws-sdk/credential-provider-http": "^3.972.76", + "@aws-sdk/credential-provider-ini": "^3.973.19", + "@aws-sdk/credential-provider-process": "^3.972.74", + "@aws-sdk/credential-provider-sso": "^3.973.18", + "@aws-sdk/credential-provider-web-identity": "^3.972.80", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/credential-provider-imds": "^4.5.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.74", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.74.tgz", + "integrity": "sha512-Yj6kZFJQdbR+PygTkqdVLPHHpdDpOS9GCYVcDCuweCRRig9LF24ELQBMfyUw1vspvrZJUfnOGVyMjhaL094ahw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.18.tgz", + "integrity": "sha512-FnpPhdY4qUGtX/8X6eKxh+uD+9Y19i9FYibgk4SBe0M935s9vocHkHzaxaMNk+YkHmvzgdu8cHA5SCoeU+kkew==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/nested-clients": "^3.997.48", + "@aws-sdk/token-providers": "3.1150.0", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.80", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.80.tgz", + "integrity": "sha512-TJF25Ndwy2HxcVj6oxMgp8qVlJAawlExHYBXqhuC6Zt2m+A7WpdkjBD6QglNbiArlteYljZHEY3R7KKmno6ISA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/nested-clients": "^3.997.48", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/lib-storage": { + "version": "3.1152.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.1152.0.tgz", + "integrity": "sha512-UU1fDLKBd8az+4AmNx30ggnA4D4HolUd1MxT/J6ka76qKcckI0IxjAy6FpX/vwiUf2KDNxxN2YX0cfwjMqzWoQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.35.2", + "@smithy/types": "^4.19.0", + "buffer": "5.6.0", + "events": "3.3.0", + "stream-browserify": "3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.1152.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.79", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.79.tgz", + "integrity": "sha512-UfgAQosf4IEXldMsyntVuHsJnaQzlH6pmkOr6dQy7CaOYSlGz3mWiW4Di7XfQ/kPJP6xE7oMshz1V0E8RqGdRg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.48", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.48", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.48.tgz", + "integrity": "sha512-B01jdXJtAW/XOiUzGdI2mz67WyEHbGEc9X6Euzw3bjLAa+txTqdZDER4X2FsIEcXzV2ABqudVZ5thali8tCCzQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.48", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.48", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.48.tgz", + "integrity": "sha512-cFDXfEbXaGwbnpvR/DyqfxfjxpjXVRVthQLgkwvcfDS5u3+8G9WSj1X1Ag/pTRg/QOeOno1Tq3ycEdrKsCUcOA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.6", + "@smithy/signature-v4": "^5.7.4", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1150.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1150.0.tgz", + "integrity": "sha512-wDU36ofhabZYu+d2KSriyDbwzxNoaOp+4dG0p3qkabk7HeALR5nIWyQbjoyJZLn3iqWkv/xNdyUfbfevDVDzzg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/nested-clients": "^3.997.48", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.6", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.6.tgz", + "integrity": "sha512-v/clNZzZnDxGyvpHMOGpJKVXFAExJzUNAAjaWGdcx8QAcXLGwTaOkw33p5SHAi0YAioK32xB3hWwOekRVfmfKg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.41.tgz", + "integrity": "sha512-ctjVSyCMegrWfXlx6VqzSBFI6UqmQ5ZlnfMhdLIiWmhoH8UAQxSCP5N3OpG7X3k4LnS7ou74C4mt20+bfTW2aQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@biomejs/biome": { "version": "2.5.15", "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.15.tgz", @@ -947,6 +1277,87 @@ "license": "MIT", "peer": true }, + "node_modules/@smithy/core": { + "version": "3.36.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.36.0.tgz", + "integrity": "sha512-3hJiwG1F5t1SoXqp/yOCmYXziyx7ZVqF63S5W6YU201e/qwVaHPex+E/3vvyY9a3ORq/b+BydeNZpP2saQ9X2g==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz", + "integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", + "integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.4.tgz", + "integrity": "sha512-tHy0K0VtqNd5Y7Y41h0a0Lhh0L1GzC08dTWg0F7vRJWFtTENg7IZikf3wQkanYIRdb7ngoIPMTmqgUi401fEeQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz", + "integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -1370,6 +1781,42 @@ "node": ">=12" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "node_modules/chai": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/chai/-/chai-6.3.0.tgz", @@ -1459,6 +1906,15 @@ "@types/estree": "^1.0.0" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/expect-type": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", @@ -1502,6 +1958,32 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, "node_modules/lightningcss": { "version": "1.33.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", @@ -1870,6 +2352,20 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/rolldown": { "version": "1.2.14", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.14.tgz", @@ -1905,6 +2401,26 @@ "@rolldown/binding-win32-x64-msvc": "1.2.14" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/source-map-js": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", @@ -1923,6 +2439,25 @@ "dev": true, "license": "MIT" }, + "node_modules/stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "license": "MIT", + "dependencies": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/tinybench": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.2.tgz", @@ -1960,6 +2495,12 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/tsx": { "version": "4.23.15", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz", @@ -2021,6 +2562,12 @@ "dev": true, "license": "MIT" }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/vite": { "version": "8.3.4", "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.4.tgz", diff --git a/package.json b/package.json index ad31d40..a67909a 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "test": "vitest run" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1152.0", + "@aws-sdk/lib-storage": "^3.1152.0", "commander": "^15.0.0", "zod": "^4.6.7" }, diff --git a/plans/README.md b/plans/README.md index d000a57..fbb3ecd 100644 --- a/plans/README.md +++ b/plans/README.md @@ -46,7 +46,7 @@ Fazlar tablodaki sırayla uygulanır; sıranın tek kaynağı bu tablodur. Her f | --- | --- | --- | --- | | 1 | İskelet, config ve `list` | Tamamlandı | [faz-01-iskelet-ve-list.md](faz-01-iskelet-ve-list.md) | | 2 | Mirror, snapshot ve `local` hedefi | Tamamlandı | [faz-02-mirror-ve-local.md](faz-02-mirror-ve-local.md) | -| 3 | `s3` hedefi | Bekliyor | [faz-03-s3.md](faz-03-s3.md) | +| 3 | `s3` hedefi | Devam ediyor | [faz-03-s3.md](faz-03-s3.md) | | 4 | Git LFS | Bekliyor | [faz-04-lfs.md](faz-04-lfs.md) | | 5 | Dokümantasyon ve son kontrol | Bekliyor | [faz-05-dokuman-ve-son-kontrol.md](faz-05-dokuman-ve-son-kontrol.md) | diff --git a/plans/faz-03-s3.md b/plans/faz-03-s3.md index ec360de..43402f8 100644 --- a/plans/faz-03-s3.md +++ b/plans/faz-03-s3.md @@ -1,6 +1,6 @@ # Faz 3 — `s3` hedefi -**Durum:** Bekliyor +**Durum:** Devam ediyor ## Amaç diff --git a/src/commands/backup.ts b/src/commands/backup.ts index 052cb6c..96ad99c 100644 --- a/src/commands/backup.ts +++ b/src/commands/backup.ts @@ -5,6 +5,7 @@ import { resolveKeep } from "../core/retention.js"; import { backupRepos, type Summary } from "../core/run.js"; import type { ListFailure } from "../sources/types.js"; import { createTarget } from "../targets/index.js"; +import type { Target } from "../targets/types.js"; import { ConfigError, ExitCode, PrerequisiteError } from "../util/errors.js"; import type { Exec } from "../util/exec.js"; import { readToolInfo } from "../util/tool.js"; @@ -20,6 +21,18 @@ async function checkGit(exec: Exec): Promise { } } +/** Fails with exit code 2 when a target's storage is unreachable or rejects the credentials. */ +async function checkTargets(targets: Target[]): Promise { + for (const target of targets) { + try { + await target.checkAccess?.(); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + throw new PrerequisiteError(`target "${target.name}": ${reason}`); + } + } +} + /** Lines printed to stderr at the end of a run. */ export function renderSummary(summary: Summary, listFailures: ListFailure[]): string[] { const lines = [ @@ -62,6 +75,7 @@ export async function runBackup(options: BackupOptions, deps: CommandDeps): Prom createTarget(target, { keep: resolveKeep(target.keep, config.keep) }), ); await checkGit(deps.exec); + await checkTargets(targets); const controller = new AbortController(); const abort = () => controller.abort(); diff --git a/src/targets/index.ts b/src/targets/index.ts index 758e24a..afc82ef 100644 --- a/src/targets/index.ts +++ b/src/targets/index.ts @@ -1,6 +1,6 @@ import type { TargetConfig } from "../config/schema.js"; -import { ConfigError } from "../util/errors.js"; import { createLocalTarget } from "./local.js"; +import { createS3Target } from "./s3.js"; import type { Target } from "./types.js"; export type { Manifest, SnapshotFiles, Target } from "./types.js"; @@ -11,6 +11,16 @@ export function createTarget(config: TargetConfig, options: { keep: number }): T case "local": return createLocalTarget({ name: config.name, path: config.path, keep: options.keep }); case "s3": - throw new ConfigError(`target "${config.name}": s3 targets are not available yet`); + return createS3Target({ + name: config.name, + bucket: config.bucket, + prefix: config.prefix, + endpoint: config.endpoint, + region: config.region, + forcePathStyle: config.forcePathStyle, + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + keep: options.keep, + }); } } diff --git a/src/targets/local.ts b/src/targets/local.ts index 566bb3e..ed21134 100644 --- a/src/targets/local.ts +++ b/src/targets/local.ts @@ -1,6 +1,6 @@ import { copyFile, mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import path from "node:path"; -import { isStamp } from "../util/time.js"; +import { parseSnapshotName as parseName } from "./names.js"; import type { Manifest, SnapshotFiles, Target } from "./types.js"; export interface LocalTargetOptions { @@ -12,14 +12,6 @@ export interface LocalTargetOptions { const PARTIAL = ".partial"; -/** Splits `20260101T000000Z.bundle` into stamp and extension; null for other names. */ -function parseName(file: string): { stamp: string; ext: string } | null { - const dot = file.indexOf("."); - if (dot < 0) return null; - const stamp = file.slice(0, dot); - return isStamp(stamp) ? { stamp, ext: file.slice(dot) } : null; -} - async function listDir(dir: string): Promise { try { return await readdir(dir); diff --git a/src/targets/names.ts b/src/targets/names.ts new file mode 100644 index 0000000..c4daea5 --- /dev/null +++ b/src/targets/names.ts @@ -0,0 +1,9 @@ +import { isStamp } from "../util/time.js"; + +/** Splits `20260101T000000Z.bundle` into stamp and extension; null for other names. */ +export function parseSnapshotName(file: string): { stamp: string; ext: string } | null { + const dot = file.indexOf("."); + if (dot < 0) return null; + const stamp = file.slice(0, dot); + return isStamp(stamp) ? { stamp, ext: file.slice(dot) } : null; +} diff --git a/src/targets/s3.ts b/src/targets/s3.ts new file mode 100644 index 0000000..ed6ce15 --- /dev/null +++ b/src/targets/s3.ts @@ -0,0 +1,294 @@ +import { createReadStream } from "node:fs"; +import { + AbortMultipartUploadCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + HeadBucketCommand, + ListMultipartUploadsCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, + type S3ClientConfig, +} from "@aws-sdk/client-s3"; +import { Upload } from "@aws-sdk/lib-storage"; +import { ConfigError } from "../util/errors.js"; +import { parseSnapshotName } from "./names.js"; +import type { Manifest, SnapshotFiles, Target } from "./types.js"; + +export const DEFAULT_REGION = "us-east-1"; +/** Part size of multipart uploads: 16 MiB allows objects up to 160 GB (10 000 parts). */ +const PART_SIZE = 16 * 1024 * 1024; +/** `DeleteObjects` accepts at most 1000 keys per request. */ +const DELETE_BATCH = 1000; + +export interface S3TargetOptions { + name: string; + bucket: string; + prefix?: string; + endpoint?: string; + region?: string; + forcePathStyle?: boolean; + accessKeyId?: string; + secretAccessKey?: string; + keep: number; + /** Replaces the SDK client; used by tests. */ + client?: Pick; +} + +/** `/git/` and `git//backups/` become `git` and `git/backups`; blank becomes the bucket root. */ +export function normalizePrefix(prefix: string | undefined): string { + return (prefix ?? "").split("/").filter(Boolean).join("/"); +} + +/** Object key `//`; `name` is empty for the repository's "directory" prefix. */ +export function objectKey(prefix: string, repoKey: string, name = ""): string { + const segments = repoKey.split("/"); + if (segments.some((s) => s === "" || s === "." || s === "..")) { + throw new Error(`invalid repository key "${repoKey}"`); + } + return [prefix, repoKey, name].filter((part) => part !== "").join("/") + (name === "" ? "/" : ""); +} + +/** + * SDK client settings. Credentials come from the configuration (which already holds the + * `GHBACKUP_S3_*` fallback); without any, the SDK's default chain applies. + */ +export function s3ClientConfig( + options: Pick< + S3TargetOptions, + "name" | "endpoint" | "region" | "forcePathStyle" | "accessKeyId" | "secretAccessKey" + >, +): S3ClientConfig { + const { accessKeyId, secretAccessKey } = options; + if ((accessKeyId === undefined) !== (secretAccessKey === undefined)) { + throw new ConfigError( + `target "${options.name}": accessKeyId and secretAccessKey must be set together ` + + "(in the config or as GHBACKUP_S3_ACCESS_KEY_ID and GHBACKUP_S3_SECRET_ACCESS_KEY)", + ); + } + const config: S3ClientConfig = { + region: options.region ?? DEFAULT_REGION, + forcePathStyle: options.forcePathStyle ?? false, + // Since SDK 3.729 requests carry CRC32 checksums by default, which several S3-compatible + // services reject; only send them where the API requires them. + requestChecksumCalculation: "WHEN_REQUIRED", + responseChecksumValidation: "WHEN_REQUIRED", + }; + if (options.endpoint !== undefined) config.endpoint = options.endpoint; + if (accessKeyId !== undefined && secretAccessKey !== undefined) { + config.credentials = { accessKeyId, secretAccessKey }; + } + return config; +} + +export function maskSecrets(text: string, secrets: readonly (string | undefined)[]): string { + let out = text; + for (const secret of secrets) if (secret) out = out.split(secret).join("***"); + return out; +} + +/** One-line description of an SDK or network error with credentials masked. */ +export function describeS3Error(err: unknown, secrets: readonly (string | undefined)[]): string { + if (!(err instanceof Error)) return maskSecrets(String(err), secrets); + const meta = err as Error & { $metadata?: { httpStatusCode?: number }; Code?: string }; + const name = meta.Code ?? (err.name !== "Error" ? err.name : undefined); + const message = err.message && err.message !== "UnknownError" ? err.message : undefined; + const parts = [name, message].filter((part): part is string => Boolean(part)); + let text = parts.length > 0 ? parts.join(": ") : "unknown error"; + const status = meta.$metadata?.httpStatusCode; + if (status !== undefined) text += ` (HTTP ${status})`; + return maskSecrets(text, secrets); +} + +interface ListedObject { + key: string; + name: string; +} + +export function createS3Target(options: S3TargetOptions): Target { + const prefix = normalizePrefix(options.prefix); + const secrets = [options.accessKeyId, options.secretAccessKey]; + const client: Pick = + options.client ?? new S3Client(s3ClientConfig({ ...options, name: options.name })); + const { bucket } = options; + + /** Runs one operation and turns SDK errors into readable, credential-free ones. */ + async function guard(what: string, run: () => Promise): Promise { + try { + return await run(); + } catch (err) { + throw new Error(`${what}: ${describeS3Error(err, secrets)}`); + } + } + + /** Objects directly under the repository's prefix (all pages). */ + async function listRepo(repoKey: string): Promise { + const dir = objectKey(prefix, repoKey); + const found: ListedObject[] = []; + let token: string | undefined; + do { + const page = await client.send( + new ListObjectsV2Command({ Bucket: bucket, Prefix: dir, ContinuationToken: token }), + ); + for (const object of page.Contents ?? []) { + if (object.Key === undefined) continue; + const name = object.Key.slice(dir.length); + if (name !== "" && !name.includes("/")) found.push({ key: object.Key, name }); + } + token = page.IsTruncated ? page.NextContinuationToken : undefined; + } while (token !== undefined); + return found; + } + + const completeStamps = (objects: ListedObject[]): string[] => + objects + .map((o) => parseSnapshotName(o.name)) + .filter((p): p is { stamp: string; ext: string } => p?.ext === ".json") + .map((p) => p.stamp) + .sort(); + + async function upload(key: string, file: string): Promise { + const body = createReadStream(file); + try { + await new Upload({ + client: client as S3Client, + params: { Bucket: bucket, Key: key, Body: body }, + partSize: PART_SIZE, + queueSize: 4, + }).done(); + } finally { + body.destroy(); + } + } + + async function deleteKeys(keys: string[]): Promise { + for (let i = 0; i < keys.length; i += DELETE_BATCH) { + const result = await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: keys.slice(i, i + DELETE_BATCH).map((Key) => ({ Key })), Quiet: true }, + }), + ); + const failed = result.Errors?.[0]; + if (failed) { + throw new Error( + `${failed.Key ?? "object"}: ${failed.Code ?? "error"} ${failed.Message ?? ""}`.trim(), + ); + } + } + } + + /** Aborts unfinished multipart uploads under the repository prefix; best effort. */ + async function abortMultipartUploads(repoKey: string): Promise { + const dir = objectKey(prefix, repoKey); + try { + let keyMarker: string | undefined; + let uploadIdMarker: string | undefined; + do { + const page = await client.send( + new ListMultipartUploadsCommand({ + Bucket: bucket, + Prefix: dir, + KeyMarker: keyMarker, + UploadIdMarker: uploadIdMarker, + }), + ); + for (const u of page.Uploads ?? []) { + if (u.Key === undefined || u.UploadId === undefined) continue; + await client.send( + new AbortMultipartUploadCommand({ Bucket: bucket, Key: u.Key, UploadId: u.UploadId }), + ); + } + keyMarker = page.IsTruncated ? page.NextKeyMarker : undefined; + uploadIdMarker = page.IsTruncated ? page.NextUploadIdMarker : undefined; + } while (keyMarker !== undefined || uploadIdMarker !== undefined); + } catch { + // Not every S3-compatible service supports listing multipart uploads, and a policy may + // withhold it. Orphaned parts only cost storage and the snapshot itself is already safe. + } + } + + return { + name: options.name, + type: "s3", + keep: options.keep, + + async checkAccess() { + await guard(`cannot access bucket "${bucket}"`, () => + client.send(new HeadBucketCommand({ Bucket: bucket })), + ); + }, + + async latestManifest(repoKey) { + return guard(`${repoKey}: cannot read latest manifest`, async () => { + const latest = completeStamps(await listRepo(repoKey)).at(-1); + if (latest === undefined) return null; + const response = await client.send( + new GetObjectCommand({ + Bucket: bucket, + Key: objectKey(prefix, repoKey, `${latest}.json`), + }), + ); + try { + return JSON.parse((await response.Body?.transformToString()) ?? "") as Manifest; + } catch { + return null; + } + }); + }, + + async putSnapshot(repoKey, stamp, files: SnapshotFiles) { + await guard(`${repoKey}: cannot upload snapshot ${stamp}`, async () => { + await upload(objectKey(prefix, repoKey, `${stamp}.bundle`), files.bundle); + if (files.lfsTar) + await upload(objectKey(prefix, repoKey, `${stamp}.lfs.tar`), files.lfsTar); + await client.send( + new PutObjectCommand({ + Bucket: bucket, + Key: objectKey(prefix, repoKey, `${stamp}.json`), + Body: `${JSON.stringify(files.manifest, null, 2)}\n`, + ContentType: "application/json", + }), + ); + }); + }, + + async listSnapshots(repoKey) { + return guard(`${repoKey}: cannot list snapshots`, async () => + completeStamps(await listRepo(repoKey)), + ); + }, + + async deleteSnapshot(repoKey, stamp) { + await guard(`${repoKey}: cannot delete snapshot ${stamp}`, async () => { + // The manifest goes first so a half-deleted snapshot never looks complete. + await client.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: objectKey(prefix, repoKey, `${stamp}.json`), + }), + ); + const rest = (await listRepo(repoKey)) + .filter((o) => parseSnapshotName(o.name)?.stamp === stamp) + .map((o) => o.key); + await deleteKeys(rest); + }); + }, + + async cleanupIncomplete(repoKey) { + await guard(`${repoKey}: cannot clean up incomplete uploads`, async () => { + const objects = await listRepo(repoKey); + const complete = new Set(completeStamps(objects)); + const orphans = objects + .filter((o) => { + const parsed = parseSnapshotName(o.name); + return parsed !== null && !complete.has(parsed.stamp); + }) + .map((o) => o.key); + await deleteKeys(orphans); + await abortMultipartUploads(repoKey); + }); + }, + }; +} diff --git a/src/targets/types.ts b/src/targets/types.ts index d3b28d2..8be8ff9 100644 --- a/src/targets/types.ts +++ b/src/targets/types.ts @@ -40,4 +40,6 @@ export interface Target { deleteSnapshot(repoKey: string, stamp: string): Promise; /** Removes leftovers of interrupted uploads (files without a manifest). */ cleanupIncomplete(repoKey: string): Promise; + /** Fails with a readable reason when the storage cannot be reached; run before a backup starts. */ + checkAccess?(): Promise; } From 76655c065888398e4c3eb7614ec197b7e13e9b6e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:54:27 +0000 Subject: [PATCH 2/5] Add unit tests for the S3 target Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01EreR27QcUGyx3TeBypytVW --- test/unit/s3-target.test.ts | 353 ++++++++++++++++++++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 test/unit/s3-target.test.ts diff --git a/test/unit/s3-target.test.ts b/test/unit/s3-target.test.ts new file mode 100644 index 0000000..7e038d9 --- /dev/null +++ b/test/unit/s3-target.test.ts @@ -0,0 +1,353 @@ +import { writeFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { loadConfig } from "../../src/config/load.js"; +import { createProgram } from "../../src/program.js"; +import { + createS3Target, + describeS3Error, + maskSecrets, + normalizePrefix, + objectKey, + s3ClientConfig, +} from "../../src/targets/s3.js"; +import { ConfigError } from "../../src/util/errors.js"; +import { exec } from "../../src/util/exec.js"; +import { makeTempDir } from "../helpers/git.js"; + +describe("normalizePrefix", () => { + it.each([ + [undefined, ""], + ["", ""], + ["/", ""], + ["git", "git"], + ["/git/", "git"], + ["git//backups/", "git/backups"], + ["///a///b///", "a/b"], + ])("%j -> %j", (input, expected) => { + expect(normalizePrefix(input)).toBe(expected); + }); +}); + +describe("objectKey", () => { + it.each([ + ["git", "github/o/r", "20260101T000000Z.bundle", "git/github/o/r/20260101T000000Z.bundle"], + ["", "github/o/r", "20260101T000000Z.json", "github/o/r/20260101T000000Z.json"], + ["git", "github/o/r", undefined, "git/github/o/r/"], + ["", "github/o/r.wiki", undefined, "github/o/r.wiki/"], + ])("prefix %j, repo %j, name %j", (prefix, repoKey, name, expected) => { + expect(objectKey(prefix, repoKey, name)).toBe(expected); + }); + + it.each(["../x", "github/../x", "/abs", "github//r", "./x"])("rejects %j", (repoKey) => { + expect(() => objectKey("p", repoKey)).toThrow(/invalid repository key/); + }); +}); + +describe("s3ClientConfig", () => { + const base = { name: "bucket" }; + + it("defaults to us-east-1, virtual-host style and no explicit credentials", () => { + const config = s3ClientConfig(base); + expect(config.region).toBe("us-east-1"); + expect(config.forcePathStyle).toBe(false); + expect(config.credentials).toBeUndefined(); + expect(config.endpoint).toBeUndefined(); + }); + + it("only sends checksums where the API requires them", () => { + const config = s3ClientConfig(base); + expect(config.requestChecksumCalculation).toBe("WHEN_REQUIRED"); + expect(config.responseChecksumValidation).toBe("WHEN_REQUIRED"); + }); + + it("passes endpoint, region, path style and static credentials", () => { + const config = s3ClientConfig({ + ...base, + endpoint: "http://localhost:9000", + region: "auto", + forcePathStyle: true, + accessKeyId: "id", + secretAccessKey: "secret", + }); + expect(config).toMatchObject({ + endpoint: "http://localhost:9000", + region: "auto", + forcePathStyle: true, + credentials: { accessKeyId: "id", secretAccessKey: "secret" }, + }); + }); + + it.each([{ accessKeyId: "id" }, { secretAccessKey: "secret" }])( + "rejects half a key pair: %j", + (keys) => { + expect(() => s3ClientConfig({ ...base, ...keys })).toThrow(ConfigError); + }, + ); +}); + +describe("credential precedence (config > GHBACKUP_S3_* env > SDK chain)", () => { + function load(target: Record, env: Record) { + const dir = makeTempDir("ghb-s3cfg-"); + const file = path.join(dir, "config.json"); + writeFileSync( + file, + JSON.stringify({ targets: [{ type: "s3", name: "s", bucket: "b", ...target }] }), + ); + writeFileSync(path.join(dir, ".keep"), ""); + const { config, secrets } = loadConfig({ + flags: { config: file }, + env, + homeDir: dir, + cwd: dir, + }); + const s3 = config.targets[0]; + if (s3?.type !== "s3") throw new Error("expected an s3 target"); + return { s3, secrets }; + } + + it("prefers the configuration file", () => { + const { s3 } = load( + { accessKeyId: "file-id", secretAccessKey: "file-secret" }, + { GHBACKUP_S3_ACCESS_KEY_ID: "env-id", GHBACKUP_S3_SECRET_ACCESS_KEY: "env-secret" }, + ); + expect(s3).toMatchObject({ accessKeyId: "file-id", secretAccessKey: "file-secret" }); + }); + + it("falls back to the environment, and masks what it finds", () => { + const { s3, secrets } = load( + {}, + { GHBACKUP_S3_ACCESS_KEY_ID: "env-id", GHBACKUP_S3_SECRET_ACCESS_KEY: "env-secret" }, + ); + expect(s3).toMatchObject({ accessKeyId: "env-id", secretAccessKey: "env-secret" }); + expect(secrets).toEqual(["env-id", "env-secret"]); + }); + + it("leaves credentials unset for the SDK chain when neither is given", () => { + const { s3 } = load({}, {}); + expect(s3.accessKeyId).toBeUndefined(); + expect(s3ClientConfig(s3).credentials).toBeUndefined(); + }); +}); + +describe("error descriptions", () => { + it("masks credentials", () => { + expect(maskSecrets("id=AKIA1 key=s3cr3t", ["AKIA1", "s3cr3t", undefined, ""])).toBe( + "id=*** key=***", + ); + }); + + it("combines name, message and HTTP status and masks keys", () => { + const err = Object.assign(new Error("The key AKIA1 does not exist"), { + name: "InvalidAccessKeyId", + $metadata: { httpStatusCode: 403 }, + }); + expect(describeS3Error(err, ["AKIA1"])).toBe( + "InvalidAccessKeyId: The key *** does not exist (HTTP 403)", + ); + }); + + it("drops the SDK's placeholder message", () => { + const err = Object.assign(new Error("UnknownError"), { + name: "NotFound", + $metadata: { httpStatusCode: 404 }, + }); + expect(describeS3Error(err, [])).toBe("NotFound (HTTP 404)"); + }); +}); + +/** Minimal in-memory client that understands the commands the target sends. */ +function fakeClient(initial: string[], pageSize = 1000) { + const keys = new Set(initial); + const calls: Array<{ name: string; input: Record }> = []; + // biome-ignore lint/suspicious/noExplicitAny: loosely typed stand-in for SDK commands + const send = async (command: { constructor: { name: string }; input: Record }) => { + const name = command.constructor.name; + const input = command.input; + calls.push({ name, input }); + switch (name) { + case "ListObjectsV2Command": { + const all = [...keys].filter((k) => k.startsWith(input.Prefix)).sort(); + const start = input.ContinuationToken ? Number(input.ContinuationToken) : 0; + const page = all.slice(start, start + pageSize); + const more = start + pageSize < all.length; + return { + Contents: page.map((Key) => ({ Key })), + IsTruncated: more, + NextContinuationToken: more ? String(start + pageSize) : undefined, + }; + } + case "DeleteObjectCommand": + keys.delete(input.Key); + return {}; + case "DeleteObjectsCommand": + for (const { Key } of input.Delete.Objects) keys.delete(Key); + return {}; + case "ListMultipartUploadsCommand": + throw Object.assign(new Error("not supported"), { name: "NotImplemented" }); + case "GetObjectCommand": + return { + Body: { transformToString: async () => JSON.stringify({ stamp: "from-fake" }) }, + }; + default: + throw new Error(`unexpected command ${name}`); + } + }; + return { client: { send } as never, keys, calls }; +} + +const stampKey = (stamp: string, ext: string) => `git/github/o/r/${stamp}.${ext}`; + +describe("s3 target against a fake client", () => { + const options = { name: "s", bucket: "b", prefix: "/git/", keep: 3 }; + + it("follows continuation tokens when listing snapshots", async () => { + const initial = []; + for (let i = 1; i <= 5; i++) initial.push(stampKey(`2026010${i}T000000Z`, "json")); + for (let i = 1; i <= 5; i++) initial.push(stampKey(`2026010${i}T000000Z`, "bundle")); + const { client, calls } = fakeClient(initial, 3); + const target = createS3Target({ ...options, client }); + + const stamps = await target.listSnapshots("github/o/r"); + expect(stamps).toEqual([1, 2, 3, 4, 5].map((i) => `2026010${i}T000000Z`)); + expect(calls.filter((c) => c.name === "ListObjectsV2Command")).toHaveLength(4); + }); + + it("treats a snapshot without a manifest as incomplete and ignores other repositories", async () => { + const { client } = fakeClient([ + stampKey("20260101T000000Z", "bundle"), + stampKey("20260101T000000Z", "json"), + stampKey("20260102T000000Z", "bundle"), + "git/github/o/r.wiki/20260109T000000Z.json", + "git/github/o/r/nested/20260110T000000Z.json", + ]); + const target = createS3Target({ ...options, client }); + expect(await target.listSnapshots("github/o/r")).toEqual(["20260101T000000Z"]); + expect(await target.latestManifest("github/o/r")).toEqual({ stamp: "from-fake" }); + expect(await target.latestManifest("github/o/none")).toBeNull(); + }); + + it("deletes the manifest first, then the rest in groups of 1000", async () => { + const stamp = "20260101T000000Z"; + const initial = [ + stampKey(stamp, "json"), + stampKey(stamp, "bundle"), + stampKey(stamp, "lfs.tar"), + ]; + // 2500 more objects with the same stamp so deletion needs three batches. + for (let i = 0; i < 2500; i++) initial.push(`git/github/o/r/${stamp}.part${i}`); + initial.push(stampKey("20260102T000000Z", "json")); + const { client, keys, calls } = fakeClient(initial); + const target = createS3Target({ ...options, client }); + + await target.deleteSnapshot("github/o/r", stamp); + + const deletes = calls.filter((c) => c.name.startsWith("Delete")); + expect(deletes[0]?.name).toBe("DeleteObjectCommand"); + expect(deletes[0]?.input.Key).toBe(stampKey(stamp, "json")); + const batches = deletes + .slice(1) + .map((c) => (c.input.Delete as { Objects: unknown[] }).Objects.length); + expect(batches).toEqual([1000, 1000, 502]); + expect([...keys]).toEqual([stampKey("20260102T000000Z", "json")]); + }); + + it("reports DeleteObjects failures", async () => { + const stamp = "20260101T000000Z"; + const { client } = fakeClient([stampKey(stamp, "json"), stampKey(stamp, "bundle")]); + const original = (client as { send: (c: unknown) => Promise }).send; + (client as { send: unknown }).send = async (c: { constructor: { name: string } }) => + c.constructor.name === "DeleteObjectsCommand" + ? { Errors: [{ Key: "k", Code: "AccessDenied", Message: "no" }] } + : original(c); + const target = createS3Target({ ...options, client }); + await expect(target.deleteSnapshot("github/o/r", stamp)).rejects.toThrow( + /cannot delete snapshot.*AccessDenied/, + ); + }); + + it("removes only objects of snapshots without a manifest", async () => { + const { client, keys } = fakeClient([ + stampKey("20260101T000000Z", "bundle"), + stampKey("20260101T000000Z", "json"), + stampKey("20260102T000000Z", "bundle"), + stampKey("20260102T000000Z", "lfs.tar"), + "git/github/o/r/notes.txt", + "git/github/o/r.wiki/20260102T000000Z.bundle", + ]); + const target = createS3Target({ ...options, client }); + // The fake rejects ListMultipartUploads; that must not fail the cleanup. + await target.cleanupIncomplete("github/o/r"); + expect([...keys].sort()).toEqual([ + "git/github/o/r.wiki/20260102T000000Z.bundle", + "git/github/o/r/20260101T000000Z.bundle", + "git/github/o/r/20260101T000000Z.json", + "git/github/o/r/notes.txt", + ]); + }); + + it("rejects repository keys that escape the prefix", async () => { + const { client } = fakeClient([]); + const target = createS3Target({ ...options, client }); + await expect(target.listSnapshots("../outside")).rejects.toThrow(/invalid repository key/); + }); + + it("names the bucket and hides credentials when the access check fails", async () => { + const client = { + send: async () => { + throw Object.assign(new Error("bad key AKIA1 / s3cr3t"), { + name: "InvalidAccessKeyId", + $metadata: { httpStatusCode: 403 }, + }); + }, + } as never; + const target = createS3Target({ + ...options, + accessKeyId: "AKIA1", + secretAccessKey: "s3cr3t", + client, + }); + const error = await target.checkAccess?.().catch((e: Error) => e); + expect((error as Error).message).toBe( + 'cannot access bucket "b": InvalidAccessKeyId: bad key *** / *** (HTTP 403)', + ); + }); +}); + +describe("backup with an unreachable bucket", () => { + it("exits with 2 before doing any work and never prints the credentials", async () => { + const root = makeTempDir("ghb-s3fail-"); + const configFile = path.join(root, "config.json"); + writeFileSync( + configFile, + JSON.stringify({ + targets: [ + { + type: "s3", + name: "remote", + bucket: "nope", + endpoint: "http://127.0.0.1:1", + forcePathStyle: true, + accessKeyId: "AKIAUNITTESTID", + secretAccessKey: "unit-test-secret", + }, + ], + }), + ); + const err: string[] = []; + const code = await createProgram({ + exec, + env: {}, + stdout: () => {}, + stderr: (line) => err.push(line), + homeDir: root, + cwd: root, + }).run(["backup", "--config", configFile, "--cache-dir", path.join(root, "cache")]); + + const output = err.join("\n"); + expect(code).toBe(2); + expect(output).toContain('target "remote": cannot access bucket "nope"'); + expect(output).not.toContain("AKIAUNITTESTID"); + expect(output).not.toContain("unit-test-secret"); + }, 30_000); +}); From 677de3a09b6573b2345481315e0a430634f15ee0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:55:18 +0000 Subject: [PATCH 3/5] Add S3 integration tests that run against an S3-compatible server Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01EreR27QcUGyx3TeBypytVW --- test/integration/s3-target.test.ts | 397 +++++++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 test/integration/s3-target.test.ts diff --git a/test/integration/s3-target.test.ts b/test/integration/s3-target.test.ts new file mode 100644 index 0000000..8debe60 --- /dev/null +++ b/test/integration/s3-target.test.ts @@ -0,0 +1,397 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { + CreateBucketCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from "@aws-sdk/client-s3"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import type { CommandDeps } from "../../src/commands/context.js"; +import { createProgram } from "../../src/program.js"; +import { createTarget } from "../../src/targets/index.js"; +import { s3ClientConfig } from "../../src/targets/s3.js"; +import type { Manifest, Target } from "../../src/targets/types.js"; +import { ExitCode } from "../../src/util/errors.js"; +import { formatStamp } from "../../src/util/time.js"; +import { fakeSource, isolatedExec, makeTempDir, repoInfo, TestRemote } from "../helpers/git.js"; + +/** + * Runs against any S3-compatible server (CI uses S3Proxy): set GHBACKUP_TEST_S3_ENDPOINT, + * GHBACKUP_TEST_S3_ACCESS_KEY_ID, GHBACKUP_TEST_S3_SECRET_ACCESS_KEY and optionally + * GHBACKUP_TEST_S3_BUCKET. Skipped otherwise. + */ +const endpoint = process.env.GHBACKUP_TEST_S3_ENDPOINT; +const accessKeyId = process.env.GHBACKUP_TEST_S3_ACCESS_KEY_ID ?? "test-access"; +const secretAccessKey = process.env.GHBACKUP_TEST_S3_SECRET_ACCESS_KEY ?? "test-secret"; +const bucket = process.env.GHBACKUP_TEST_S3_BUCKET ?? "ghb-test"; + +const KEY = "github/owner/repo"; + +const manifest = (stamp: string, refsHash = "h"): Manifest => ({ + version: 1, + source: "github", + fullName: "owner/repo", + stamp, + refsHash, + refs: [], + bundleBytes: 3, + tool: { name: "t", version: "1" }, +}); + +describe.skipIf(!endpoint)("s3 target (integration)", () => { + const s3 = new S3Client( + s3ClientConfig({ name: "test", endpoint, forcePathStyle: true, accessKeyId, secretAccessKey }), + ); + let root: string; + let prefix: string; + + const s3Config = (extra: Record = {}) => ({ + type: "s3" as const, + name: "bucket", + bucket, + prefix, + endpoint, + forcePathStyle: true, + accessKeyId, + secretAccessKey, + ...extra, + }); + const makeTarget = (keep = 2, extra: Record = {}): Target => + createTarget(s3Config(extra), { keep }); + + async function keysUnder(under: string): Promise { + const keys: string[] = []; + let token: string | undefined; + do { + const page = await s3.send( + new ListObjectsV2Command({ Bucket: bucket, Prefix: under, ContinuationToken: token }), + ); + for (const o of page.Contents ?? []) if (o.Key) keys.push(o.Key); + token = page.IsTruncated ? page.NextContinuationToken : undefined; + } while (token); + return keys.sort(); + } + const put = (key: string, body = "x") => + s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body })); + + beforeAll(async () => { + try { + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + } catch (err) { + const name = (err as Error).name; + if (name !== "BucketAlreadyOwnedByYou" && name !== "BucketAlreadyExists") throw err; + } + }); + afterAll(() => s3.destroy()); + + beforeEach(() => { + root = makeTempDir("ghb-s3-"); + prefix = `ghb-test/${randomUUID()}`; + writeFileSync(path.join(root, "a.bundle"), "abc"); + writeFileSync(path.join(root, "a.tar"), "tar"); + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + const files = (extra: { lfsTar?: string } = {}) => ({ + bundle: path.join(root, "a.bundle"), + ...extra, + }); + + it("is empty before the first snapshot", async () => { + const target = makeTarget(); + expect(await target.latestManifest(KEY)).toBeNull(); + expect(await target.listSnapshots(KEY)).toEqual([]); + await target.cleanupIncomplete(KEY); + await target.checkAccess?.(); + }); + + it("stores bundle, lfs.tar and manifest under the shared layout", async () => { + const target = makeTarget(); + await target.putSnapshot(KEY, "20260101T000000Z", { + ...files({ lfsTar: path.join(root, "a.tar") }), + manifest: manifest("20260101T000000Z"), + }); + expect(await keysUnder(`${prefix}/`)).toEqual( + ["bundle", "json", "lfs.tar"].map((ext) => `${prefix}/${KEY}/20260101T000000Z.${ext}`), + ); + expect(await target.latestManifest(KEY)).toEqual(manifest("20260101T000000Z")); + }); + + it("normalizes slashes around the prefix", async () => { + const target = makeTarget(2, { prefix: `/${prefix}//` }); + await target.putSnapshot(KEY, "20260101T000000Z", { + ...files(), + manifest: manifest("20260101T000000Z"), + }); + expect(await keysUnder(`${prefix}/`)).toContain(`${prefix}/${KEY}/20260101T000000Z.json`); + }); + + it("uploads large bundles in several parts and downloads them intact", async () => { + const big = path.join(root, "big.bundle"); + writeFileSync(big, randomBytes(20 * 1024 * 1024)); + const target = makeTarget(); + await target.putSnapshot(KEY, "20260101T000000Z", { + bundle: big, + manifest: manifest("20260101T000000Z"), + }); + const { GetObjectCommand } = await import("@aws-sdk/client-s3"); + const object = await s3.send( + new GetObjectCommand({ Bucket: bucket, Key: `${prefix}/${KEY}/20260101T000000Z.bundle` }), + ); + const bytes = await object.Body?.transformToByteArray(); + expect(Buffer.from(bytes ?? []).equals(readFileSync(big))).toBe(true); + }, 60_000); + + it("returns the newest manifest and lists stamps oldest first", async () => { + const target = makeTarget(); + for (const [stamp, hash] of [ + ["20260102T000000Z", "two"], + ["20260101T000000Z", "one"], + ] as const) { + await target.putSnapshot(KEY, stamp, { ...files(), manifest: manifest(stamp, hash) }); + } + expect(await target.listSnapshots(KEY)).toEqual(["20260101T000000Z", "20260102T000000Z"]); + expect((await target.latestManifest(KEY))?.refsHash).toBe("two"); + }); + + it("deletes every object of a snapshot and nothing else", async () => { + const target = makeTarget(); + for (const stamp of ["20260101T000000Z", "20260102T000000Z"]) { + await target.putSnapshot(KEY, stamp, { + ...files({ lfsTar: path.join(root, "a.tar") }), + manifest: manifest(stamp), + }); + } + await target.deleteSnapshot(KEY, "20260101T000000Z"); + expect((await keysUnder(`${prefix}/`)).every((k) => k.includes("20260102T000000Z"))).toBe(true); + expect(await target.listSnapshots(KEY)).toEqual(["20260102T000000Z"]); + }); + + it("removes objects of interrupted snapshots but keeps complete ones and other files", async () => { + const target = makeTarget(); + await target.putSnapshot(KEY, "20260101T000000Z", { + ...files(), + manifest: manifest("20260101T000000Z"), + }); + await put(`${prefix}/${KEY}/20260102T000000Z.bundle`, "half"); + await put(`${prefix}/${KEY}/20260102T000000Z.lfs.tar`, "half"); + await put(`${prefix}/${KEY}/notes.txt`, "keep me"); + expect(await target.listSnapshots(KEY)).toEqual(["20260101T000000Z"]); + await target.cleanupIncomplete(KEY); + expect(await keysUnder(`${prefix}/`)).toEqual([ + `${prefix}/${KEY}/20260101T000000Z.bundle`, + `${prefix}/${KEY}/20260101T000000Z.json`, + `${prefix}/${KEY}/notes.txt`, + ]); + }); + + it("lists more than 1000 objects across pages", async () => { + const target = makeTarget(); + const start = Date.UTC(2026, 0, 1); + const stamps = Array.from({ length: 1005 }, (_, i) => formatStamp(new Date(start + i * 1000))); + for (let i = 0; i < stamps.length; i += 50) { + await Promise.all( + stamps + .slice(i, i + 50) + .map((stamp) => put(`${prefix}/${KEY}/${stamp}.json`, JSON.stringify(manifest(stamp)))), + ); + } + expect(await target.listSnapshots(KEY)).toEqual(stamps); + expect((await target.latestManifest(KEY))?.stamp).toBe(stamps.at(-1)); + + await target.cleanupIncomplete(KEY); + expect(await target.listSnapshots(KEY)).toHaveLength(1005); + }, 120_000); + + it("deletes a snapshot with more than 1000 matching objects", async () => { + const target = makeTarget(); + const stamp = "20260101T000000Z"; + await target.putSnapshot(KEY, stamp, { ...files(), manifest: manifest(stamp) }); + await Promise.all( + Array.from({ length: 1005 }, (_, i) => put(`${prefix}/${KEY}/${stamp}.extra${i}`)), + ); + await target.deleteSnapshot(KEY, stamp); + expect(await keysUnder(`${prefix}/`)).toEqual([]); + }, 120_000); + + it("fails the access check for a missing bucket, naming it", async () => { + const target = createTarget(s3Config({ bucket: `missing-${randomUUID()}` }), { keep: 2 }); + await expect(target.checkAccess?.()).rejects.toThrow(/cannot access bucket "missing-/); + }); + + it("fails the access check for wrong credentials without leaking them", async () => { + const target = createTarget(s3Config({ secretAccessKey: "definitely-wrong-secret" }), { + keep: 2, + }); + const error = await target.checkAccess?.().catch((e: Error) => e); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch(/cannot access bucket/); + expect((error as Error).message).not.toContain("definitely-wrong-secret"); + }); +}); + +describe.skipIf(!endpoint)("backup to local and s3 targets (integration)", () => { + let root: string; + let home: string; + let clock: number; + let prefix: string; + const s3 = new S3Client( + s3ClientConfig({ name: "test", endpoint, forcePathStyle: true, accessKeyId, secretAccessKey }), + ); + + beforeAll(async () => { + try { + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + } catch { + // Created by the other suite or earlier runs. + } + }); + afterAll(() => s3.destroy()); + + beforeEach(() => { + root = makeTempDir("ghb-s3run-"); + home = path.join(root, "home"); + clock = Date.UTC(2026, 0, 1); + prefix = `ghb-test/${randomUUID()}`; + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + function setup( + repos: ReturnType[], + targetOverrides: Record = {}, + env: Record = {}, + ) { + const configFile = path.join(root, "config.json"); + writeFileSync( + configFile, + JSON.stringify({ + keep: 7, + targets: [ + { type: "local", name: "disk", path: path.join(root, "out") }, + { + type: "s3", + name: "bucket", + bucket, + prefix, + endpoint, + forcePathStyle: true, + accessKeyId, + secretAccessKey, + ...targetOverrides, + }, + ], + }), + ); + const out: string[] = []; + const err: string[] = []; + const deps: CommandDeps = { + exec: isolatedExec(home), + env, + stdout: (text) => out.push(text), + stderr: (line) => err.push(line), + homeDir: home, + cwd: root, + createSource: () => fakeSource(repos), + now: () => { + clock += 1000; + return new Date(clock); + }, + sleep: async () => {}, + }; + const backup = async (...flags: string[]) => { + out.length = 0; + const code = await createProgram(deps).run([ + "backup", + "--config", + configFile, + "--cache-dir", + path.join(root, "cache"), + "--json", + ...flags, + ]); + return { code, json: out.length ? JSON.parse(out.join("")) : undefined }; + }; + return { backup, err }; + } + + const localDir = (key: string) => path.join(root, "out", key); + const localFiles = (key: string) => readdirSync(localDir(key)).sort(); + async function s3Names(key: string): Promise { + const dir = `${prefix}/${key}/`; + const page = await s3.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: dir })); + return (page.Contents ?? []).map((o) => (o.Key ?? "").slice(dir.length)).sort(); + } + async function s3Text(key: string): Promise { + const { GetObjectCommand } = await import("@aws-sdk/client-s3"); + const object = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + return (await object.Body?.transformToString()) ?? ""; + } + + it("writes the same file names and manifests to both targets", async () => { + const remote = new TestRemote(root, "a", home); + remote.commit("a.txt", "1"); + const { backup } = setup([repoInfo("o/a", remote)]); + + expect((await backup()).code).toBe(ExitCode.Success); + + const names = localFiles("test/o/a"); + expect(names).toHaveLength(2); + expect(await s3Names("test/o/a")).toEqual(names); + const json = names.find((n) => n.endsWith(".json")) as string; + expect(await s3Text(`${prefix}/test/o/a/${json}`)).toBe( + readFileSync(path.join(localDir("test/o/a"), json), "utf8"), + ); + }); + + it("creates no new snapshot for unchanged repositories and keeps the newest N", async () => { + const remote = new TestRemote(root, "a", home); + remote.commit("a.txt", "1"); + const { backup } = setup([repoInfo("o/a", remote)], { keep: 2 }); + + expect((await backup()).code).toBe(0); + const first = await s3Names("test/o/a"); + const second = await backup(); + expect(second.json.summary).toMatchObject({ ok: 0, unchanged: 1 }); + expect(await s3Names("test/o/a")).toEqual(first); + + for (const n of [2, 3, 4]) { + remote.commit("a.txt", String(n)); + expect((await backup()).code).toBe(0); + } + const names = await s3Names("test/o/a"); + expect(names.filter((n) => n.endsWith(".json"))).toHaveLength(2); + expect(names.filter((n) => n.endsWith(".bundle"))).toHaveLength(2); + // The local target has no per-target keep and still holds every snapshot (keep 7). + expect(localFiles("test/o/a").filter((n) => n.endsWith(".json"))).toHaveLength(4); + }, 60_000); + + it("reads the credentials from GHBACKUP_S3_* when the config has none", async () => { + const remote = new TestRemote(root, "a", home); + remote.commit("a.txt", "1"); + const { backup } = setup( + [repoInfo("o/a", remote)], + { accessKeyId: undefined, secretAccessKey: undefined }, + { GHBACKUP_S3_ACCESS_KEY_ID: accessKeyId, GHBACKUP_S3_SECRET_ACCESS_KEY: secretAccessKey }, + ); + expect((await backup()).code).toBe(0); + expect(await s3Names("test/o/a")).toHaveLength(2); + }); + + it("exits with 2 and a masked message when the credentials are wrong", async () => { + const remote = new TestRemote(root, "a", home); + remote.commit("a.txt", "1"); + const { backup, err } = setup([repoInfo("o/a", remote)], { + secretAccessKey: "wrong-secret-value", + }); + + const { code } = await backup(); + expect(code).toBe(ExitCode.Usage); + const output = err.join("\n"); + expect(output).toContain('target "bucket": cannot access bucket'); + expect(output).not.toContain("wrong-secret-value"); + // Nothing was written anywhere: the check runs before any backup work. + expect(() => readdirSync(path.join(root, "out"))).toThrow(); + }); +}); From 80b6846410aa5b67d3774bd2756d74383c7c64ca Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:55:27 +0000 Subject: [PATCH 4/5] Run S3 integration tests against an S3Proxy service container in CI Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01EreR27QcUGyx3TeBypytVW --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43beba5..d53f013 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,23 @@ jobs: fail-fast: false matrix: node: [22, 24] + services: + # S3-compatible server for the S3 integration tests. The credentials are fixed test + # values for this throwaway container, not secrets. + s3: + image: andrewgaul/s3proxy:latest + ports: + - 9000:80 + env: + S3PROXY_AUTHORIZATION: aws-v2-or-v4 + S3PROXY_IDENTITY: test-access + S3PROXY_CREDENTIAL: test-secret + JCLOUDS_PROVIDER: filesystem + JCLOUDS_FILESYSTEM_BASEDIR: /data + env: + GHBACKUP_TEST_S3_ENDPOINT: http://localhost:9000 + GHBACKUP_TEST_S3_ACCESS_KEY_ID: test-access + GHBACKUP_TEST_S3_SECRET_ACCESS_KEY: test-secret steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -20,5 +37,15 @@ jobs: - run: npm ci - run: npm run lint - run: npm run typecheck + - name: Wait for the S3 server + run: | + for i in $(seq 1 60); do + code=$(curl -s -o /dev/null -w '%{http_code}' "$GHBACKUP_TEST_S3_ENDPOINT/" || true) + # An unauthenticated request is answered with 403 once S3Proxy is up. + if [ "$code" = "403" ]; then echo "S3 server ready after ${i}s"; exit 0; fi + sleep 1 + done + echo "S3 server did not become ready" >&2 + exit 1 - run: npm test - run: npm run build From 25ac3af6c19ddd93513505f299d0a475c9d155be Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:57:00 +0000 Subject: [PATCH 5/5] Document S3 targets and complete phase 3 in the plan Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01EreR27QcUGyx3TeBypytVW --- LEARNINGS.md | 6 +++++ README.md | 39 +++++++++++++++++++++++++++++++-- developer-todos.md | 4 ++-- plans/README.md | 4 ++-- plans/faz-01-iskelet-ve-list.md | 2 +- plans/faz-03-s3.md | 32 +++++++++++++++++---------- plans/kararlar.md | 4 +++- plans/mimari.md | 17 ++++++++++---- 8 files changed, 85 insertions(+), 23 deletions(-) diff --git a/LEARNINGS.md b/LEARNINGS.md index f2fad3a..f6b752c 100644 --- a/LEARNINGS.md +++ b/LEARNINGS.md @@ -5,3 +5,9 @@ ## 2026-10-14 — Bulut oturumları kullanıcı düzeyindeki CLAUDE.md'yi yüklemez Claude Code bulut oturumlarında `~/.claude/CLAUDE.md` bulunmaz. Yalnızca repodaki `CLAUDE.md`, `.claude/skills` ve `.claude/settings.json` gelir. Bu yüzden planlama ve geliştirme skill'leri uyulması gereken kuralları (credential'lar, git, doküman güncelliği, geriye dönük uyumluluk) kendi içinde taşır. Bulut ortamı `CLAUDE_CODE_REMOTE=true` ile tanınır. + +## 2026-10-14 — MinIO resmi imajı kalktı; CI'da S3Proxy +`minio/minio` Docker Hub'dan kaldırıldı (MinIO ücretsiz imaj dağıtımını durdurdu, repo arşivlendi). S3 entegrasyon testleri CI'da `andrewgaul/s3proxy` (filesystem backend, kimlik env ile) servis container'ına karşı çalışır; yerelde aynı sunucu Maven Central'daki `org.gaul:s3proxy` fat jar'ıyla (Java yeterli, Docker gerekmez) başlatılır. Kullanıcıların kendi barındırdığı MinIO hâlâ desteklenen bir S3 uç noktasıdır. + +## 2026-10-14 — AWS SDK v3 varsayılan olarak CRC32 sağlama toplamı ekler +Üçüncü parti S3 servisleri bu başlıkları reddedebildiği için istemci `requestChecksumCalculation` ve `responseChecksumValidation` için `WHEN_REQUIRED` ile oluşturulur (K-035); `DeleteObjects` gibi zorunlu kılan işlemler yine sağlama toplamı gönderir. diff --git a/README.md b/README.md index 4b6846b..4dbfc40 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A Node.js/TypeScript CLI (`github-backup`) that backs up every repository you can reach through the GitHub CLI (`gh`): your personal repositories and those of all organizations you belong to, with wikis and Git LFS objects. Backups go to a local directory and/or any S3-compatible bucket (AWS S3, Cloudflare R2, Backblaze B2, MinIO) as git bundle snapshots, and a new snapshot is created only when refs change. -Status: under development. `list` and `backup` to local targets work; S3 targets and Git LFS are not implemented yet. See the plan (Turkish) in [plans/README.md](plans/README.md). +Status: under development. `list` and `backup` to local and S3 targets work; Git LFS is not implemented yet. See the plan (Turkish) in [plans/README.md](plans/README.md). ## Requirements @@ -28,6 +28,41 @@ github-backup backup --local ~/git-backup [--keep 7] [--concurrency 4] [--no-wik Exit codes: `0` success, `1` at least one repository failed (the others are still backed up), `2` configuration or prerequisite error, `3` another run is in progress, `130` interrupted. +## S3 targets + +Any S3-compatible service works (AWS S3, Cloudflare R2, Backblaze B2, MinIO, ...). Add a target of type `s3` to the config file: + +```json +{ + "type": "s3", + "name": "r2", + "bucket": "my-backups", + "prefix": "git", + "endpoint": "https://ACCOUNT_ID.r2.cloudflarestorage.com", + "region": "auto", + "forcePathStyle": false, + "accessKeyId": "YOUR_ACCESS_KEY_ID", + "secretAccessKey": "YOUR_SECRET_ACCESS_KEY", + "keep": 14 +} +``` + +| Field | Meaning | +| --- | --- | +| `bucket` | Bucket name (required). | +| `prefix` | Key prefix inside the bucket; leading, trailing and doubled `/` are ignored. Default: bucket root. | +| `endpoint` | Service URL; leave out for AWS S3. | +| `region` | Default `us-east-1`; R2 uses `auto`. | +| `forcePathStyle` | Use `https://host/bucket/key` instead of `https://bucket.host/key` (MinIO and some self-hosted services need it). Default `false`. | +| `accessKeyId`, `secretAccessKey` | Static credentials. If both are left out, `GHBACKUP_S3_ACCESS_KEY_ID` and `GHBACKUP_S3_SECRET_ACCESS_KEY` are used, and then the AWS SDK's default chain (environment, shared credentials file, instance role). Setting only one of the two is an error. | +| `keep` | Snapshots kept per repository on this target; overrides the global `keep` and `--keep`. | + +The key needs `s3:ListBucket` (also used by the start-up bucket check), `s3:GetObject`, `s3:PutObject` and `s3:DeleteObject`; to clean up multipart uploads that were interrupted, also `s3:ListBucketMultipartUploads` and `s3:AbortMultipartUpload` (without them the cleanup is skipped). Restrict the key to this one bucket. + +Before any work starts, `backup` checks every S3 target with a `HeadBucket` request. If the bucket is missing or the credentials are rejected, it prints the target name and the reason (with keys masked) and exits with code 2. Credentials in the config file are readable by anyone who can read that file: `chmod 600` it (a warning is printed otherwise). + +Objects are uploaded as streaming multipart uploads, so large bundles are not held in memory, and an object appears only when its upload completes. The manifest is uploaded last. + ## Snapshot layout ```text @@ -36,7 +71,7 @@ Exit codes: `0` success, `1` at least one repository failed (the others are stil ///.wiki/... # wiki, same layout ``` -`` is the UTC time as `YYYYMMDDTHHMMSSZ`. A snapshot without its `.json` manifest is incomplete and is removed on the next run. +`` is the target's `path`, or `s3:///` for S3 targets; both use the same layout. `` is the UTC time as `YYYYMMDDTHHMMSSZ`. A snapshot without its `.json` manifest is incomplete and is removed on the next run. ## Restoring diff --git a/developer-todos.md b/developer-todos.md index dc20341..1d6f360 100644 --- a/developer-todos.md +++ b/developer-todos.md @@ -23,8 +23,8 @@ Sıradaki fazı engelleyen işler. ### Faz 3 — `s3` hedefi -- [ ] 👤 S3 uyumlu servisi seç, bucket oluştur, yalnızca o bucket'a yetkili erişim anahtarı üret ve config'ine ekle. -- [ ] 💻 Gerçek bucket'a yedek alıp bir bundle'ı indirerek doğrula. +- [ ] 👤 S3 uyumlu servisi seç, bucket oluştur, yalnızca o bucket'a yetkili (`s3:ListBucket`, `GetObject`, `PutObject`, `DeleteObject`, isteğe bağlı `ListBucketMultipartUploads`/`AbortMultipartUpload`) erişim anahtarı üret ve config'ine ekle (`chmod 600`). +- [ ] 💻 Gerçek bucket'a `node dist/cli.js backup --target --include '/*'` çalıştır, bir bundle'ı indirip `git clone /tmp/x` ile doğrula; komutu tekrarlayınca özetin `unchanged` göstermesini kontrol et: gerçek bucket ve `gh` oturumu gerekir, bulutta yok. ### Faz 4 — Git LFS diff --git a/plans/README.md b/plans/README.md index fbb3ecd..fa0670d 100644 --- a/plans/README.md +++ b/plans/README.md @@ -46,7 +46,7 @@ Fazlar tablodaki sırayla uygulanır; sıranın tek kaynağı bu tablodur. Her f | --- | --- | --- | --- | | 1 | İskelet, config ve `list` | Tamamlandı | [faz-01-iskelet-ve-list.md](faz-01-iskelet-ve-list.md) | | 2 | Mirror, snapshot ve `local` hedefi | Tamamlandı | [faz-02-mirror-ve-local.md](faz-02-mirror-ve-local.md) | -| 3 | `s3` hedefi | Devam ediyor | [faz-03-s3.md](faz-03-s3.md) | +| 3 | `s3` hedefi | Tamamlandı | [faz-03-s3.md](faz-03-s3.md) | | 4 | Git LFS | Bekliyor | [faz-04-lfs.md](faz-04-lfs.md) | | 5 | Dokümantasyon ve son kontrol | Bekliyor | [faz-05-dokuman-ve-son-kontrol.md](faz-05-dokuman-ve-son-kontrol.md) | @@ -59,7 +59,7 @@ Fazlar tablodaki sırayla uygulanır; sıranın tek kaynağı bu tablodur. Her f - Token ve S3 anahtarları loglanmaz, hata mesajlarında maskelenir. - Her faz sonunda `npm run lint`, `npm run typecheck`, `npm test`, `npm run build` temiz ve CI yeşil olmalı. - Yeni davranış testle gelir; git işlemleri gerçek geçici repolarla test edilir. -- Test S3 kimliği (MinIO `minioadmin`) CI workflow'unda sabit yazılır; başka secret yönetimi kullanılmaz. +- Test S3 kimliği (S3Proxy `test-access`/`test-secret`) CI workflow'unda sabit yazılır; başka secret yönetimi kullanılmaz. ## Açık noktalar diff --git a/plans/faz-01-iskelet-ve-list.md b/plans/faz-01-iskelet-ve-list.md index 764f79c..fa04dad 100644 --- a/plans/faz-01-iskelet-ve-list.md +++ b/plans/faz-01-iskelet-ve-list.md @@ -68,5 +68,5 @@ Kod tarafı bitti; bu ortamda doğrulanabilen tüm kabul kriterleri sağlandı ( - `commands/backup.ts` yalnızca stub; `BackupOptions` şimdilik `CommonOptions` ile aynı, Faz 2'de `ConfigFlags`'in (cacheDir, concurrency, keep, wiki, lfs, target, local) zaten kapsadığı alanlar kullanılır. `git` varlığının kontrolü (`git --version`) `backup`'ın ön koşul adımına Faz 2'de eklenecek (`list` yalnızca `gh` ister). - Config'te `keep` çözümü (hedef `keep` > genel `keep`) henüz bir yardımcıya bağlanmadı; hedef şemasında `keep` isteğe bağlı kalır. Faz 2'de tek fonksiyonla çözülmeli. - Araç sürümleri: TypeScript 7, Vitest 5, Biome 2.5, commander 15, zod 4. Hepsi sorunsuz çalıştı. Biome'da `rules.preset` kullanılır (`recommended` alanı kaldırılıyor). -- CI yalnızca `.github/workflows/ci.yml` ile Node 22/24 matrisinde çalışır; MinIO servisi Faz 3'te eklenecek. +- CI yalnızca `.github/workflows/ci.yml` ile Node 22/24 matrisinde çalışır; S3 test sunucusu (S3Proxy) Faz 3'te eklendi. - Test `config.example.json`'un şemaya uyduğunu doğrular; şema değişirse örnek dosya, mimari.md ve README birlikte güncellenmeli (`REPO.md`). diff --git a/plans/faz-03-s3.md b/plans/faz-03-s3.md index 43402f8..f255595 100644 --- a/plans/faz-03-s3.md +++ b/plans/faz-03-s3.md @@ -1,10 +1,10 @@ # Faz 3 — `s3` hedefi -**Durum:** Devam ediyor +**Durum:** Tamamlandı ## Amaç -Snapshot'lar S3 uyumlu bir bucket'a (AWS S3, R2, B2, MinIO) `local` ile aynı düzende yüklenir ve saklama uygulanır. +Snapshot'lar S3 uyumlu bir bucket'a (AWS S3, R2, B2, MinIO vb.) `local` ile aynı düzende yüklenir ve saklama uygulanır. ## Ön koşullar @@ -19,7 +19,7 @@ Snapshot'lar S3 uyumlu bir bucket'a (AWS S3, R2, B2, MinIO) `local` ile aynı d - Silme: `DeleteObjects` (1000'lik gruplar). - Ön kontrol: `backup` başında her s3 hedefi için `HeadBucket`; erişim yoksa hedef adı ve hata ile çıkış 2. - Hata mesajlarında anahtarlar maskelenir. -- CI: `ci.yml`'e MinIO servis container'ı (`minio/minio`, `minioadmin`/`minioadmin`), test bucket'ı test kurulumu oluşturur. +- CI: `ci.yml`'e S3Proxy servis container'ı (`andrewgaul/s3proxy`, kimlik `test-access`/`test-secret`; `minio/minio` Docker Hub'dan kalktığı için, bkz. K-025); test bucket'ını entegrasyon testleri oluşturur. ## Kapsam dışı @@ -27,12 +27,12 @@ Snapshot'lar S3 uyumlu bir bucket'a (AWS S3, R2, B2, MinIO) `local` ile aynı d ## Görevler -- [ ] Doğrulama: MinIO'nun GitHub Actions servis container'ı olarak çalıştırılma biçimini (komut argümanı `server /data` verilemiyorsa `bitnami/minio` gibi alternatif veya adım içinde `docker run`) resmi dokümandan doğrula ve `ci.yml`'i buna göre yaz. -- [ ] `targets/s3.ts`. -- [ ] `GHBACKUP_TEST_S3_ENDPOINT` varsa çalışan entegrasyon testleri: yükleme, `latestManifest`, değişmeyen repo, saklama, 1000+ anahtar sayfalama (sahte küçük dosyalarla), path-style. -- [ ] Birim testi: kimlik önceliği, prefix normalizasyonu (baş/son `/`). -- [ ] `HeadBucket` ön kontrolü + hata mesajı. -- [ ] CI'da MinIO. +- [x] Doğrulama: `minio/minio` imajı kalktığı için CI'da S3Proxy; `services.` `command`/`entrypoint`/`options` destekler, S3Proxy argümansız çalışır. +- [x] `targets/s3.ts`. +- [x] `GHBACKUP_TEST_S3_ENDPOINT` varsa çalışan entegrasyon testleri: yükleme, `latestManifest`, değişmeyen repo, saklama, 1000+ anahtar sayfalama (sahte küçük dosyalarla), path-style. +- [x] Birim testi: kimlik önceliği, prefix normalizasyonu (baş/son `/`). +- [x] `HeadBucket` ön kontrolü + hata mesajı. +- [x] CI'da S3Proxy. ## 👤 Kullanıcı aksiyonları @@ -41,7 +41,7 @@ Snapshot'lar S3 uyumlu bir bucket'a (AWS S3, R2, B2, MinIO) `local` ile aynı d ## Kabul kriterleri -- MinIO entegrasyon testleri CI'da geçer. +- S3 entegrasyon testleri (S3Proxy) CI'da geçer. - Aynı repo için `local` ve `s3` hedeflerinde dosya adları ve manifest içerikleri aynı düzendedir. - Yanlış kimlikle `backup` çıkış 2 ve maskelenmiş anlaşılır mesaj verir. @@ -51,4 +51,14 @@ Snapshot'lar S3 uyumlu bir bucket'a (AWS S3, R2, B2, MinIO) `local` ile aynı d ## Devam notları -_Henüz başlanmadı._ Faz 2'den devralınanlar için [faz-02-mirror-ve-local.md](faz-02-mirror-ve-local.md) **Devam notları**'na bak: `Target` arayüzü (`targets/types.ts`, `cleanupIncomplete` dahil), `createTarget`'taki `s3` yer tutucusu, `applyRetention`, `HeadBucket` ön kontrolünün takılacağı yer ve test yardımcıları. +Kod tarafı bitti. Doğrulananlar: + +- Lint, typecheck, test ve build temiz. S3 entegrasyon testleri (15 test) yerelde S3Proxy 4.1.1'e karşı geçti. CI (Node 22 ve 24, S3Proxy servis container'ı) yeşil. +- Kabul kriterleri: `local` ve `s3` aynı dosya adları/manifest içeriği (`test/integration/s3-target.test.ts`), yanlış kimlikle `backup` çıkış 2 ve maskelenmiş mesaj (hem sunucusuz `test/unit/s3-target.test.ts` hem entegrasyon). +- Açık kalan tek madde gerçek bir bucket'la elle deneme (yukarıdaki 👤/💻; bulutta gerçek bucket ve `gh` oturumu yok). + +Sonraki faz (Faz 4, LFS) için: + +- `Target` arayüzü ve `local`/`s3` hedefleri `lfsTar`'ı zaten yazıyor, siliyor ve manifest'siz bırakılanı temizliyor (`s3`'te `.lfs.tar` büyük nesne için multipart'tan geçer; 20 MiB'lık dosya testle doğrulandı). Faz 4'te yalnızca üretim kalıyor: Faz 2 devam notlarındaki "Faz 4 (LFS) için" bölümüne bak. +- S3 entegrasyon testlerini yerelde koşmak için: Docker olmadan `java -jar s3proxy-4.1.1-jar-with-dependencies.jar --properties s3proxy.conf` (conf: `s3proxy.endpoint=http://127.0.0.1:9000`, `s3proxy.authorization=aws-v2-or-v4`, `s3proxy.identity=test-access`, `s3proxy.credential=test-secret`, `jclouds.provider=filesystem`, `jclouds.filesystem.basedir=`), sonra `GHBACKUP_TEST_S3_ENDPOINT=http://127.0.0.1:9000 GHBACKUP_TEST_S3_ACCESS_KEY_ID=test-access GHBACKUP_TEST_S3_SECRET_ACCESS_KEY=test-secret npm test`. +- Yarım multipart temizliği en iyi çabadır (K-036); S3Proxy `ListMultipartUploads`'ı destekliyor mu, testlerde ayrıca doğrulanmadı (yalnızca hatanın saklamayı bozmadığı sahte istemciyle test edildi). diff --git a/plans/kararlar.md b/plans/kararlar.md index 48b9810..7d4ab6f 100644 --- a/plans/kararlar.md +++ b/plans/kararlar.md @@ -28,7 +28,7 @@ Projenin güncel kararları. Bir karar değişirse satırı güncellenir ve tari | K-022 | 2026-10-14 | S3 yüklemeleri AWS SDK v3 `@aws-sdk/lib-storage` `Upload` ile stream + multipart; `endpoint`, `region`, `forcePathStyle` ayarlanabilir. | Tek PUT 5 GB sınırına takılır; S3 uyumlu servisler desteklenir. | | K-023 | 2026-10-14 | Node `>=22`, npm, TypeScript strict ESM; `commander`, `zod`. | Kullanıcı tercihi (npm); 22 ve 24 LTS. | | K-024 | 2026-10-14 | Dağıtım yalnızca repodan: `npm ci && npm run build`, `node dist/cli.js` veya `npm link`. npm yayını yok. | Kullanıcı tercihi; yayın süreci gerekmez. | -| K-025 | 2026-10-14 | Test Vitest (birim + geçici git repolarıyla entegrasyon + CI'da MinIO ile S3), lint/format Biome, typecheck `tsc --noEmit`; GitHub Actions'ta Node 22 ve 24. | Az config, hızlı; S3 kodu gerçek bir S3 API'sine karşı test edilir. | +| K-025 | 2026-10-14 | Test Vitest (birim + geçici git repolarıyla entegrasyon + CI'da S3Proxy ile S3), lint/format Biome, typecheck `tsc --noEmit`; GitHub Actions'ta Node 22 ve 24. | Az config, hızlı; S3 kodu gerçek bir S3 API'sine karşı test edilir. 2026-10-14: `minio/minio` Docker Hub'dan kalktı (MinIO ücretsiz imaj dağıtımını durdurdu, repo arşivlendi), CI'da MinIO yerine `andrewgaul/s3proxy` kullanılır. | | K-026 | 2026-10-14 | Dil: `plans/` Türkçe; kod, kod yorumları, kök README, CLI çıktısı ve commit mesajları İngilizce. | Açık kaynak CLI için yaygın düzen. | | K-027 | 2026-10-14 | Eşzamanlı çalıştırmayı önlemek için önbellekte PID'li kilit dosyası; ölü PID'nin kilidi devralınır. | Üst üste binen cron çalıştırmaları aynı mirror'ı bozmasın. | | K-028 | 2026-10-14 | Mirror yalnızca `refs/heads/*` ve `refs/tags/*` alır; `refs/pull/*` alınmaz. | Bundle küçük kalır, geri yüklenen repo temiz olur. | @@ -38,3 +38,5 @@ Projenin güncel kararları. Bir karar değişirse satırı güncellenir ve tari | K-032 | 2026-10-14 | `keep` çözümü: hedefte `keep` varsa o, yoksa genel `keep` (bayrak > env > dosya ile belirlenen). Hedefteki değer `--keep` bayrağını ezer. | Hedef başına saklama süresi config'te bilinçli verilir (ör. S3 için 14); tek `resolveKeep` fonksiyonu. | | K-033 | 2026-10-14 | Mirror fetch'i `git fetch --no-tags --no-prune origin`; her fetch'ten sonra `HEAD`, `git ls-remote --symref` ile uzağın varsayılan branch'ine ayarlanır. | Tag'ler refspec'ten gelir, prune hiçbir koşulda yapılmaz (K-008); bundle'dan klonlanan repo checkout ile açılır. | | K-034 | 2026-10-14 | Wiki ayrı birimdir (`.wiki` anahtarı, kendi mirror'ı ve snapshot'ları). Wiki'de "repository not found" `skipped: no wiki`; ref'siz mirror `skipped: empty`. | Wiki açık ama sayfasız repolar hata üretmesin; wiki yedeği ana repodan bağımsız saklanır. | +| K-035 | 2026-10-14 | S3 istemcisi `requestChecksumCalculation` ve `responseChecksumValidation` için `WHEN_REQUIRED` kullanır. | AWS SDK JS v3'ün varsayılan CRC32 sağlama toplamı R2, B2 gibi üçüncü parti servislerde imza/başlık hatası verebilir; yalnızca API'nin zorunlu kıldığı yerde (ör. `DeleteObjects`) eklenir. | +| K-036 | 2026-10-14 | `Target` arayüzünde isteğe bağlı `checkAccess()` vardır; `backup` her hedef için kilitten önce çağırır ve hata çıkış 2 verir (`s3`: `HeadBucket`). Yarım multipart temizliği en iyi çaba esaslıdır. | Yanlış kimlik/bucket ilk repoda değil başta anlaşılsın; multipart listelemeyi desteklemeyen servisler saklama adımını bozmasın. | diff --git a/plans/mimari.md b/plans/mimari.md index 1346ff8..105a8fb 100644 --- a/plans/mimari.md +++ b/plans/mimari.md @@ -40,6 +40,7 @@ src/ index.ts # tür → uygulama kaydı targets/ types.ts # Target arayüzü, Manifest, SnapshotFiles + names.ts # `.` dosya/nesne adı ayrıştırma (local ve s3 ortak) local.ts s3.ts index.ts @@ -53,7 +54,7 @@ src/ errors.ts # ExitCode, CliError, ConfigError, PrerequisiteError test/ unit/ - integration/ # geçici git repoları, local hedef, MinIO + integration/ # geçici git repoları, local hedef, S3 (S3Proxy) ``` ## Genişletilebilirlik @@ -70,6 +71,7 @@ test/ - `listSnapshots(repoKey): Promise`: manifest'i olan (tamamlanmış) snapshot'ların zaman damgaları, eskiden yeniye. - `deleteSnapshot(repoKey, stamp)`: bundle, lfs.tar ve manifest silinir (manifest ilk). - `cleanupIncomplete(repoKey)`: manifest'i olmayan yarım dosyaları ve `.partial` dosyalarını siler. + - `checkAccess?()`: isteğe bağlı; depolamaya erişilemiyorsa okunabilir bir nedenle hata fırlatır (`s3`: `HeadBucket`). `backup` bunu kilitten önce çağırır. - `repoKey` = `//` (wiki için `//.wiki`). - Kaynak türü → uygulama eşlemesi `sources/index.ts` içindeki `createSource` fonksiyonundadır; `core/discover.ts` her kaynak yapılandırması için ön koşulları kontrol eder, listeler ve o kaynağın filtrelerini uygular. - Dış komut çağıran modüller `Exec` fonksiyon tipini (`util/exec.ts`) bağımlılık olarak alır; testlerde sahte `exec` verilir. Komutlar `CommandDeps` ile `exec`, `env`, `stdout`, `stderr` alır. @@ -131,7 +133,14 @@ Ortak düzen (her iki hedefte aynı): - ``: UTC, `YYYYMMDDTHHMMSSZ` (sıralanabilir, S3 anahtarında güvenli). - Manifest: `{ version: 1, source, fullName, stamp, refsHash, refs: [{name, sha}], bundleBytes, lfsTarBytes?, tool: { name, version } }`. - `local`: `` = `path`. Dosyalar önce `.partial` uzantısıyla yazılır, sonra `rename`. -- `s3`: `` = `s3:///`. `@aws-sdk/lib-storage` `Upload` ile stream yükleme. `endpoint`, `region`, `forcePathStyle` ayarlanabilir (R2, B2, MinIO). Kimlik: config'teki `accessKeyId`/`secretAccessKey`, yoksa `GHBACKUP_S3_ACCESS_KEY_ID`/`GHBACKUP_S3_SECRET_ACCESS_KEY`, yoksa AWS SDK'nın varsayılan zinciri. +- `s3`: `` = `s3:///`; `prefix` baş/son `/` ve çift `/` temizlenerek normalleştirilir, boşsa kök bucket'tır. `@aws-sdk/lib-storage` `Upload` ile stream yükleme (16 MiB part, 4 paralel part; hata olursa SDK multipart'ı iptal eder). `endpoint`, `region` (varsayılan `us-east-1`), `forcePathStyle` ayarlanabilir (R2, B2, MinIO). Kimlik: config'teki `accessKeyId`/`secretAccessKey`, yoksa `GHBACKUP_S3_ACCESS_KEY_ID`/`GHBACKUP_S3_SECRET_ACCESS_KEY`, yoksa AWS SDK'nın varsayılan zinciri; yalnızca biri verilirse `ConfigError` (çıkış 2). + - İstemci `requestChecksumCalculation` ve `responseChecksumValidation` için `WHEN_REQUIRED` kullanır (SDK varsayılanı olan CRC32 üçüncü parti servislerde imza hatası verebilir; K-035). + - `latestManifest`/`listSnapshots`: `//` altı `ListObjectsV2` ile sayfalı listelenir; yalnızca doğrudan alt nesneler ve `.` adlıları sayılır, manifest'i (`.json`) olanlar tamamlanmıştır. + - `putSnapshot`: bundle, lfs.tar, en son manifest (`PutObject`, `application/json`). `Upload` bitene kadar nesne görünmez; manifest yazılmadan snapshot tamamlanmış sayılmaz. + - `deleteSnapshot`: önce manifest `DeleteObject`, sonra aynı `stamp`'li nesneler `DeleteObjects` ile 1000'lik gruplar halinde; yanıttaki `Errors` hata sayılır. + - `cleanupIncomplete`: manifest'siz `stamp`'li nesneleri siler, ardından `ListMultipartUploads`/`AbortMultipartUpload` ile repo altındaki yarım multipart yüklemeleri iptal eder (desteklemeyen veya yetkisiz servislerde sessizce atlanır). + - `checkAccess`: `HeadBucket`. `backup`, kilitten önce her hedef için çağırır; hata `target "": cannot access bucket "": (HTTP )` mesajıyla çıkış 2 verir. Mesajlardaki anahtarlar `***` olarak maskelenir. + - Gereken izinler: `s3:ListBucket` (HeadBucket dahil), `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`; yarım multipart temizliği için ayrıca `s3:ListBucketMultipartUploads` ve `s3:AbortMultipartUpload`. ## Yapılandırma @@ -208,13 +217,13 @@ Bayraklar (her iki komutta ortak olanlar dahil): - Birim (Vitest): config birleştirme ve öncelik, şema hataları, glob filtre, zaman damgası, ref özeti, saklama seçimi, `gh` çıktısının ayrıştırılması (sahte `exec` ile). - Entegrasyon (Vitest): geçici dizinde `git init --bare` ile "uzak" repolar; `file://` URL'leri döndüren test kaynağı; mirror → bundle → `local` hedef → `git clone ` ile geri yükleme doğrulaması; değişmeyen repo için yeni snapshot oluşmaması; saklama. - LFS entegrasyon: `git-lfs` kuruluysa yerel LFS deposu olmadan çalışan senaryo; mümkün değilse birim + elle doğrulama. -- S3 entegrasyon: `GHBACKUP_TEST_S3_ENDPOINT` tanımlıysa MinIO'ya karşı çalışır, değilse atlanır. CI'da MinIO servis container'ı ile çalışır (kimlik `minioadmin`/`minioadmin`, workflow'da sabit). +- S3 entegrasyon: `GHBACKUP_TEST_S3_ENDPOINT` (+ `GHBACKUP_TEST_S3_ACCESS_KEY_ID`, `GHBACKUP_TEST_S3_SECRET_ACCESS_KEY`, isteğe bağlı `GHBACKUP_TEST_S3_BUCKET`) tanımlıysa herhangi bir S3 uyumlu sunucuya karşı çalışır, değilse atlanır. CI'da `andrewgaul/s3proxy` (S3Proxy, filesystem backend) servis container'ı ile çalışır; kimlik `test-access`/`test-secret`, workflow'da sabit. Yerelde aynı sunucu Maven Central'daki `org.gaul:s3proxy` jar'ıyla (`java -jar s3proxy--jar-with-dependencies.jar --properties s3proxy.conf`) başlatılabilir. Birim testleri (`test/unit/s3-target.test.ts`) sahte bir istemciyle sayfalama, 1000'lik silme grupları ve hata maskelemeyi sunucusuz doğrular. - Gerçek GitHub hesabıyla uçtan uca deneme elle yapılır (💻). ## Kalite ve CI - `npm run lint` (Biome check), `npm run format` (Biome format --write), `npm run typecheck` (`tsc --noEmit`), `npm test` (Vitest), `npm run build` (`tsc` → `dist/`). -- GitHub Actions `ci.yml`: push ve PR'da Node 22 ve 24 matrisinde lint, typecheck, test, build; MinIO servis container'ı. +- GitHub Actions `ci.yml`: push ve PR'da Node 22 ve 24 matrisinde lint, typecheck, test, build; S3Proxy servis container'ı (testlerden önce `403` dönene kadar beklenir). - Dağıtım yok: kullanıcı repoyu klonlar, `npm ci && npm run build`, `node dist/cli.js` veya `npm link` ile `github-backup` komutu. ## Loglama