diff --git a/docs/stackit_alpha.md b/docs/stackit_alpha.md index 7f03a0753..3098c2166 100644 --- a/docs/stackit_alpha.md +++ b/docs/stackit_alpha.md @@ -41,5 +41,6 @@ stackit alpha [flags] ### SEE ALSO * [stackit](./stackit.md) - Manage STACKIT resources using the command line +* [stackit alpha bucket](./stackit_alpha_bucket.md) - Manages objects in STACKIT Object Storage buckets (data plane) * [stackit alpha vpc](./stackit_alpha_vpc.md) - Manages vpcs diff --git a/docs/stackit_alpha_bucket.md b/docs/stackit_alpha_bucket.md new file mode 100644 index 000000000..52232121e --- /dev/null +++ b/docs/stackit_alpha_bucket.md @@ -0,0 +1,60 @@ +## stackit alpha bucket + +Manages objects in STACKIT Object Storage buckets (data plane) + +### Synopsis + +Manages objects in STACKIT Object Storage buckets. + +Buckets and objects can be addressed with s3 URIs (s3://BUCKET[/KEY]) or +STACKIT Object Storage URLs in virtual hosted style +(https://BUCKET.object.storage.REGION.onstackit.cloud[/KEY]) or path style +(https://object.storage.REGION.onstackit.cloud/BUCKET[/KEY]). + +Credentials (access key ID / secret access key) are required and can be provided +with the --access-key-id and --secret-access-key flags, the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID +and STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variables, or the config file. +Create credentials with 'stackit object-storage credentials create'. + +``` +stackit alpha bucket [flags] +``` + +### Examples + +``` + List all buckets + $ stackit alpha bucket ls + + Upload a file to a bucket + $ stackit alpha bucket cp local/file.txt s3://my-bucket/file.txt +``` + +### Options + +``` + -h, --help[=true|false] Help for "stackit alpha bucket" +``` + +### Options inherited from parent commands + +``` + -y, --assume-yes[=true|false] If set, skips all confirmation prompts + --async[=true|false] If set, runs the command asynchronously + -o, --output-format string Output format, (one of: [json, pretty, none, yaml]) + -p, --project-id string Project ID + --region string Target region for region-specific requests + --verbosity string Verbosity of the CLI, (one of: [debug, info, warning, error]) (default "info") +``` + +### SEE ALSO + +* [stackit alpha](./stackit_alpha.md) - Contains alpha STACKIT CLI commands +* [stackit alpha bucket cp](./stackit_alpha_bucket_cp.md) - Copies objects between local and Object Storage +* [stackit alpha bucket ls](./stackit_alpha_bucket_ls.md) - Lists buckets and objects +* [stackit alpha bucket mb](./stackit_alpha_bucket_mb.md) - Creates a bucket +* [stackit alpha bucket mv](./stackit_alpha_bucket_mv.md) - Moves objects between local and Object Storage +* [stackit alpha bucket presign](./stackit_alpha_bucket_presign.md) - Generates a presigned URL for an object +* [stackit alpha bucket rb](./stackit_alpha_bucket_rb.md) - Deletes a bucket +* [stackit alpha bucket rm](./stackit_alpha_bucket_rm.md) - Deletes objects + diff --git a/docs/stackit_alpha_bucket_cp.md b/docs/stackit_alpha_bucket_cp.md new file mode 100644 index 000000000..7eea92582 --- /dev/null +++ b/docs/stackit_alpha_bucket_cp.md @@ -0,0 +1,60 @@ +## stackit alpha bucket cp + +Copies objects between local and Object Storage + +### Synopsis + +Copies a single object or a directory tree between local storage and Object Storage. + +One of SRC and DESTINATION must be a bucket URI (s3://BUCKET[/KEY]) or a STACKIT +Object Storage URL (https://BUCKET.object.storage.REGION.onstackit.cloud[/KEY]) +and the other a local path. +Without --recursive, a single object is transferred. +With --recursive, all objects under the prefix (download) or all files under the +local directory (upload) are transferred. + +``` +stackit alpha bucket cp SRC DESTINATION [flags] +``` + +### Examples + +``` + Upload a file to a bucket + $ stackit alpha bucket cp local/file.txt s3://my-bucket/file.txt + + Download a single object + $ stackit alpha bucket cp s3://my-bucket/file.txt local/file.txt + + Upload a directory recursively + $ stackit alpha bucket cp ./local/dir s3://my-bucket/prefix/ --recursive + + Download a prefix recursively + $ stackit alpha bucket cp s3://my-bucket/prefix/ ./local/dir --recursive +``` + +### Options + +``` + --access-key-id string Object Storage access key ID. Can also be set with the object_storage_access_key_id config key or the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID environment variable. + --endpoint string Object Storage API endpoint. Defaults to https://object.storage..onstackit.cloud. Can also be set with the object_storage_data_plane_custom_endpoint config key or the STACKIT_OBJECT_STORAGE_DATA_PLANE_CUSTOM_ENDPOINT environment variable. + -h, --help[=true|false] Help for "stackit alpha bucket cp" + --recursive[=true|false] Transfer a directory tree or all objects under a prefix + --secret-access-key string Object Storage secret access key. Can also be set with the object_storage_secret_access_key config key or the STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variable. +``` + +### Options inherited from parent commands + +``` + -y, --assume-yes[=true|false] If set, skips all confirmation prompts + --async[=true|false] If set, runs the command asynchronously + -o, --output-format string Output format, (one of: [json, pretty, none, yaml]) + -p, --project-id string Project ID + --region string Target region for region-specific requests + --verbosity string Verbosity of the CLI, (one of: [debug, info, warning, error]) (default "info") +``` + +### SEE ALSO + +* [stackit alpha bucket](./stackit_alpha_bucket.md) - Manages objects in STACKIT Object Storage buckets (data plane) + diff --git a/docs/stackit_alpha_bucket_ls.md b/docs/stackit_alpha_bucket_ls.md new file mode 100644 index 000000000..e47c04b7f --- /dev/null +++ b/docs/stackit_alpha_bucket_ls.md @@ -0,0 +1,55 @@ +## stackit alpha bucket ls + +Lists buckets and objects + +### Synopsis + +Lists all buckets, or the contents of a bucket. + +Without an argument, lists all buckets. +With a bucket URI (s3://BUCKET), lists the "directories" and objects at the bucket root. +With an object URI (s3://BUCKET/PREFIX), lists the "directories" and objects under the prefix. +With --recursive, lists all objects under the prefix flat, including all nested keys. + +``` +stackit alpha bucket ls [BUCKET_URI] [flags] +``` + +### Examples + +``` + List all buckets + $ stackit alpha bucket ls + + List the contents of a bucket + $ stackit alpha bucket ls s3://my-bucket + + Recursively list all objects under a prefix + $ stackit alpha bucket ls s3://my-bucket/path/to/ --recursive +``` + +### Options + +``` + --access-key-id string Object Storage access key ID. Can also be set with the object_storage_access_key_id config key or the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID environment variable. + --endpoint string Object Storage API endpoint. Defaults to https://object.storage..onstackit.cloud. Can also be set with the object_storage_data_plane_custom_endpoint config key or the STACKIT_OBJECT_STORAGE_DATA_PLANE_CUSTOM_ENDPOINT environment variable. + -h, --help[=true|false] Help for "stackit alpha bucket ls" + --recursive[=true|false] List all objects under the prefix recursively, including nested keys + --secret-access-key string Object Storage secret access key. Can also be set with the object_storage_secret_access_key config key or the STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variable. +``` + +### Options inherited from parent commands + +``` + -y, --assume-yes[=true|false] If set, skips all confirmation prompts + --async[=true|false] If set, runs the command asynchronously + -o, --output-format string Output format, (one of: [json, pretty, none, yaml]) + -p, --project-id string Project ID + --region string Target region for region-specific requests + --verbosity string Verbosity of the CLI, (one of: [debug, info, warning, error]) (default "info") +``` + +### SEE ALSO + +* [stackit alpha bucket](./stackit_alpha_bucket.md) - Manages objects in STACKIT Object Storage buckets (data plane) + diff --git a/docs/stackit_alpha_bucket_mb.md b/docs/stackit_alpha_bucket_mb.md new file mode 100644 index 000000000..66312a174 --- /dev/null +++ b/docs/stackit_alpha_bucket_mb.md @@ -0,0 +1,43 @@ +## stackit alpha bucket mb + +Creates a bucket + +### Synopsis + +Creates a bucket in STACKIT Object Storage. + +``` +stackit alpha bucket mb BUCKET_URI [flags] +``` + +### Examples + +``` + Create a bucket + $ stackit alpha bucket mb s3://my-bucket +``` + +### Options + +``` + --access-key-id string Object Storage access key ID. Can also be set with the object_storage_access_key_id config key or the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID environment variable. + --endpoint string Object Storage API endpoint. Defaults to https://object.storage..onstackit.cloud. Can also be set with the object_storage_data_plane_custom_endpoint config key or the STACKIT_OBJECT_STORAGE_DATA_PLANE_CUSTOM_ENDPOINT environment variable. + -h, --help[=true|false] Help for "stackit alpha bucket mb" + --secret-access-key string Object Storage secret access key. Can also be set with the object_storage_secret_access_key config key or the STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variable. +``` + +### Options inherited from parent commands + +``` + -y, --assume-yes[=true|false] If set, skips all confirmation prompts + --async[=true|false] If set, runs the command asynchronously + -o, --output-format string Output format, (one of: [json, pretty, none, yaml]) + -p, --project-id string Project ID + --region string Target region for region-specific requests + --verbosity string Verbosity of the CLI, (one of: [debug, info, warning, error]) (default "info") +``` + +### SEE ALSO + +* [stackit alpha bucket](./stackit_alpha_bucket.md) - Manages objects in STACKIT Object Storage buckets (data plane) + diff --git a/docs/stackit_alpha_bucket_mv.md b/docs/stackit_alpha_bucket_mv.md new file mode 100644 index 000000000..b31fd97e1 --- /dev/null +++ b/docs/stackit_alpha_bucket_mv.md @@ -0,0 +1,57 @@ +## stackit alpha bucket mv + +Moves objects between local and Object Storage + +### Synopsis + +Moves a single object or a directory tree, then deletes the source. + +One of SRC and DESTINATION must be a bucket URI (s3://BUCKET[/KEY]) or a STACKIT +Object Storage URL (https://BUCKET.object.storage.REGION.onstackit.cloud[/KEY]) +and the other a local path. +Bucket-to-bucket moves use a server-side copy. +Without --recursive, a single object is moved. +With --recursive, all objects under the prefix (or all files under the local directory) are moved. + +``` +stackit alpha bucket mv SRC DESTINATION [flags] +``` + +### Examples + +``` + Move a file to a bucket + $ stackit alpha bucket mv local/file.txt s3://my-bucket/file.txt + + Move an object to a new key + $ stackit alpha bucket mv s3://my-bucket/old/file.txt s3://my-bucket/new/file.txt + + Move a prefix to another bucket recursively + $ stackit alpha bucket mv s3://src-bucket/prefix/ s3://dst-bucket/prefix/ --recursive +``` + +### Options + +``` + --access-key-id string Object Storage access key ID. Can also be set with the object_storage_access_key_id config key or the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID environment variable. + --endpoint string Object Storage API endpoint. Defaults to https://object.storage..onstackit.cloud. Can also be set with the object_storage_data_plane_custom_endpoint config key or the STACKIT_OBJECT_STORAGE_DATA_PLANE_CUSTOM_ENDPOINT environment variable. + -h, --help[=true|false] Help for "stackit alpha bucket mv" + --recursive[=true|false] Move a directory tree or all objects under a prefix + --secret-access-key string Object Storage secret access key. Can also be set with the object_storage_secret_access_key config key or the STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variable. +``` + +### Options inherited from parent commands + +``` + -y, --assume-yes[=true|false] If set, skips all confirmation prompts + --async[=true|false] If set, runs the command asynchronously + -o, --output-format string Output format, (one of: [json, pretty, none, yaml]) + -p, --project-id string Project ID + --region string Target region for region-specific requests + --verbosity string Verbosity of the CLI, (one of: [debug, info, warning, error]) (default "info") +``` + +### SEE ALSO + +* [stackit alpha bucket](./stackit_alpha_bucket.md) - Manages objects in STACKIT Object Storage buckets (data plane) + diff --git a/docs/stackit_alpha_bucket_presign.md b/docs/stackit_alpha_bucket_presign.md new file mode 100644 index 000000000..8f1c2fbed --- /dev/null +++ b/docs/stackit_alpha_bucket_presign.md @@ -0,0 +1,48 @@ +## stackit alpha bucket presign + +Generates a presigned URL for an object + +### Synopsis + +Generates a presigned URL that grants time-limited read access to an object +without exposing the Object Storage credentials. + +``` +stackit alpha bucket presign BUCKET_URI [flags] +``` + +### Examples + +``` + Generate a presigned URL valid for 1 hour (default) + $ stackit alpha bucket presign s3://my-bucket/file.txt + + Generate a presigned URL valid for 15 minutes + $ stackit alpha bucket presign s3://my-bucket/file.txt --expires 15m +``` + +### Options + +``` + --access-key-id string Object Storage access key ID. Can also be set with the object_storage_access_key_id config key or the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID environment variable. + --endpoint string Object Storage API endpoint. Defaults to https://object.storage..onstackit.cloud. Can also be set with the object_storage_data_plane_custom_endpoint config key or the STACKIT_OBJECT_STORAGE_DATA_PLANE_CUSTOM_ENDPOINT environment variable. + --expires duration Expiration of the presigned URL (e.g. 15m, 1h, 72h). Maximum: 7 days (default 1h0m0s) + -h, --help[=true|false] Help for "stackit alpha bucket presign" + --secret-access-key string Object Storage secret access key. Can also be set with the object_storage_secret_access_key config key or the STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variable. +``` + +### Options inherited from parent commands + +``` + -y, --assume-yes[=true|false] If set, skips all confirmation prompts + --async[=true|false] If set, runs the command asynchronously + -o, --output-format string Output format, (one of: [json, pretty, none, yaml]) + -p, --project-id string Project ID + --region string Target region for region-specific requests + --verbosity string Verbosity of the CLI, (one of: [debug, info, warning, error]) (default "info") +``` + +### SEE ALSO + +* [stackit alpha bucket](./stackit_alpha_bucket.md) - Manages objects in STACKIT Object Storage buckets (data plane) + diff --git a/docs/stackit_alpha_bucket_rb.md b/docs/stackit_alpha_bucket_rb.md new file mode 100644 index 000000000..5e3740935 --- /dev/null +++ b/docs/stackit_alpha_bucket_rb.md @@ -0,0 +1,43 @@ +## stackit alpha bucket rb + +Deletes a bucket + +### Synopsis + +Deletes a bucket in STACKIT Object Storage. The bucket must be empty. + +``` +stackit alpha bucket rb BUCKET_URI [flags] +``` + +### Examples + +``` + Delete an empty bucket + $ stackit alpha bucket rb s3://my-bucket +``` + +### Options + +``` + --access-key-id string Object Storage access key ID. Can also be set with the object_storage_access_key_id config key or the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID environment variable. + --endpoint string Object Storage API endpoint. Defaults to https://object.storage..onstackit.cloud. Can also be set with the object_storage_data_plane_custom_endpoint config key or the STACKIT_OBJECT_STORAGE_DATA_PLANE_CUSTOM_ENDPOINT environment variable. + -h, --help[=true|false] Help for "stackit alpha bucket rb" + --secret-access-key string Object Storage secret access key. Can also be set with the object_storage_secret_access_key config key or the STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variable. +``` + +### Options inherited from parent commands + +``` + -y, --assume-yes[=true|false] If set, skips all confirmation prompts + --async[=true|false] If set, runs the command asynchronously + -o, --output-format string Output format, (one of: [json, pretty, none, yaml]) + -p, --project-id string Project ID + --region string Target region for region-specific requests + --verbosity string Verbosity of the CLI, (one of: [debug, info, warning, error]) (default "info") +``` + +### SEE ALSO + +* [stackit alpha bucket](./stackit_alpha_bucket.md) - Manages objects in STACKIT Object Storage buckets (data plane) + diff --git a/docs/stackit_alpha_bucket_rm.md b/docs/stackit_alpha_bucket_rm.md new file mode 100644 index 000000000..f58ef5d32 --- /dev/null +++ b/docs/stackit_alpha_bucket_rm.md @@ -0,0 +1,50 @@ +## stackit alpha bucket rm + +Deletes objects + +### Synopsis + +Deletes a single object, or all objects under a prefix with --recursive. + +To delete an empty bucket, use 'stackit alpha bucket rb'. + +``` +stackit alpha bucket rm BUCKET_URI [flags] +``` + +### Examples + +``` + Delete a single object + $ stackit alpha bucket rm s3://my-bucket/file.txt + + Delete all objects under a prefix without prompting + $ stackit alpha bucket rm s3://my-bucket/path/to/ --recursive --force +``` + +### Options + +``` + --access-key-id string Object Storage access key ID. Can also be set with the object_storage_access_key_id config key or the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID environment variable. + --endpoint string Object Storage API endpoint. Defaults to https://object.storage..onstackit.cloud. Can also be set with the object_storage_data_plane_custom_endpoint config key or the STACKIT_OBJECT_STORAGE_DATA_PLANE_CUSTOM_ENDPOINT environment variable. + --force[=true|false] Do not prompt for confirmation + -h, --help[=true|false] Help for "stackit alpha bucket rm" + --recursive[=true|false] Delete all objects under the prefix + --secret-access-key string Object Storage secret access key. Can also be set with the object_storage_secret_access_key config key or the STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variable. +``` + +### Options inherited from parent commands + +``` + -y, --assume-yes[=true|false] If set, skips all confirmation prompts + --async[=true|false] If set, runs the command asynchronously + -o, --output-format string Output format, (one of: [json, pretty, none, yaml]) + -p, --project-id string Project ID + --region string Target region for region-specific requests + --verbosity string Verbosity of the CLI, (one of: [debug, info, warning, error]) (default "info") +``` + +### SEE ALSO + +* [stackit alpha bucket](./stackit_alpha_bucket.md) - Manages objects in STACKIT Object Storage buckets (data plane) + diff --git a/internal/cmd/alpha/alpha.go b/internal/cmd/alpha/alpha.go index dac1d5ce1..3b6971a99 100644 --- a/internal/cmd/alpha/alpha.go +++ b/internal/cmd/alpha/alpha.go @@ -5,6 +5,7 @@ import ( "github.com/spf13/cobra" + "github.com/stackitcloud/stackit-cli/internal/cmd/alpha/bucket" "github.com/stackitcloud/stackit-cli/internal/cmd/alpha/vpc" "github.com/stackitcloud/stackit-cli/internal/pkg/args" "github.com/stackitcloud/stackit-cli/internal/pkg/examples" @@ -35,5 +36,6 @@ func NewCmd(params *types.CmdParams) *cobra.Command { } func addSubcommands(cmd *cobra.Command, params *types.CmdParams) { + cmd.AddCommand(bucket.NewCmd(params)) cmd.AddCommand(vpc.NewCmd(params)) } diff --git a/internal/cmd/alpha/bucket/bucket.go b/internal/cmd/alpha/bucket/bucket.go new file mode 100644 index 000000000..c05db2436 --- /dev/null +++ b/internal/cmd/alpha/bucket/bucket.go @@ -0,0 +1,58 @@ +package bucket + +import ( + "github.com/spf13/cobra" + + "github.com/stackitcloud/stackit-cli/internal/cmd/alpha/bucket/cp" + "github.com/stackitcloud/stackit-cli/internal/cmd/alpha/bucket/ls" + "github.com/stackitcloud/stackit-cli/internal/cmd/alpha/bucket/mb" + "github.com/stackitcloud/stackit-cli/internal/cmd/alpha/bucket/mv" + "github.com/stackitcloud/stackit-cli/internal/cmd/alpha/bucket/presign" + "github.com/stackitcloud/stackit-cli/internal/cmd/alpha/bucket/rb" + "github.com/stackitcloud/stackit-cli/internal/cmd/alpha/bucket/rm" + "github.com/stackitcloud/stackit-cli/internal/pkg/args" + "github.com/stackitcloud/stackit-cli/internal/pkg/examples" + "github.com/stackitcloud/stackit-cli/internal/pkg/types" + "github.com/stackitcloud/stackit-cli/internal/pkg/utils" +) + +// The bucket command group is in alpha because the Object Storage data-plane CLI integration is still experimental. +func NewCmd(params *types.CmdParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "bucket", + Short: "Manages objects in STACKIT Object Storage buckets (data plane)", + Long: `Manages objects in STACKIT Object Storage buckets. + +Buckets and objects can be addressed with s3 URIs (s3://BUCKET[/KEY]) or +STACKIT Object Storage URLs in virtual hosted style +(https://BUCKET.object.storage.REGION.onstackit.cloud[/KEY]) or path style +(https://object.storage.REGION.onstackit.cloud/BUCKET[/KEY]). + +Credentials (access key ID / secret access key) are required and can be provided +with the --access-key-id and --secret-access-key flags, the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID +and STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variables, or the config file. +Create credentials with 'stackit object-storage credentials create'.`, + Args: args.NoArgs, + Run: utils.CmdHelp, + Example: examples.Build( + examples.NewExample( + "List all buckets", + "$ stackit alpha bucket ls"), + examples.NewExample( + "Upload a file to a bucket", + `$ stackit alpha bucket cp local/file.txt s3://my-bucket/file.txt`), + ), + } + addSubcommands(cmd, params) + return cmd +} + +func addSubcommands(cmd *cobra.Command, params *types.CmdParams) { + cmd.AddCommand(cp.NewCmd(params)) + cmd.AddCommand(ls.NewCmd(params)) + cmd.AddCommand(mb.NewCmd(params)) + cmd.AddCommand(mv.NewCmd(params)) + cmd.AddCommand(presign.NewCmd(params)) + cmd.AddCommand(rb.NewCmd(params)) + cmd.AddCommand(rm.NewCmd(params)) +} diff --git a/internal/cmd/alpha/bucket/cp/cp.go b/internal/cmd/alpha/bucket/cp/cp.go new file mode 100644 index 000000000..47f868798 --- /dev/null +++ b/internal/cmd/alpha/bucket/cp/cp.go @@ -0,0 +1,139 @@ +package cp + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/stackitcloud/stackit-cli/internal/pkg/examples" + "github.com/stackitcloud/stackit-cli/internal/pkg/flags" + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/print" + "github.com/stackitcloud/stackit-cli/internal/pkg/services/object-storage/bucket" + "github.com/stackitcloud/stackit-cli/internal/pkg/spinner" + "github.com/stackitcloud/stackit-cli/internal/pkg/types" +) + +const ( + srcArg = "SRC" + dstArg = "DESTINATION" + recursiveFlag = "recursive" +) + +type inputModel struct { + *globalflags.GlobalFlagModel + bucket.ClientFlags + Source string + Destination string + Recursive bool +} + +type copyResult struct { + Objects int `json:"objects"` +} + +func NewCmd(params *types.CmdParams) *cobra.Command { + cmd := &cobra.Command{ + Use: fmt.Sprintf("cp %s %s", srcArg, dstArg), + Short: "Copies objects between local and Object Storage", + Long: `Copies a single object or a directory tree between local storage and Object Storage. + +One of SRC and DESTINATION must be a bucket URI (s3://BUCKET[/KEY]) or a STACKIT +Object Storage URL (https://BUCKET.object.storage.REGION.onstackit.cloud[/KEY]) +and the other a local path. +Without --recursive, a single object is transferred. +With --recursive, all objects under the prefix (download) or all files under the +local directory (upload) are transferred.`, + Args: cobra.ExactArgs(2), + Example: examples.Build( + examples.NewExample( + "Upload a file to a bucket", + "$ stackit alpha bucket cp local/file.txt s3://my-bucket/file.txt"), + examples.NewExample( + "Download a single object", + "$ stackit alpha bucket cp s3://my-bucket/file.txt local/file.txt"), + examples.NewExample( + "Upload a directory recursively", + "$ stackit alpha bucket cp ./local/dir s3://my-bucket/prefix/ --recursive"), + examples.NewExample( + "Download a prefix recursively", + "$ stackit alpha bucket cp s3://my-bucket/prefix/ ./local/dir --recursive"), + ), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + model, err := parseInput(params.Printer, cmd, args) + if err != nil { + return err + } + + srcIsRef := bucket.IsObjectRef(model.Source) + dstIsRef := bucket.IsObjectRef(model.Destination) + if srcIsRef == dstIsRef { + return fmt.Errorf("one of %s and %s must be a bucket URI (s3://...) or a STACKIT Object Storage URL (https://...) and the other a local path", srcArg, dstArg) + } + + refURI := model.Destination + if srcIsRef { + refURI = model.Source + } + ref, err := bucket.ParseBucketURI(refURI) + if err != nil { + return err + } + + apiClient, err := bucket.NewClient(bucket.ClientConfigFromReference(model.ClientFlags, model.Region, ref)) + if err != nil { + return err + } + + var count int + var direction string + if srcIsRef { + count, err = spinner.Run2(params.Printer, "Downloading", func() (int, error) { + return bucket.Download(ctx, apiClient, ref, model.Destination, model.Recursive) + }) + if err != nil { + return err + } + direction = fmt.Sprintf("Downloaded %d object(s) to %q", count, model.Destination) + } else { + count, err = spinner.Run2(params.Printer, "Uploading", func() (int, error) { + return bucket.Upload(ctx, apiClient, model.Source, ref, model.Recursive) + }) + if err != nil { + return err + } + direction = fmt.Sprintf("Uploaded %d object(s) to %q", count, ref.String()) + } + + return outputResult(params.Printer, model.OutputFormat, direction, count) + }, + } + configureFlags(cmd) + return cmd +} + +func configureFlags(cmd *cobra.Command) { + bucket.ConfigureClientFlags(cmd.Flags()) + cmd.Flags().Bool(recursiveFlag, false, "Transfer a directory tree or all objects under a prefix") +} + +func parseInput(p *print.Printer, cmd *cobra.Command, inputArgs []string) (*inputModel, error) { + model := inputModel{ + GlobalFlagModel: globalflags.Parse(p, cmd), + ClientFlags: bucket.ClientFlagsFromCommand(p, cmd), + Source: inputArgs[0], + Destination: inputArgs[1], + Recursive: flags.FlagToBoolValue(p, cmd, recursiveFlag), + } + p.DebugInputModel(model) + return &model, nil +} + +func outputResult(p *print.Printer, outputFormat, direction string, objects int) error { + return p.OutputResult(outputFormat, copyResult{Objects: objects}, func() error { + p.Outputln(direction) + return nil + }) +} diff --git a/internal/cmd/alpha/bucket/cp/cp_test.go b/internal/cmd/alpha/bucket/cp/cp_test.go new file mode 100644 index 000000000..0acc3454b --- /dev/null +++ b/internal/cmd/alpha/bucket/cp/cp_test.go @@ -0,0 +1,109 @@ +package cp + +import ( + "testing" + + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/testparams" + "github.com/stackitcloud/stackit-cli/internal/pkg/testutils" +) + +const ( + testRegion = "eu01" + testSrc = "./local/file.txt" + testDst = "s3://my-bucket/file.txt" + testSrcRef = "s3://my-bucket/file.txt" + testDstLocal = "./local/file.txt" +) + +func fixtureFlagValues(mods ...func(flagValues map[string]string)) map[string]string { + flagValues := map[string]string{ + globalflags.RegionFlag: testRegion, + } + for _, mod := range mods { + mod(flagValues) + } + return flagValues +} + +func fixtureInputModel(mods ...func(model *inputModel)) *inputModel { + model := &inputModel{ + GlobalFlagModel: &globalflags.GlobalFlagModel{ + Verbosity: globalflags.VerbosityDefault, + Region: testRegion, + }, + Source: testSrc, + Destination: testDst, + } + for _, mod := range mods { + mod(model) + } + return model +} + +func TestParseInput(t *testing.T) { + tests := []struct { + description string + argValues []string + flagValues map[string]string + isValid bool + expectedModel *inputModel + }{ + { + description: "upload", + argValues: []string{testSrc, testDst}, + flagValues: fixtureFlagValues(), + isValid: true, + expectedModel: fixtureInputModel(), + }, + { + description: "download", + argValues: []string{testSrcRef, testDstLocal}, + flagValues: fixtureFlagValues(), + isValid: true, + expectedModel: fixtureInputModel(func(model *inputModel) { + model.Source = testSrcRef + model.Destination = testDstLocal + }), + }, + { + description: "with recursive", + argValues: []string{testSrc, testDst}, + flagValues: fixtureFlagValues(func(flagValues map[string]string) { + flagValues[recursiveFlag] = "true" + }), + isValid: true, + expectedModel: fixtureInputModel(func(model *inputModel) { + model.Recursive = true + }), + }, + { + description: "too few args", + argValues: []string{testSrc}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + { + description: "too many args", + argValues: []string{testSrc, testDst, "extra"}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + testutils.TestParseInput(t, NewCmd, parseInput, tt.expectedModel, tt.argValues, tt.flagValues, tt.isValid) + }) + } +} + +func TestOutputResult(t *testing.T) { + params := testparams.NewTestParams() + if err := outputResult(params.Printer, "", "Uploaded 1 object(s) to \"s3://my-bucket/file.txt\"", 1); err != nil { + t.Fatalf("outputResult() unexpected error: %v", err) + } + if output := params.Out.String(); output == "" { + t.Fatalf("outputResult() expected non-empty output") + } +} diff --git a/internal/cmd/alpha/bucket/ls/ls.go b/internal/cmd/alpha/bucket/ls/ls.go new file mode 100644 index 000000000..2c24e28e1 --- /dev/null +++ b/internal/cmd/alpha/bucket/ls/ls.go @@ -0,0 +1,166 @@ +package ls + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/stackitcloud/stackit-cli/internal/pkg/args" + "github.com/stackitcloud/stackit-cli/internal/pkg/examples" + "github.com/stackitcloud/stackit-cli/internal/pkg/flags" + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/print" + "github.com/stackitcloud/stackit-cli/internal/pkg/services/object-storage/bucket" + "github.com/stackitcloud/stackit-cli/internal/pkg/spinner" + "github.com/stackitcloud/stackit-cli/internal/pkg/types" +) + +const ( + objectURIArg = "BUCKET_URI" + recursiveFlag = "recursive" + + timeFormat = "2006-01-02 15:04:05" +) + +type inputModel struct { + *globalflags.GlobalFlagModel + bucket.ClientFlags + URI string + Recursive bool +} + +// listResult is the JSON/YAML output of a bucket object listing +type listResult struct { + Prefixes []string `json:"prefixes,omitempty"` + Objects []bucket.ObjectInfo `json:"objects,omitempty"` +} + +func NewCmd(params *types.CmdParams) *cobra.Command { + cmd := &cobra.Command{ + Use: fmt.Sprintf("ls [%s]", objectURIArg), + Short: "Lists buckets and objects", + Long: `Lists all buckets, or the contents of a bucket. + +Without an argument, lists all buckets. +With a bucket URI (s3://BUCKET), lists the "directories" and objects at the bucket root. +With an object URI (s3://BUCKET/PREFIX), lists the "directories" and objects under the prefix. +With --recursive, lists all objects under the prefix flat, including all nested keys.`, + Args: args.SingleOptionalArg(objectURIArg, nil), + Example: examples.Build( + examples.NewExample( + "List all buckets", + "$ stackit alpha bucket ls"), + examples.NewExample( + "List the contents of a bucket", + "$ stackit alpha bucket ls s3://my-bucket"), + examples.NewExample( + "Recursively list all objects under a prefix", + "$ stackit alpha bucket ls s3://my-bucket/path/to/ --recursive"), + ), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + model, err := parseInput(params.Printer, cmd, args) + if err != nil { + return err + } + + ref := bucket.Reference{} + if model.URI != "" { + ref, err = bucket.ParseBucketURI(model.URI) + if err != nil { + return err + } + } + + apiClient, err := bucket.NewClient(bucket.ClientConfigFromReference(model.ClientFlags, model.Region, ref)) + if err != nil { + return err + } + + if model.URI == "" { + buckets, err := spinner.Run2(params.Printer, "Listing buckets", func() ([]bucket.BucketInfo, error) { + return bucket.ListBuckets(ctx, apiClient) + }) + if err != nil { + return err + } + return outputResultBuckets(params.Printer, model.OutputFormat, buckets) + } + + if model.Recursive { + objects, err := spinner.Run2(params.Printer, "Listing objects", func() ([]bucket.ObjectInfo, error) { + return bucket.ListObjects(ctx, apiClient, ref) + }) + if err != nil { + return err + } + return outputResultObjects(params.Printer, model.OutputFormat, nil, objects) + } + + result, err := spinner.Run2(params.Printer, "Listing objects", func() (listResult, error) { + prefixes, objects, err := bucket.ListPrefixes(ctx, apiClient, ref) + return listResult{Prefixes: prefixes, Objects: objects}, err + }) + if err != nil { + return err + } + return outputResultObjects(params.Printer, model.OutputFormat, result.Prefixes, result.Objects) + }, + } + configureFlags(cmd) + return cmd +} + +func configureFlags(cmd *cobra.Command) { + bucket.ConfigureClientFlags(cmd.Flags()) + cmd.Flags().Bool(recursiveFlag, false, "List all objects under the prefix recursively, including nested keys") +} + +func parseInput(p *print.Printer, cmd *cobra.Command, inputArgs []string) (*inputModel, error) { + uri := "" + if len(inputArgs) > 0 { + uri = inputArgs[0] + } + model := inputModel{ + GlobalFlagModel: globalflags.Parse(p, cmd), + ClientFlags: bucket.ClientFlagsFromCommand(p, cmd), + URI: uri, + Recursive: flags.FlagToBoolValue(p, cmd, recursiveFlag), + } + p.DebugInputModel(model) + return &model, nil +} + +func outputResultBuckets(p *print.Printer, outputFormat string, buckets []bucket.BucketInfo) error { + return p.OutputResult(outputFormat, buckets, func() error { + for _, bucket := range buckets { + creationDate := "n/a" + if !bucket.CreationDate.IsZero() { + creationDate = bucket.CreationDate.Format(timeFormat) + } + p.Outputf("%s\t%s\n", creationDate, bucket.Name) + } + return nil + }) +} + +func outputResultObjects(p *print.Printer, outputFormat string, prefixes []string, objects []bucket.ObjectInfo) error { + result := listResult{ + Prefixes: prefixes, + Objects: objects, + } + return p.OutputResult(outputFormat, result, func() error { + for _, prefix := range prefixes { + p.Outputln(prefix) + } + for _, object := range objects { + modified := "n/a" + if !object.LastModified.IsZero() { + modified = object.LastModified.Format(timeFormat) + } + p.Outputf("%s\t%d\t%s\n", modified, object.Size, object.Key) + } + return nil + }) +} diff --git a/internal/cmd/alpha/bucket/ls/ls_test.go b/internal/cmd/alpha/bucket/ls/ls_test.go new file mode 100644 index 000000000..73eaeffc1 --- /dev/null +++ b/internal/cmd/alpha/bucket/ls/ls_test.go @@ -0,0 +1,117 @@ +package ls + +import ( + "testing" + + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/testparams" + "github.com/stackitcloud/stackit-cli/internal/pkg/testutils" + + "github.com/stackitcloud/stackit-cli/internal/pkg/services/object-storage/bucket" +) + +const ( + testRegion = "eu01" + testURI = "s3://my-bucket/prefix/" +) + +func fixtureFlagValues(mods ...func(flagValues map[string]string)) map[string]string { + flagValues := map[string]string{ + globalflags.RegionFlag: testRegion, + } + for _, mod := range mods { + mod(flagValues) + } + return flagValues +} + +func fixtureInputModel(mods ...func(model *inputModel)) *inputModel { + model := &inputModel{ + GlobalFlagModel: &globalflags.GlobalFlagModel{ + Verbosity: globalflags.VerbosityDefault, + Region: testRegion, + }, + } + for _, mod := range mods { + mod(model) + } + return model +} + +func TestParseInput(t *testing.T) { + tests := []struct { + description string + argValues []string + flagValues map[string]string + isValid bool + expectedModel *inputModel + }{ + { + description: "no uri", + argValues: []string{}, + flagValues: fixtureFlagValues(), + isValid: true, + expectedModel: fixtureInputModel(), + }, + { + description: "with uri", + argValues: []string{testURI}, + flagValues: fixtureFlagValues(), + isValid: true, + expectedModel: fixtureInputModel(func(model *inputModel) { + model.URI = testURI + }), + }, + { + description: "with recursive", + argValues: []string{testURI}, + flagValues: fixtureFlagValues(func(flagValues map[string]string) { + flagValues[recursiveFlag] = "true" + }), + isValid: true, + expectedModel: fixtureInputModel(func(model *inputModel) { + model.URI = testURI + model.Recursive = true + }), + }, + { + description: "too many args", + argValues: []string{testURI, "extra"}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + testutils.TestParseInput(t, NewCmd, parseInput, tt.expectedModel, tt.argValues, tt.flagValues, tt.isValid) + }) + } +} + +func TestOutputResultObjects(t *testing.T) { + params := testparams.NewTestParams() + prefixes := []string{"dir1/"} + objects := []bucket.ObjectInfo{ + {Key: "dir1/file.txt", Size: 123}, + } + if err := outputResultObjects(params.Printer, "", prefixes, objects); err != nil { + t.Fatalf("outputResultObjects() unexpected error: %v", err) + } + if output := params.Out.String(); output == "" { + t.Fatalf("outputResultObjects() expected non-empty output") + } +} + +func TestOutputResultBuckets(t *testing.T) { + params := testparams.NewTestParams() + buckets := []bucket.BucketInfo{ + {Name: "my-bucket"}, + } + if err := outputResultBuckets(params.Printer, "", buckets); err != nil { + t.Fatalf("outputResultBuckets() unexpected error: %v", err) + } + if output := params.Out.String(); output == "" { + t.Fatalf("outputResultBuckets() expected non-empty output") + } +} diff --git a/internal/cmd/alpha/bucket/mb/mb.go b/internal/cmd/alpha/bucket/mb/mb.go new file mode 100644 index 000000000..01d452b6b --- /dev/null +++ b/internal/cmd/alpha/bucket/mb/mb.go @@ -0,0 +1,125 @@ +package mb + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/stackitcloud/stackit-cli/internal/pkg/args" + "github.com/stackitcloud/stackit-cli/internal/pkg/examples" + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/print" + "github.com/stackitcloud/stackit-cli/internal/pkg/services/object-storage/bucket" + "github.com/stackitcloud/stackit-cli/internal/pkg/spinner" + "github.com/stackitcloud/stackit-cli/internal/pkg/types" +) + +const ( + bucketURIArg = "BUCKET_URI" + defaultRegion = "eu01" +) + +type inputModel struct { + *globalflags.GlobalFlagModel + bucket.ClientFlags + URI string +} + +type bucketResult struct { + Bucket string `json:"bucket"` +} + +func validateBucketURI(uri string) error { + ref, err := bucket.ParseBucketURI(uri) + if err != nil { + return err + } + if !ref.IsBucket() { + return fmt.Errorf("expected a bucket URI (s3://BUCKET or https://BUCKET.object.storage.REGION.onstackit.cloud), got %q", uri) + } + return nil +} + +func NewCmd(params *types.CmdParams) *cobra.Command { + cmd := &cobra.Command{ + Use: fmt.Sprintf("mb %s", bucketURIArg), + Short: "Creates a bucket", + Long: "Creates a bucket in STACKIT Object Storage.", + Args: args.SingleArg(bucketURIArg, validateBucketURI), + Example: examples.Build( + examples.NewExample( + "Create a bucket", + "$ stackit alpha bucket mb s3://my-bucket"), + ), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + model, err := parseInput(params.Printer, cmd, args) + if err != nil { + return err + } + + ref, err := bucket.ParseBucketURI(model.URI) + if err != nil { + return err + } + region := model.Region + if ref.Region != "" { + region = ref.Region + } + + apiClient, err := bucket.NewClient(bucket.ClientConfigFromReference(model.ClientFlags, model.Region, ref)) + if err != nil { + return err + } + + prompt := fmt.Sprintf("Are you sure you want to create bucket %q?", model.URI) + err = params.Printer.PromptForConfirmation(prompt) + if err != nil { + return err + } + + err = spinner.Run(params.Printer, "Creating bucket", func() error { + return apiClient.CreateBucket(ctx, ref.Bucket, locationConstraint(region)) + }) + if err != nil { + return fmt.Errorf("create bucket: %w", err) + } + + return outputResult(params.Printer, model.OutputFormat, model.URI) + }, + } + configureFlags(cmd) + return cmd +} + +func configureFlags(cmd *cobra.Command) { + bucket.ConfigureClientFlags(cmd.Flags()) +} + +func parseInput(p *print.Printer, cmd *cobra.Command, inputArgs []string) (*inputModel, error) { + model := inputModel{ + GlobalFlagModel: globalflags.Parse(p, cmd), + ClientFlags: bucket.ClientFlagsFromCommand(p, cmd), + URI: inputArgs[0], + } + p.DebugInputModel(model) + return &model, nil +} + +// locationConstraint returns the LocationConstraint for the region, which +// must be omitted for the default region (the constraint value is a region +// code, and the default region requires no constraint body) +func locationConstraint(region string) string { + if region == "" || region == defaultRegion { + return "" + } + return region +} + +func outputResult(p *print.Printer, outputFormat, uri string) error { + return p.OutputResult(outputFormat, bucketResult{Bucket: uri}, func() error { + p.Outputf("Created bucket %q\n", uri) + return nil + }) +} diff --git a/internal/cmd/alpha/bucket/mb/mb_test.go b/internal/cmd/alpha/bucket/mb/mb_test.go new file mode 100644 index 000000000..bbd82e942 --- /dev/null +++ b/internal/cmd/alpha/bucket/mb/mb_test.go @@ -0,0 +1,129 @@ +package mb + +import ( + "testing" + + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/testparams" + "github.com/stackitcloud/stackit-cli/internal/pkg/testutils" +) + +const ( + testRegion = "eu01" + testBucket = "s3://my-bucket" +) + +func fixtureArgValues(mods ...func(argValues []string)) []string { + argValues := []string{testBucket} + for _, mod := range mods { + mod(argValues) + } + return argValues +} + +func fixtureFlagValues(mods ...func(flagValues map[string]string)) map[string]string { + flagValues := map[string]string{ + globalflags.RegionFlag: testRegion, + } + for _, mod := range mods { + mod(flagValues) + } + return flagValues +} + +func fixtureInputModel(mods ...func(model *inputModel)) *inputModel { + model := &inputModel{ + GlobalFlagModel: &globalflags.GlobalFlagModel{ + Verbosity: globalflags.VerbosityDefault, + Region: testRegion, + }, + URI: testBucket, + } + for _, mod := range mods { + mod(model) + } + return model +} + +func TestParseInput(t *testing.T) { + tests := []struct { + description string + argValues []string + flagValues map[string]string + isValid bool + expectedModel *inputModel + }{ + { + description: "base", + argValues: fixtureArgValues(), + flagValues: fixtureFlagValues(), + isValid: true, + expectedModel: fixtureInputModel(), + }, + { + description: "no values", + argValues: []string{}, + flagValues: map[string]string{}, + isValid: false, + }, + { + description: "no arg values", + argValues: []string{}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + { + description: "invalid uri, wrong scheme", + argValues: []string{"https://my-bucket"}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + { + description: "invalid uri, object instead of bucket", + argValues: []string{"s3://my-bucket/key"}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + { + description: "invalid uri, invalid bucket name", + argValues: []string{"s3://My-Bucket"}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + testutils.TestParseInput(t, NewCmd, parseInput, tt.expectedModel, tt.argValues, tt.flagValues, tt.isValid) + }) + } +} + +func TestOutputResult(t *testing.T) { + params := testparams.NewTestParams() + if err := outputResult(params.Printer, "", testBucket); err != nil { + t.Fatalf("outputResult() unexpected error: %v", err) + } + if output := params.Out.String(); output == "" { + t.Fatalf("outputResult() expected non-empty output") + } +} + +func TestLocationConstraint(t *testing.T) { + tests := []struct { + description string + region string + expected string + }{ + {description: "empty region omits the constraint", region: "", expected: ""}, + {description: "default region omits the constraint", region: defaultRegion, expected: ""}, + {description: "other region", region: "eu02", expected: "eu02"}, + } + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + if got := locationConstraint(tt.region); got != tt.expected { + t.Fatalf("locationConstraint(%q) = %q, want %q", tt.region, got, tt.expected) + } + }) + } +} diff --git a/internal/cmd/alpha/bucket/mv/mv.go b/internal/cmd/alpha/bucket/mv/mv.go new file mode 100644 index 000000000..9ad654082 --- /dev/null +++ b/internal/cmd/alpha/bucket/mv/mv.go @@ -0,0 +1,242 @@ +package mv + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + + "github.com/stackitcloud/stackit-cli/internal/pkg/examples" + "github.com/stackitcloud/stackit-cli/internal/pkg/flags" + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/print" + "github.com/stackitcloud/stackit-cli/internal/pkg/services/object-storage/bucket" + "github.com/stackitcloud/stackit-cli/internal/pkg/spinner" + "github.com/stackitcloud/stackit-cli/internal/pkg/types" +) + +const ( + srcArg = "SRC" + dstArg = "DESTINATION" + recursiveFlag = "recursive" +) + +type inputModel struct { + *globalflags.GlobalFlagModel + bucket.ClientFlags + Source string + Destination string + Recursive bool +} + +type moveResult struct { + Objects int `json:"objects"` +} + +func NewCmd(params *types.CmdParams) *cobra.Command { + cmd := &cobra.Command{ + Use: fmt.Sprintf("mv %s %s", srcArg, dstArg), + Short: "Moves objects between local and Object Storage", + Long: `Moves a single object or a directory tree, then deletes the source. + +One of SRC and DESTINATION must be a bucket URI (s3://BUCKET[/KEY]) or a STACKIT +Object Storage URL (https://BUCKET.object.storage.REGION.onstackit.cloud[/KEY]) +and the other a local path. +Bucket-to-bucket moves use a server-side copy. +Without --recursive, a single object is moved. +With --recursive, all objects under the prefix (or all files under the local directory) are moved.`, + Args: cobra.ExactArgs(2), + Example: examples.Build( + examples.NewExample( + "Move a file to a bucket", + "$ stackit alpha bucket mv local/file.txt s3://my-bucket/file.txt"), + examples.NewExample( + "Move an object to a new key", + "$ stackit alpha bucket mv s3://my-bucket/old/file.txt s3://my-bucket/new/file.txt"), + examples.NewExample( + "Move a prefix to another bucket recursively", + "$ stackit alpha bucket mv s3://src-bucket/prefix/ s3://dst-bucket/prefix/ --recursive"), + ), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + model, err := parseInput(params.Printer, cmd, args) + if err != nil { + return err + } + + srcIsRef := bucket.IsObjectRef(model.Source) + dstIsRef := bucket.IsObjectRef(model.Destination) + if !srcIsRef && !dstIsRef { + return fmt.Errorf("one of %s and %s must be a bucket URI (s3://...) or a STACKIT Object Storage URL (https://...)", srcArg, dstArg) + } + + refURI := model.Destination + if srcIsRef { + refURI = model.Source + } + ref, err := bucket.ParseBucketURI(refURI) + if err != nil { + return err + } + + apiClient, err := bucket.NewClient(bucket.ClientConfigFromReference(model.ClientFlags, model.Region, ref)) + if err != nil { + return err + } + + prompt := fmt.Sprintf("Are you sure you want to move %q to %q? The source will be deleted", model.Source, model.Destination) + err = params.Printer.PromptForConfirmation(prompt) + if err != nil { + return err + } + + count, err := spinner.Run2(params.Printer, "Moving", func() (int, error) { + return runMove(ctx, apiClient, model, srcIsRef) + }) + if err != nil { + return err + } + + direction := fmt.Sprintf("Moved %d object(s) from %q to %q", count, model.Source, model.Destination) + return outputResult(params.Printer, model.OutputFormat, direction, count) + }, + } + configureFlags(cmd) + return cmd +} + +func configureFlags(cmd *cobra.Command) { + bucket.ConfigureClientFlags(cmd.Flags()) + cmd.Flags().Bool(recursiveFlag, false, "Move a directory tree or all objects under a prefix") +} + +func parseInput(p *print.Printer, cmd *cobra.Command, inputArgs []string) (*inputModel, error) { + model := inputModel{ + GlobalFlagModel: globalflags.Parse(p, cmd), + ClientFlags: bucket.ClientFlagsFromCommand(p, cmd), + Source: inputArgs[0], + Destination: inputArgs[1], + Recursive: flags.FlagToBoolValue(p, cmd, recursiveFlag), + } + p.DebugInputModel(model) + return &model, nil +} + +func runMove(ctx context.Context, apiClient *bucket.Client, model *inputModel, srcIsRef bool) (int, error) { + if !srcIsRef { + dstRef, err := bucket.ParseBucketURI(model.Destination) + if err != nil { + return 0, err + } + count, err := bucket.Upload(ctx, apiClient, model.Source, dstRef, model.Recursive) + if err != nil { + return 0, err + } + if err := deleteLocal(model.Source); err != nil { + return count, fmt.Errorf("objects were uploaded to %q, but deleting the local source failed: %w", dstRef.String(), err) + } + return count, nil + } + + srcRef, err := bucket.ParseBucketURI(model.Source) + if err != nil { + return 0, err + } + if bucket.IsObjectRef(model.Destination) { + dstRef, err := bucket.ParseBucketURI(model.Destination) + if err != nil { + return 0, err + } + if isCrossEndpointMove(srcRef, dstRef) { + return 0, fmt.Errorf("cross-region or cross-endpoint bucket-to-bucket moves are not supported: %q and %q resolve to different endpoints or regions", model.Source, model.Destination) + } + return moveBucketToBucket(ctx, apiClient, srcRef, model) + } + count, err := bucket.Download(ctx, apiClient, srcRef, model.Destination, model.Recursive) + if err != nil { + return 0, err + } + if err := deleteSource(ctx, apiClient, srcRef, model.Recursive); err != nil { + return count, fmt.Errorf("objects were downloaded to %q, but deleting the source failed: %w", model.Destination, err) + } + return count, nil +} + +// isCrossEndpointMove reports whether src and dst explicitly resolve to +// different Object Storage endpoints or regions. s3:// references carry no endpoint or +// region (they are derived from the --region flag), so only explicit +// mismatches are detected. +func isCrossEndpointMove(src, dst bucket.Reference) bool { + return (src.Endpoint != "" && dst.Endpoint != "" && src.Endpoint != dst.Endpoint) || + (src.Region != "" && dst.Region != "" && src.Region != dst.Region) +} + +func moveBucketToBucket(ctx context.Context, apiClient *bucket.Client, srcRef bucket.Reference, model *inputModel) (int, error) { + dstRef, err := bucket.ParseBucketURI(model.Destination) + if err != nil { + return 0, err + } + srcs, err := bucket.ResolveObjectRefs(ctx, apiClient, srcRef, model.Recursive) + if err != nil { + return 0, err + } + + moved := 0 + for _, src := range srcs { + dst := destinationForMove(src, srcRef, dstRef, model.Recursive) + if err := bucket.CopyObject(ctx, apiClient, src, dst); err != nil { + return moved, err + } + moved++ + } + _, err = bucket.DeleteObjects(ctx, apiClient, srcs) + if err != nil { + return moved, fmt.Errorf("%d object(s) were copied to %q, but deleting the source failed: %w", moved, dstRef.String(), err) + } + return moved, nil +} + +func destinationForMove(src, srcRef, dstRef bucket.Reference, recursive bool) bucket.Reference { + if !recursive { + if dstRef.IsBucket() { + return bucket.Reference{Bucket: dstRef.Bucket, Key: src.Key} + } + return dstRef + } + rel := strings.TrimPrefix(src.Key, srcRef.Key) + rel = strings.TrimPrefix(rel, "/") + base := strings.TrimSuffix(dstRef.Key, "/") + if base == "" { + return bucket.Reference{Bucket: dstRef.Bucket, Key: rel} + } + return bucket.Reference{Bucket: dstRef.Bucket, Key: base + "/" + rel} +} + +func deleteSource(ctx context.Context, apiClient *bucket.Client, ref bucket.Reference, recursive bool) error { + refs, err := bucket.ResolveObjectRefs(ctx, apiClient, ref, recursive) + if err != nil { + return err + } + _, err = bucket.DeleteObjects(ctx, apiClient, refs) + return err +} + +func deleteLocal(path string) error { + stat, err := os.Stat(path) + if err != nil { + return fmt.Errorf("stat local source %q: %w", path, err) + } + if stat.IsDir() { + return os.RemoveAll(path) + } + return os.Remove(path) +} + +func outputResult(p *print.Printer, outputFormat, direction string, objects int) error { + return p.OutputResult(outputFormat, moveResult{Objects: objects}, func() error { + p.Outputln(direction) + return nil + }) +} diff --git a/internal/cmd/alpha/bucket/mv/mv_test.go b/internal/cmd/alpha/bucket/mv/mv_test.go new file mode 100644 index 000000000..31f271890 --- /dev/null +++ b/internal/cmd/alpha/bucket/mv/mv_test.go @@ -0,0 +1,119 @@ +package mv + +import ( + "testing" + + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/services/object-storage/bucket" + "github.com/stackitcloud/stackit-cli/internal/pkg/testparams" + "github.com/stackitcloud/stackit-cli/internal/pkg/testutils" +) + +const ( + testRegion = "eu01" + testSrc = "./local/file.txt" + testDst = "s3://my-bucket/file.txt" +) + +func fixtureFlagValues(mods ...func(flagValues map[string]string)) map[string]string { + flagValues := map[string]string{ + globalflags.RegionFlag: testRegion, + } + for _, mod := range mods { + mod(flagValues) + } + return flagValues +} + +func fixtureInputModel(mods ...func(model *inputModel)) *inputModel { + model := &inputModel{ + GlobalFlagModel: &globalflags.GlobalFlagModel{ + Verbosity: globalflags.VerbosityDefault, + Region: testRegion, + }, + Source: testSrc, + Destination: testDst, + } + for _, mod := range mods { + mod(model) + } + return model +} + +func TestParseInput(t *testing.T) { + tests := []struct { + description string + argValues []string + flagValues map[string]string + isValid bool + expectedModel *inputModel + }{ + { + description: "base", + argValues: []string{testSrc, testDst}, + flagValues: fixtureFlagValues(), + isValid: true, + expectedModel: fixtureInputModel(), + }, + { + description: "with recursive", + argValues: []string{testSrc, testDst}, + flagValues: fixtureFlagValues(func(flagValues map[string]string) { + flagValues[recursiveFlag] = "true" + }), + isValid: true, + expectedModel: fixtureInputModel(func(model *inputModel) { + model.Recursive = true + }), + }, + { + description: "too few args", + argValues: []string{testSrc}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + testutils.TestParseInput(t, NewCmd, parseInput, tt.expectedModel, tt.argValues, tt.flagValues, tt.isValid) + }) + } +} + +func TestOutputResult(t *testing.T) { + params := testparams.NewTestParams() + if err := outputResult(params.Printer, "", "Moved 1 object(s) from \"./local/file.txt\" to \"s3://my-bucket/file.txt\"", 1); err != nil { + t.Fatalf("outputResult() unexpected error: %v", err) + } + if output := params.Out.String(); output == "" { + t.Fatalf("outputResult() expected non-empty output") + } +} + +func TestIsCrossEndpointMove(t *testing.T) { + srcEU01 := bucket.Reference{Bucket: "src", Endpoint: "https://object.storage.eu01.onstackit.cloud", Region: "eu01"} + dstEU01 := bucket.Reference{Bucket: "dst", Endpoint: "https://object.storage.eu01.onstackit.cloud", Region: "eu01"} + dstEU02 := bucket.Reference{Bucket: "dst", Endpoint: "https://object.storage.eu02.onstackit.cloud", Region: "eu02"} + bare := bucket.Reference{Bucket: "dst"} + + tests := []struct { + description string + src bucket.Reference + dst bucket.Reference + expected bool + }{ + {description: "same region", src: srcEU01, dst: dstEU01, expected: false}, + {description: "different regions", src: srcEU01, dst: dstEU02, expected: true}, + {description: "different endpoints, same region", src: bucket.Reference{Endpoint: "https://a.example"}, dst: bucket.Reference{Endpoint: "https://b.example"}, expected: true}, + {description: "one side without region", src: srcEU01, dst: bare, expected: false}, + {description: "both plain references", src: bare, dst: bare, expected: false}, + } + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + if got := isCrossEndpointMove(tt.src, tt.dst); got != tt.expected { + t.Fatalf("isCrossEndpointMove = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/internal/cmd/alpha/bucket/presign/presign.go b/internal/cmd/alpha/bucket/presign/presign.go new file mode 100644 index 000000000..558078f3d --- /dev/null +++ b/internal/cmd/alpha/bucket/presign/presign.go @@ -0,0 +1,120 @@ +package presign + +import ( + "context" + "fmt" + "time" + + "github.com/spf13/cobra" + + "github.com/stackitcloud/stackit-cli/internal/pkg/args" + "github.com/stackitcloud/stackit-cli/internal/pkg/examples" + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/print" + "github.com/stackitcloud/stackit-cli/internal/pkg/services/object-storage/bucket" + "github.com/stackitcloud/stackit-cli/internal/pkg/types" +) + +const ( + objectURIArg = "BUCKET_URI" + expiresFlag = "expires" + + defaultExpires = time.Hour + maxExpires = 7 * 24 * time.Hour +) + +type inputModel struct { + *globalflags.GlobalFlagModel + bucket.ClientFlags + URI string + Expires time.Duration +} + +type presignResult struct { + URL string `json:"url"` +} + +func validateObjectURI(uri string) error { + ref, err := bucket.ParseBucketURI(uri) + if err != nil { + return err + } + if ref.IsBucket() { + return fmt.Errorf("expected an object URI (s3://BUCKET/KEY or https://BUCKET.object.storage.REGION.onstackit.cloud/KEY), got %q", uri) + } + return nil +} + +func NewCmd(params *types.CmdParams) *cobra.Command { + cmd := &cobra.Command{ + Use: fmt.Sprintf("presign %s", objectURIArg), + Short: "Generates a presigned URL for an object", + Long: `Generates a presigned URL that grants time-limited read access to an object +without exposing the Object Storage credentials.`, + Args: args.SingleArg(objectURIArg, validateObjectURI), + Example: examples.Build( + examples.NewExample( + "Generate a presigned URL valid for 1 hour (default)", + "$ stackit alpha bucket presign s3://my-bucket/file.txt"), + examples.NewExample( + "Generate a presigned URL valid for 15 minutes", + "$ stackit alpha bucket presign s3://my-bucket/file.txt --expires 15m"), + ), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + model, err := parseInput(params.Printer, cmd, args) + if err != nil { + return err + } + if model.Expires <= 0 || model.Expires > maxExpires { + return fmt.Errorf("--%s must be between 0 and %s", expiresFlag, maxExpires) + } + + ref, err := bucket.ParseBucketURI(model.URI) + if err != nil { + return err + } + + apiClient, err := bucket.NewClient(bucket.ClientConfigFromReference(model.ClientFlags, model.Region, ref)) + if err != nil { + return err + } + + presignedURL, err := bucket.PresignGetObject(ctx, apiClient, ref, model.Expires) + if err != nil { + return err + } + + return outputResult(params.Printer, model.OutputFormat, presignedURL) + }, + } + configureFlags(cmd) + return cmd +} + +func configureFlags(cmd *cobra.Command) { + bucket.ConfigureClientFlags(cmd.Flags()) + cmd.Flags().Duration(expiresFlag, defaultExpires, "Expiration of the presigned URL (e.g. 15m, 1h, 72h). Maximum: 7 days") +} + +func parseInput(p *print.Printer, cmd *cobra.Command, inputArgs []string) (*inputModel, error) { + model := inputModel{ + GlobalFlagModel: globalflags.Parse(p, cmd), + ClientFlags: bucket.ClientFlagsFromCommand(p, cmd), + URI: inputArgs[0], + } + expires, err := cmd.Flags().GetDuration(expiresFlag) + if err != nil { + return nil, fmt.Errorf("parse --%s flag: %w", expiresFlag, err) + } + model.Expires = expires + p.DebugInputModel(model) + return &model, nil +} + +func outputResult(p *print.Printer, outputFormat, url string) error { + return p.OutputResult(outputFormat, presignResult{URL: url}, func() error { + p.Outputf("%s\n", url) + return nil + }) +} diff --git a/internal/cmd/alpha/bucket/presign/presign_test.go b/internal/cmd/alpha/bucket/presign/presign_test.go new file mode 100644 index 000000000..bd9dfc119 --- /dev/null +++ b/internal/cmd/alpha/bucket/presign/presign_test.go @@ -0,0 +1,97 @@ +package presign + +import ( + "testing" + "time" + + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/testparams" + "github.com/stackitcloud/stackit-cli/internal/pkg/testutils" +) + +const ( + testRegion = "eu01" + testURI = "s3://my-bucket/file.txt" +) + +func fixtureFlagValues(mods ...func(flagValues map[string]string)) map[string]string { + flagValues := map[string]string{ + globalflags.RegionFlag: testRegion, + } + for _, mod := range mods { + mod(flagValues) + } + return flagValues +} + +func fixtureInputModel(mods ...func(model *inputModel)) *inputModel { + model := &inputModel{ + GlobalFlagModel: &globalflags.GlobalFlagModel{ + Verbosity: globalflags.VerbosityDefault, + Region: testRegion, + }, + URI: testURI, + Expires: defaultExpires, + } + for _, mod := range mods { + mod(model) + } + return model +} + +func TestParseInput(t *testing.T) { + tests := []struct { + description string + argValues []string + flagValues map[string]string + isValid bool + expectedModel *inputModel + }{ + { + description: "base", + argValues: []string{testURI}, + flagValues: fixtureFlagValues(), + isValid: true, + expectedModel: fixtureInputModel(), + }, + { + description: "with expires", + argValues: []string{testURI}, + flagValues: fixtureFlagValues(func(flagValues map[string]string) { + flagValues[expiresFlag] = "15m" + }), + isValid: true, + expectedModel: fixtureInputModel(func(model *inputModel) { + model.Expires = 15 * time.Minute + }), + }, + { + description: "no values", + argValues: []string{}, + flagValues: map[string]string{}, + isValid: false, + }, + { + description: "invalid uri, bucket only", + argValues: []string{"s3://my-bucket"}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + testutils.TestParseInput(t, NewCmd, parseInput, tt.expectedModel, tt.argValues, tt.flagValues, tt.isValid) + }) + } +} + +func TestOutputResult(t *testing.T) { + params := testparams.NewTestParams() + if err := outputResult(params.Printer, "", "https://bucket.eu01.stackit.cloud/my-bucket/file.txt?X-Amz-Signature=abc"); err != nil { + t.Fatalf("outputResult() unexpected error: %v", err) + } + if output := params.Out.String(); output == "" { + t.Fatalf("outputResult() expected non-empty output") + } +} diff --git a/internal/cmd/alpha/bucket/rb/rb.go b/internal/cmd/alpha/bucket/rb/rb.go new file mode 100644 index 000000000..6de7c7baf --- /dev/null +++ b/internal/cmd/alpha/bucket/rb/rb.go @@ -0,0 +1,106 @@ +package rb + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/stackitcloud/stackit-cli/internal/pkg/args" + "github.com/stackitcloud/stackit-cli/internal/pkg/examples" + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/print" + "github.com/stackitcloud/stackit-cli/internal/pkg/services/object-storage/bucket" + "github.com/stackitcloud/stackit-cli/internal/pkg/spinner" + "github.com/stackitcloud/stackit-cli/internal/pkg/types" +) + +const bucketURIArg = "BUCKET_URI" + +type inputModel struct { + *globalflags.GlobalFlagModel + bucket.ClientFlags + URI string +} + +func validateBucketURI(uri string) error { + ref, err := bucket.ParseBucketURI(uri) + if err != nil { + return err + } + if !ref.IsBucket() { + return fmt.Errorf("expected a bucket URI (s3://BUCKET or https://BUCKET.object.storage.REGION.onstackit.cloud), got %q", uri) + } + return nil +} + +func NewCmd(params *types.CmdParams) *cobra.Command { + cmd := &cobra.Command{ + Use: fmt.Sprintf("rb %s", bucketURIArg), + Short: "Deletes a bucket", + Long: "Deletes a bucket in STACKIT Object Storage. The bucket must be empty.", + Args: args.SingleArg(bucketURIArg, validateBucketURI), + Example: examples.Build( + examples.NewExample( + "Delete an empty bucket", + "$ stackit alpha bucket rb s3://my-bucket"), + ), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + model, err := parseInput(params.Printer, cmd, args) + if err != nil { + return err + } + + ref, err := bucket.ParseBucketURI(model.URI) + if err != nil { + return err + } + + apiClient, err := bucket.NewClient(bucket.ClientConfigFromReference(model.ClientFlags, model.Region, ref)) + if err != nil { + return err + } + + prompt := fmt.Sprintf("Are you sure you want to delete bucket %q? All objects in the bucket must be deleted first", model.URI) + err = params.Printer.PromptForConfirmation(prompt) + if err != nil { + return err + } + + err = spinner.Run(params.Printer, "Deleting bucket", func() error { + return apiClient.DeleteBucket(ctx, ref.Bucket) + }) + if err != nil { + return fmt.Errorf("delete bucket: %w", err) + } + + return outputResult(params.Printer, model.OutputFormat, model.URI) + }, + } + configureFlags(cmd) + return cmd +} + +func configureFlags(cmd *cobra.Command) { + bucket.ConfigureClientFlags(cmd.Flags()) +} + +func parseInput(p *print.Printer, cmd *cobra.Command, inputArgs []string) (*inputModel, error) { + model := inputModel{ + GlobalFlagModel: globalflags.Parse(p, cmd), + ClientFlags: bucket.ClientFlagsFromCommand(p, cmd), + URI: inputArgs[0], + } + p.DebugInputModel(model) + return &model, nil +} + +func outputResult(p *print.Printer, outputFormat, uri string) error { + return p.OutputResult(outputFormat, struct { + Bucket string `json:"bucket"` + }{Bucket: uri}, func() error { + p.Outputf("Deleted bucket %q\n", uri) + return nil + }) +} diff --git a/internal/cmd/alpha/bucket/rb/rb_test.go b/internal/cmd/alpha/bucket/rb/rb_test.go new file mode 100644 index 000000000..19355ac1c --- /dev/null +++ b/internal/cmd/alpha/bucket/rb/rb_test.go @@ -0,0 +1,92 @@ +package rb + +import ( + "testing" + + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/testparams" + "github.com/stackitcloud/stackit-cli/internal/pkg/testutils" +) + +const ( + testRegion = "eu01" + testBucket = "s3://my-bucket" +) + +func fixtureArgValues(mods ...func(argValues []string)) []string { + argValues := []string{testBucket} + for _, mod := range mods { + mod(argValues) + } + return argValues +} + +func fixtureFlagValues(mods ...func(flagValues map[string]string)) map[string]string { + flagValues := map[string]string{ + globalflags.RegionFlag: testRegion, + } + for _, mod := range mods { + mod(flagValues) + } + return flagValues +} + +func fixtureInputModel(mods ...func(model *inputModel)) *inputModel { + model := &inputModel{ + GlobalFlagModel: &globalflags.GlobalFlagModel{ + Verbosity: globalflags.VerbosityDefault, + Region: testRegion, + }, + URI: testBucket, + } + for _, mod := range mods { + mod(model) + } + return model +} + +func TestParseInput(t *testing.T) { + tests := []struct { + description string + argValues []string + flagValues map[string]string + isValid bool + expectedModel *inputModel + }{ + { + description: "base", + argValues: fixtureArgValues(), + flagValues: fixtureFlagValues(), + isValid: true, + expectedModel: fixtureInputModel(), + }, + { + description: "no values", + argValues: []string{}, + flagValues: map[string]string{}, + isValid: false, + }, + { + description: "invalid uri, object instead of bucket", + argValues: []string{"s3://my-bucket/key"}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + testutils.TestParseInput(t, NewCmd, parseInput, tt.expectedModel, tt.argValues, tt.flagValues, tt.isValid) + }) + } +} + +func TestOutputResult(t *testing.T) { + params := testparams.NewTestParams() + if err := outputResult(params.Printer, "", testBucket); err != nil { + t.Fatalf("outputResult() unexpected error: %v", err) + } + if output := params.Out.String(); output == "" { + t.Fatalf("outputResult() expected non-empty output") + } +} diff --git a/internal/cmd/alpha/bucket/rm/rm.go b/internal/cmd/alpha/bucket/rm/rm.go new file mode 100644 index 000000000..200506ca3 --- /dev/null +++ b/internal/cmd/alpha/bucket/rm/rm.go @@ -0,0 +1,131 @@ +package rm + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/stackitcloud/stackit-cli/internal/pkg/args" + "github.com/stackitcloud/stackit-cli/internal/pkg/examples" + "github.com/stackitcloud/stackit-cli/internal/pkg/flags" + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/print" + "github.com/stackitcloud/stackit-cli/internal/pkg/services/object-storage/bucket" + "github.com/stackitcloud/stackit-cli/internal/pkg/spinner" + "github.com/stackitcloud/stackit-cli/internal/pkg/types" +) + +const ( + objectURIArg = "BUCKET_URI" + recursiveFlag = "recursive" + forceFlag = "force" +) + +type inputModel struct { + *globalflags.GlobalFlagModel + bucket.ClientFlags + URI string + Recursive bool + Force bool +} + +func validateObjectURI(uri string) error { + ref, err := bucket.ParseBucketURI(uri) + if err != nil { + return err + } + if ref.IsBucket() { + return fmt.Errorf("expected an object or prefix URI (s3://BUCKET/KEY or a STACKIT Object Storage https URL): to delete an empty bucket, use 'stackit alpha bucket rb'") + } + return nil +} + +type deleteResult struct { + Objects int `json:"objects"` +} + +func NewCmd(params *types.CmdParams) *cobra.Command { + cmd := &cobra.Command{ + Use: fmt.Sprintf("rm %s", objectURIArg), + Short: "Deletes objects", + Long: `Deletes a single object, or all objects under a prefix with --recursive. + +To delete an empty bucket, use 'stackit alpha bucket rb'.`, + Args: args.SingleArg(objectURIArg, validateObjectURI), + Example: examples.Build( + examples.NewExample( + "Delete a single object", + "$ stackit alpha bucket rm s3://my-bucket/file.txt"), + examples.NewExample( + "Delete all objects under a prefix without prompting", + "$ stackit alpha bucket rm s3://my-bucket/path/to/ --recursive --force"), + ), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + model, err := parseInput(params.Printer, cmd, args) + if err != nil { + return err + } + + ref, err := bucket.ParseBucketURI(model.URI) + if err != nil { + return err + } + + apiClient, err := bucket.NewClient(bucket.ClientConfigFromReference(model.ClientFlags, model.Region, ref)) + if err != nil { + return err + } + + refs, err := bucket.ResolveObjectRefs(ctx, apiClient, ref, model.Recursive) + if err != nil { + return err + } + + if !model.Force { + prompt := fmt.Sprintf("Are you sure you want to delete %d object(s) from %q?", len(refs), ref.String()) + err = params.Printer.PromptForConfirmation(prompt) + if err != nil { + return err + } + } + + deleted, err := spinner.Run2(params.Printer, "Deleting objects", func() (int, error) { + return bucket.DeleteObjects(ctx, apiClient, refs) + }) + if err != nil { + return err + } + + return outputResult(params.Printer, model.OutputFormat, ref.String(), deleted) + }, + } + configureFlags(cmd) + return cmd +} + +func configureFlags(cmd *cobra.Command) { + bucket.ConfigureClientFlags(cmd.Flags()) + cmd.Flags().Bool(recursiveFlag, false, "Delete all objects under the prefix") + cmd.Flags().Bool(forceFlag, false, "Do not prompt for confirmation") +} + +func parseInput(p *print.Printer, cmd *cobra.Command, inputArgs []string) (*inputModel, error) { + model := inputModel{ + GlobalFlagModel: globalflags.Parse(p, cmd), + ClientFlags: bucket.ClientFlagsFromCommand(p, cmd), + URI: inputArgs[0], + Recursive: flags.FlagToBoolValue(p, cmd, recursiveFlag), + Force: flags.FlagToBoolValue(p, cmd, forceFlag), + } + p.DebugInputModel(model) + return &model, nil +} + +func outputResult(p *print.Printer, outputFormat, uri string, objects int) error { + return p.OutputResult(outputFormat, deleteResult{Objects: objects}, func() error { + p.Outputf("Deleted %d object(s) from %q\n", objects, uri) + return nil + }) +} diff --git a/internal/cmd/alpha/bucket/rm/rm_test.go b/internal/cmd/alpha/bucket/rm/rm_test.go new file mode 100644 index 000000000..0096796f7 --- /dev/null +++ b/internal/cmd/alpha/bucket/rm/rm_test.go @@ -0,0 +1,97 @@ +package rm + +import ( + "testing" + + "github.com/stackitcloud/stackit-cli/internal/pkg/globalflags" + "github.com/stackitcloud/stackit-cli/internal/pkg/testparams" + "github.com/stackitcloud/stackit-cli/internal/pkg/testutils" +) + +const ( + testRegion = "eu01" + testURI = "s3://my-bucket/file.txt" +) + +func fixtureFlagValues(mods ...func(flagValues map[string]string)) map[string]string { + flagValues := map[string]string{ + globalflags.RegionFlag: testRegion, + } + for _, mod := range mods { + mod(flagValues) + } + return flagValues +} + +func fixtureInputModel(mods ...func(model *inputModel)) *inputModel { + model := &inputModel{ + GlobalFlagModel: &globalflags.GlobalFlagModel{ + Verbosity: globalflags.VerbosityDefault, + Region: testRegion, + }, + URI: testURI, + } + for _, mod := range mods { + mod(model) + } + return model +} + +func TestParseInput(t *testing.T) { + tests := []struct { + description string + argValues []string + flagValues map[string]string + isValid bool + expectedModel *inputModel + }{ + { + description: "base", + argValues: []string{testURI}, + flagValues: fixtureFlagValues(), + isValid: true, + expectedModel: fixtureInputModel(), + }, + { + description: "with recursive and force", + argValues: []string{testURI}, + flagValues: fixtureFlagValues(func(flagValues map[string]string) { + flagValues[recursiveFlag] = "true" + flagValues[forceFlag] = "true" + }), + isValid: true, + expectedModel: fixtureInputModel(func(model *inputModel) { + model.Recursive = true + model.Force = true + }), + }, + { + description: "no values", + argValues: []string{}, + flagValues: map[string]string{}, + isValid: false, + }, + { + description: "invalid uri, bucket only", + argValues: []string{"s3://my-bucket"}, + flagValues: fixtureFlagValues(), + isValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + testutils.TestParseInput(t, NewCmd, parseInput, tt.expectedModel, tt.argValues, tt.flagValues, tt.isValid) + }) + } +} + +func TestOutputResult(t *testing.T) { + params := testparams.NewTestParams() + if err := outputResult(params.Printer, "", testURI, 5); err != nil { + t.Fatalf("outputResult() unexpected error: %v", err) + } + if output := params.Out.String(); output == "" { + t.Fatalf("outputResult() expected non-empty output") + } +} diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 56c53899b..83b90a4ff 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -23,22 +23,25 @@ const ( IdentityProviderCustomClientIdKey = "identity_provider_custom_client_id" AllowedUrlDomainKey = "allowed_url_domain" - AuthorizationCustomEndpointKey = "authorization_custom_endpoint" - AutomationCustomEndpointKey = "automation_custom_endpoint" - AlbCustomEndpoint = "alb_custom _endpoint" - AlbWafCustomEndpointKey = "alb_waf_custom_endpoint" - DNSCustomEndpointKey = "dns_custom_endpoint" - EdgeCustomEndpointKey = "edge_custom_endpoint" - LoadBalancerCustomEndpointKey = "load_balancer_custom_endpoint" - LogMeCustomEndpointKey = "logme_custom_endpoint" - MariaDBCustomEndpointKey = "mariadb_custom_endpoint" - MongoDBFlexCustomEndpointKey = "mongodbflex_custom_endpoint" - ObjectStorageCustomEndpointKey = "object_storage_custom_endpoint" - ObservabilityCustomEndpointKey = "observability_custom_endpoint" - OpenSearchCustomEndpointKey = "opensearch_custom_endpoint" - PostgresFlexCustomEndpointKey = "postgresflex_custom_endpoint" - RabbitMQCustomEndpointKey = "rabbitmq_custom_endpoint" - ScfCustomEndpointKey = "scf_custom_endpoint" + AuthorizationCustomEndpointKey = "authorization_custom_endpoint" + AutomationCustomEndpointKey = "automation_custom_endpoint" + AlbCustomEndpoint = "alb_custom _endpoint" + AlbWafCustomEndpointKey = "alb_waf_custom_endpoint" + DNSCustomEndpointKey = "dns_custom_endpoint" + EdgeCustomEndpointKey = "edge_custom_endpoint" + LoadBalancerCustomEndpointKey = "load_balancer_custom_endpoint" + LogMeCustomEndpointKey = "logme_custom_endpoint" + MariaDBCustomEndpointKey = "mariadb_custom_endpoint" + MongoDBFlexCustomEndpointKey = "mongodbflex_custom_endpoint" + ObjectStorageCustomEndpointKey = "object_storage_custom_endpoint" + ObjectStorageAccessKeyIdKey = "object_storage_access_key_id" + ObjectStorageSecretAccessKeyKey = "object_storage_secret_access_key" + ObjectStorageDataPlaneCustomEndpointKey = "object_storage_data_plane_custom_endpoint" + ObservabilityCustomEndpointKey = "observability_custom_endpoint" + OpenSearchCustomEndpointKey = "opensearch_custom_endpoint" + PostgresFlexCustomEndpointKey = "postgresflex_custom_endpoint" + RabbitMQCustomEndpointKey = "rabbitmq_custom_endpoint" + ScfCustomEndpointKey = "scf_custom_endpoint" // Deprecated: Will be removed after 2027-08-31. RedisCustomEndpointKey = "redis_custom_endpoint" ResourceManagerEndpointKey = "resource_manager_custom_endpoint" @@ -115,6 +118,9 @@ var ConfigKeys = []string{ MariaDBCustomEndpointKey, MongoDBFlexCustomEndpointKey, ObjectStorageCustomEndpointKey, + ObjectStorageAccessKeyIdKey, + ObjectStorageSecretAccessKeyKey, + ObjectStorageDataPlaneCustomEndpointKey, ObservabilityCustomEndpointKey, OpenSearchCustomEndpointKey, PostgresFlexCustomEndpointKey, @@ -207,6 +213,9 @@ func setConfigDefaults() { viper.SetDefault(AutomationCustomEndpointKey, "") viper.SetDefault(MongoDBFlexCustomEndpointKey, "") viper.SetDefault(ObjectStorageCustomEndpointKey, "") + viper.SetDefault(ObjectStorageAccessKeyIdKey, "") + viper.SetDefault(ObjectStorageSecretAccessKeyKey, "") + viper.SetDefault(ObjectStorageDataPlaneCustomEndpointKey, "") viper.SetDefault(OpenSearchCustomEndpointKey, "") viper.SetDefault(PostgresFlexCustomEndpointKey, "") viper.SetDefault(ResourceManagerEndpointKey, "") diff --git a/internal/pkg/services/object-storage/bucket/client.go b/internal/pkg/services/object-storage/bucket/client.go new file mode 100644 index 000000000..495b25be8 --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/client.go @@ -0,0 +1,397 @@ +package bucket + +import ( + "bytes" + "context" + "crypto/md5" //nolint:gosec // Content-MD5 is a storage protocol requirement for request integrity, not used for security + "encoding/base64" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/stackitcloud/stackit-cli/internal/pkg/config" +) + +const ( + clientTimeout = 30 * time.Minute + maxAttempts = 3 + retryBaseDelay = time.Second + + contentTypeDefault = "application/octet-stream" + contentTypeXML = "application/xml" +) + +const fallbackSigningRegion = "eu01" + +type Client struct { + endpoint string + region string + signer *Signer + httpClient *http.Client + now func() time.Time + retryDelay time.Duration +} + +func NewClient(cfg ClientConfig) (*Client, error) { + if cfg.AccessKeyId == "" || cfg.SecretAccessKey == "" { + return nil, fmt.Errorf("object Storage credentials are required: use the --%s and --%s flags, or set %s/%s in the config file or environment variables", + AccessKeyIdFlagName, SecretAccessKeyFlagName, config.ObjectStorageAccessKeyIdKey, config.ObjectStorageSecretAccessKeyKey) + } + endpoint, err := cfg.ResolvedEndpoint() + if err != nil { + return nil, err + } + region := cfg.Region + if region == "" { + region = fallbackSigningRegion + } + endpoint = strings.TrimSuffix(endpoint, "/") + return &Client{ + endpoint: endpoint, + region: region, + signer: NewSigner(cfg.AccessKeyId, cfg.SecretAccessKey, region), + httpClient: &http.Client{Timeout: clientTimeout}, + now: time.Now, + retryDelay: retryBaseDelay, + }, nil +} + +type ListObjectsV2Output struct { + IsTruncated bool + NextContinuationToken string + Objects []ObjectInfo + CommonPrefixes []string +} + +func objectPath(bucket, key string) string { + if key == "" { + return "/" + bucket + } + return "/" + bucket + "/" + escapeKey(key) +} + +func isRetryableStatus(status int) bool { + return status == http.StatusTooManyRequests || status >= 500 +} + +func (c *Client) do(ctx context.Context, method, path string, query url.Values, headers map[string]string, body io.Reader, rewind func() error) (*http.Response, error) { + target := c.endpoint + path + if len(query) > 0 { + target += "?" + canonicalQueryString(query) + } + + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + if attempt > 1 { + if rewind != nil { + if err := rewind(); err != nil { + return nil, fmt.Errorf("%s %s: rewind request body: %w", method, path, err) + } + } + delay := c.retryDelay << uint(attempt-2) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(delay): + } + } + + var bodyReader io.Reader = http.NoBody + if body != nil { + bodyReader = body + } + // Buffered bodies need a rewind for retries (their reader is consumed + // by the first attempt) + if rewind == nil { + if reader, ok := body.(*bytes.Reader); ok { + rewind = func() error { + _, err := reader.Seek(0, io.SeekStart) + return err + } + } + } + req, err := http.NewRequestWithContext(ctx, method, target, bodyReader) + if err != nil { + return nil, fmt.Errorf("%s %s: build request: %w", method, path, err) + } + // The wire path must be fully encoded so that the path the server + // receives matches the canonical path used for signing + req.URL.RawPath = canonicalPath(req) + // The storage backends do not support chunked transfer encoding, so + // the content length must be known for file bodies + if size, ok := fileContentLength(body); ok { + req.ContentLength = size + } + for name, value := range headers { + req.Header.Set(name, value) + } + // The storage server expects the declared payload hash on every request: the + // empty payload hash for bodyless requests, UNSIGNED-PAYLOAD otherwise + if req.Header.Get("X-Amz-Content-Sha256") == "" { + if body == nil { + req.Header.Set("X-Amz-Content-Sha256", emptyPayloadHash) + } else { + req.Header.Set("X-Amz-Content-Sha256", unsignedPayload) + } + } + if err := c.signer.Sign(req, c.now()); err != nil { + return nil, fmt.Errorf("%s %s: sign request: %w", method, path, err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + lastErr = err + if attempt == maxAttempts { + break + } + continue + } + if isRetryableStatus(resp.StatusCode) { + bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + _ = resp.Body.Close() + lastErr = parseAPIError(method, resp.StatusCode, bodyBytes) + if attempt == maxAttempts { + break + } + continue + } + return resp, nil + } + return nil, fmt.Errorf("%s %s: %w", method, path, lastErr) +} + +func (c *Client) doAndCheck(ctx context.Context, method, path string, query url.Values, headers map[string]string, body io.Reader, rewind func() error, out io.Writer) error { + resp, err := c.do(ctx, method, path, query, headers, body, rewind) + if err != nil { + return err + } + defer func() { + if cerr := resp.Body.Close(); cerr != nil { + fmt.Fprintf(os.Stderr, "close response body: %v\n", cerr) + } + }() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + return parseAPIError(method, resp.StatusCode, bodyBytes) + } + if out != nil { + if _, err := io.Copy(out, resp.Body); err != nil { + return fmt.Errorf("%s %s: read response body: %w", method, path, err) + } + } + return nil +} + +func (c *Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) { + var buf bytes.Buffer + if err := c.doAndCheck(ctx, "GET", "/", nil, nil, nil, nil, &buf); err != nil { + return nil, fmt.Errorf("list buckets: %w", err) + } + var result listAllMyBucketsResult + if err := xml.Unmarshal(buf.Bytes(), &result); err != nil { + return nil, fmt.Errorf("list buckets: parse response: %w", err) + } + buckets := make([]BucketInfo, 0, len(result.Buckets)) + for _, b := range result.Buckets { + buckets = append(buckets, BucketInfo(b)) + } + return buckets, nil +} + +func (c *Client) ListObjectsV2(ctx context.Context, bucket, prefix, delimiter, continuationToken string) (*ListObjectsV2Output, error) { + query := url.Values{} + query.Set("list-type", "2") + if prefix != "" { + query.Set("prefix", prefix) + } + if delimiter != "" { + query.Set("delimiter", delimiter) + } + if continuationToken != "" { + query.Set("continuation-token", continuationToken) + } + var buf bytes.Buffer + if err := c.doAndCheck(ctx, "GET", objectPath(bucket, ""), query, nil, nil, nil, &buf); err != nil { + return nil, fmt.Errorf("list objects in bucket %q: %w", bucket, err) + } + var result listBucketResult + if err := xml.Unmarshal(buf.Bytes(), &result); err != nil { + return nil, fmt.Errorf("list objects in bucket %q: parse response: %w", bucket, err) + } + output := &ListObjectsV2Output{ + IsTruncated: result.IsTruncated, + NextContinuationToken: result.NextContinuationToken, + Objects: make([]ObjectInfo, 0, len(result.Contents)), + CommonPrefixes: make([]string, 0, len(result.CommonPrefixes)), + } + for _, obj := range result.Contents { + output.Objects = append(output.Objects, ObjectInfo(obj)) + } + for _, p := range result.CommonPrefixes { + output.CommonPrefixes = append(output.CommonPrefixes, p.Prefix) + } + return output, nil +} + +func (c *Client) HeadObject(ctx context.Context, bucket, key string) error { + err := c.doAndCheck(ctx, "HEAD", objectPath(bucket, key), nil, nil, nil, nil, nil) + if IsNotFound(err) { + return ErrNotFound + } + return err +} + +func (c *Client) GetObject(ctx context.Context, bucket, key string) (io.ReadCloser, error) { + resp, err := c.do(ctx, "GET", objectPath(bucket, key), nil, nil, nil, nil) + if err != nil { + return nil, fmt.Errorf("download object %s/%s: %w", bucket, key, err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + _ = resp.Body.Close() + return nil, parseAPIError("GetObject", resp.StatusCode, bodyBytes) + } + return resp.Body, nil +} + +func (c *Client) PutObject(ctx context.Context, bucket, key string, body io.Reader) error { + var rewind func() error + if file, ok := body.(*os.File); ok { + rewind = func() error { + _, err := file.Seek(0, io.SeekStart) + return err + } + } + headers := map[string]string{ + "Content-Type": contentTypeDefault, + } + // Wrap the body so the HTTP client does not close it (it closes request + // bodies that implement io.Closer, which would break retry rewinds) + err := c.doAndCheck(ctx, "PUT", objectPath(bucket, key), nil, headers, readOnlyReader{body}, rewind, nil) + if err != nil { + return fmt.Errorf("upload to %s/%s: %w", bucket, key, err) + } + return nil +} + +type readOnlyReader struct { + r io.Reader +} + +func (r readOnlyReader) Read(p []byte) (int, error) { return r.r.Read(p) } + +func bodyHeaders(contentType string, body []byte) map[string]string { + sum := md5.Sum(body) //nolint:gosec // Content-MD5 is a storage protocol requirement, not used for security + return map[string]string{ + "Content-MD5": base64.StdEncoding.EncodeToString(sum[:]), + "Content-Type": contentType, + } +} + +func fileContentLength(body io.Reader) (int64, bool) { + if rr, ok := body.(readOnlyReader); ok { + body = rr.r + } + if file, ok := body.(*os.File); ok { + if stat, err := file.Stat(); err == nil { + return stat.Size(), true + } + } + return 0, false +} + +func (c *Client) CopyObject(ctx context.Context, src, dst Reference) error { + headers := map[string]string{ + "x-amz-copy-source": src.Bucket + "/" + escapeKey(src.Key), + } + err := c.doAndCheck(ctx, "PUT", objectPath(dst.Bucket, dst.Key), nil, headers, nil, nil, nil) + if err != nil { + return fmt.Errorf("copy %s/%s to %s/%s: %w", src.Bucket, src.Key, dst.Bucket, dst.Key, err) + } + return nil +} + +func (c *Client) DeleteObjects(ctx context.Context, bucket string, keys []string) (int, error) { + if len(keys) == 0 { + return 0, nil + } + request := deleteRequest{Quiet: true} + for _, key := range keys { + request.Objects = append(request.Objects, deleteEntry{Key: key}) + } + body, err := xml.Marshal(&request) + if err != nil { + return 0, fmt.Errorf("delete objects from bucket %q: encode request: %w", bucket, err) + } + var buf bytes.Buffer + query := url.Values{"delete": {""}} + headers := bodyHeaders(contentTypeXML, body) + if err := c.doAndCheck(ctx, "POST", objectPath(bucket, ""), query, headers, bytes.NewReader(body), nil, &buf); err != nil { + return 0, fmt.Errorf("delete objects from bucket %q: %w", bucket, err) + } + if len(buf.Bytes()) == 0 { + // Quiet mode: no per-object result + return len(keys), nil + } + var result deleteResult + if err := xml.Unmarshal(buf.Bytes(), &result); err != nil { + return 0, fmt.Errorf("delete objects from bucket %q: parse response: %w", bucket, err) + } + if len(result.Errors) > 0 { + return len(result.Deleted), fmt.Errorf("delete objects from bucket %q: %d object(s) failed: %s: %s", bucket, len(result.Errors), result.Errors[0].Code, result.Errors[0].Message) + } + if len(result.Deleted) == 0 { + // Quiet mode: the server reports no per-object results + return len(keys), nil + } + return len(result.Deleted), nil +} + +func (c *Client) CreateBucket(ctx context.Context, bucket, locationConstraint string) error { + var body []byte + var headers map[string]string + if locationConstraint != "" { + encoded, varErr := xml.Marshal(createBucketConfiguration{LocationConstraint: locationConstraint}) + if varErr != nil { + return fmt.Errorf("create bucket %q: encode request: %w", bucket, varErr) + } + body = encoded + headers = bodyHeaders(contentTypeXML, body) + } + var err error + if body != nil { + err = c.doAndCheck(ctx, "PUT", objectPath(bucket, ""), nil, headers, bytes.NewReader(body), nil, nil) + } else { + err = c.doAndCheck(ctx, "PUT", objectPath(bucket, ""), nil, headers, nil, nil, nil) + } + if err != nil { + return fmt.Errorf("create bucket %q: %w", bucket, err) + } + return nil +} + +func (c *Client) DeleteBucket(ctx context.Context, bucket string) error { + if err := c.doAndCheck(ctx, "DELETE", objectPath(bucket, ""), nil, nil, nil, nil, nil); err != nil { + return fmt.Errorf("delete bucket %q: %w", bucket, err) + } + return nil +} + +func (c *Client) PresignGetObject(bucket, key string, expires time.Duration) (string, error) { + req, err := http.NewRequest("GET", c.endpoint+objectPath(bucket, key), http.NoBody) + if err != nil { + return "", fmt.Errorf("presign URL for %s/%s: build request: %w", bucket, key, err) + } + req.URL.RawPath = canonicalPath(req) + // Object Storage presigned downloads declare an unsigned payload, matching the reference implementation + signedURL, err := c.signer.Presign(req, expires, c.now(), unsignedPayload) + if err != nil { + return "", fmt.Errorf("presign URL for %s/%s: %w", bucket, key, err) + } + return signedURL, nil +} diff --git a/internal/pkg/services/object-storage/bucket/client_test.go b/internal/pkg/services/object-storage/bucket/client_test.go new file mode 100644 index 000000000..bedca1b13 --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/client_test.go @@ -0,0 +1,257 @@ +package bucket + +import ( + "context" + "crypto/md5" //nolint:gosec // test helper for Content-MD5, not used for security + "encoding/base64" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func newTestClient(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + client, err := NewClient(ClientConfig{ + Region: "eu01", + Endpoint: server.URL, + AccessKeyId: "AKIDEXAMPLE", + SecretAccessKey: "secret", + }) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + return client +} + +func TestNewClientRequiresCredentials(t *testing.T) { + _, err := NewClient(ClientConfig{Region: "eu01"}) + if err == nil { + t.Fatal("expected an error when credentials are missing") + } +} + +func TestListBuckets(t *testing.T) { + var gotAuth, gotPayloadHash string + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + gotPayloadHash = r.Header.Get("X-Amz-Content-Sha256") + _, _ = w.Write([]byte(`` + + `bucket-a2024-05-01T12:00:00Z` + + `bucket-b2024-06-01T12:00:00Z` + + ``)) + }) + + buckets, err := client.ListBuckets(context.Background()) + if err != nil { + t.Fatalf("ListBuckets: %v", err) + } + if len(buckets) != 2 { + t.Fatalf("ListBuckets returned %d buckets, want 2", len(buckets)) + } + if buckets[0].Name != "bucket-a" || buckets[1].Name != "bucket-b" { + t.Fatalf("unexpected buckets: %+v", buckets) + } + if !strings.HasPrefix(gotAuth, "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/") { + t.Fatalf("unexpected Authorization header: %q", gotAuth) + } + if gotPayloadHash != emptyPayloadHash { + t.Fatalf("X-Amz-Content-Sha256 = %q, want %q", gotPayloadHash, emptyPayloadHash) + } +} + +func TestPutObjectSendsBodyAndPayloadHash(t *testing.T) { + var gotPayloadHash string + var gotBody []byte + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotPayloadHash = r.Header.Get("X-Amz-Content-Sha256") + gotBody, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + }) + + err := client.PutObject(context.Background(), "my-bucket", "dir/file.txt", strings.NewReader("hello")) + if err != nil { + t.Fatalf("PutObject: %v", err) + } + if gotPayloadHash != unsignedPayload { + t.Fatalf("X-Amz-Content-Sha256 = %q, want %q", gotPayloadHash, unsignedPayload) + } + if string(gotBody) != "hello" { + t.Fatalf("server received body %q, want %q", gotBody, "hello") + } +} + +func TestDeleteObjectsSendsContentMD5AndCountsQuietResponse(t *testing.T) { + var gotMD5 string + var gotBody []byte + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotMD5 = r.Header.Get("Content-MD5") + gotBody, _ = io.ReadAll(r.Body) + // Quiet mode response without per-object results + _, _ = w.Write([]byte(``)) + }) + + deleted, err := client.DeleteObjects(context.Background(), "my-bucket", []string{"a.txt", "b.txt"}) + if err != nil { + t.Fatalf("DeleteObjects: %v", err) + } + if deleted != 2 { + t.Fatalf("deleted = %d, want 2", deleted) + } + wantMD5 := base64.StdEncoding.EncodeToString(md5sum(gotBody)) + if gotMD5 != wantMD5 { + t.Fatalf("Content-MD5 = %q, want %q for body %q", gotMD5, wantMD5, gotBody) + } +} + +func TestDeleteObjectsReportsPerObjectResults(t *testing.T) { + client := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`` + + `a.txt` + + `b.txtAccessDenieddenied` + + ``)) + }) + + deleted, err := client.DeleteObjects(context.Background(), "my-bucket", []string{"a.txt", "b.txt"}) + if err == nil { + t.Fatal("expected an error when a delete fails") + } + if deleted != 1 { + t.Fatalf("deleted = %d, want 1", deleted) + } + if !strings.Contains(err.Error(), "AccessDenied") { + t.Fatalf("error = %v, want it to mention AccessDenied", err) + } +} + +func TestCreateBucketSendsLocationConstraint(t *testing.T) { + var gotMethod, gotPath string + var gotMD5, gotContentType string + var gotBody []byte + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotPath = r.URL.Path + gotMD5 = r.Header.Get("Content-MD5") + gotContentType = r.Header.Get("Content-Type") + gotBody, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + }) + + err := client.CreateBucket(context.Background(), "my-bucket", "eu02") + if err != nil { + t.Fatalf("CreateBucket: %v", err) + } + if gotMethod != http.MethodPut || gotPath != "/my-bucket" { + t.Fatalf("request = %s %s, want PUT /my-bucket", gotMethod, gotPath) + } + wantBody := `eu02` + if string(gotBody) != wantBody { + t.Fatalf("body = %q, want %q", gotBody, wantBody) + } + if gotContentType != "application/xml" { + t.Fatalf("Content-Type = %q, want %q", gotContentType, "application/xml") + } + wantMD5 := base64.StdEncoding.EncodeToString(md5sum(gotBody)) + if gotMD5 != wantMD5 { + t.Fatalf("Content-MD5 = %q, want %q for body %q", gotMD5, wantMD5, gotBody) + } +} + +func TestCreateBucketWithoutConstraintSendsEmptyBody(t *testing.T) { + var gotMethod, gotPath string + var gotMD5, gotContentType string + var gotBody []byte + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotPath = r.URL.Path + gotMD5 = r.Header.Get("Content-MD5") + gotContentType = r.Header.Get("Content-Type") + gotBody, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + }) + + err := client.CreateBucket(context.Background(), "my-bucket", "") + if err != nil { + t.Fatalf("CreateBucket: %v", err) + } + if gotMethod != http.MethodPut || gotPath != "/my-bucket" { + t.Fatalf("request = %s %s, want PUT /my-bucket", gotMethod, gotPath) + } + if len(gotBody) != 0 { + t.Fatalf("body = %q, want empty", gotBody) + } + if gotMD5 != "" { + t.Fatalf("Content-MD5 = %q, want empty", gotMD5) + } + if gotContentType != "" { + t.Fatalf("Content-Type = %q, want empty", gotContentType) + } +} + +func TestRetryOnServerErrors(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) <= 2 { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte(``)) + })) + t.Cleanup(server.Close) + client, err := NewClient(ClientConfig{ + Region: "eu01", + Endpoint: server.URL, + AccessKeyId: "AKIDEXAMPLE", + SecretAccessKey: "secret", + }) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + client.retryDelay = time.Millisecond + + buckets, err := client.ListBuckets(context.Background()) + if err != nil { + t.Fatalf("ListBuckets: %v", err) + } + if len(buckets) != 0 { + t.Fatalf("ListBuckets returned %d buckets, want 0", len(buckets)) + } + if got := attempts.Load(); got != 3 { + t.Fatalf("attempts = %d, want 3", got) + } +} + +func TestNoRetryOnClientErrors(t *testing.T) { + var attempts atomic.Int32 + client := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`AccessDeniedAccess Denied`)) + }) + + _, err := client.ListBuckets(context.Background()) + if err == nil { + t.Fatal("expected an error on 403") + } + if got := attempts.Load(); got != 1 { + t.Fatalf("attempts = %d, want 1", got) + } + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error type = %T, want *APIError", err) + } + if apiErr.Code != "AccessDenied" || apiErr.StatusCode != http.StatusForbidden { + t.Fatalf("APIError = %+v, want 403 AccessDenied", apiErr) + } +} + +func md5sum(data []byte) []byte { + sum := md5.Sum(data) //nolint:gosec // test helper for Content-MD5, not used for security + return sum[:] +} diff --git a/internal/pkg/services/object-storage/bucket/credentials.go b/internal/pkg/services/object-storage/bucket/credentials.go new file mode 100644 index 000000000..e1d6f8d82 --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/credentials.go @@ -0,0 +1,87 @@ +package bucket + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/stackitcloud/stackit-cli/internal/pkg/config" + "github.com/stackitcloud/stackit-cli/internal/pkg/flags" + "github.com/stackitcloud/stackit-cli/internal/pkg/print" +) + +const ( + AccessKeyIdFlagName = "access-key-id" + SecretAccessKeyFlagName = "secret-access-key" + EndpointFlagName = "endpoint" +) + +type ClientConfig struct { + Region string + Endpoint string + AccessKeyId string + SecretAccessKey string +} + +type ClientFlags struct { + Endpoint string + AccessKeyId string + SecretAccessKey string +} + +func ConfigureClientFlags(flagSet *pflag.FlagSet) { + flagSet.String(EndpointFlagName, "", "Object Storage API endpoint. Defaults to https://object.storage..onstackit.cloud. Can also be set with the "+config.ObjectStorageDataPlaneCustomEndpointKey+" config key or the STACKIT_OBJECT_STORAGE_DATA_PLANE_CUSTOM_ENDPOINT environment variable.") + flagSet.String(AccessKeyIdFlagName, "", "Object Storage access key ID. Can also be set with the "+config.ObjectStorageAccessKeyIdKey+" config key or the STACKIT_OBJECT_STORAGE_ACCESS_KEY_ID environment variable.") + flagSet.String(SecretAccessKeyFlagName, "", "Object Storage secret access key. Can also be set with the "+config.ObjectStorageSecretAccessKeyKey+" config key or the STACKIT_OBJECT_STORAGE_SECRET_ACCESS_KEY environment variable.") +} + +func resolveFlagValue(flagValue, configKey string) string { + if flagValue != "" { + return flagValue + } + return viper.GetString(configKey) +} + +func ClientConfigFromValues(clientFlags ClientFlags, region string) ClientConfig { + return ClientConfig{ + Region: region, + Endpoint: resolveFlagValue(clientFlags.Endpoint, config.ObjectStorageDataPlaneCustomEndpointKey), + AccessKeyId: resolveFlagValue(clientFlags.AccessKeyId, config.ObjectStorageAccessKeyIdKey), + SecretAccessKey: resolveFlagValue(clientFlags.SecretAccessKey, config.ObjectStorageSecretAccessKeyKey), + } +} + +func ClientConfigFromReference(clientFlags ClientFlags, region string, ref Reference) ClientConfig { + cfg := ClientConfigFromValues(clientFlags, region) + if cfg.Endpoint == "" { + cfg.Endpoint = ref.Endpoint + } + if ref.Region != "" { + cfg.Region = ref.Region + } + return cfg +} + +func ClientFlagsFromCommand(p *print.Printer, cmd *cobra.Command) ClientFlags { + return ClientFlags{ + Endpoint: flags.FlagToStringValue(p, cmd, EndpointFlagName), + AccessKeyId: flags.FlagToStringValue(p, cmd, AccessKeyIdFlagName), + SecretAccessKey: flags.FlagToStringValue(p, cmd, SecretAccessKeyFlagName), + } +} + +func (c ClientConfig) ResolvedEndpoint() (string, error) { + if c.Endpoint != "" { + return c.Endpoint, nil + } + if c.Region == "" { + return "", fmt.Errorf("no --endpoint given and no region available: use --region or --endpoint") + } + return defaultEndpoint(c.Region), nil +} + +func defaultEndpoint(region string) string { + return fmt.Sprintf("https://object.storage.%s.onstackit.cloud", region) +} diff --git a/internal/pkg/services/object-storage/bucket/credentials_test.go b/internal/pkg/services/object-storage/bucket/credentials_test.go new file mode 100644 index 000000000..d30cdd7b7 --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/credentials_test.go @@ -0,0 +1,128 @@ +package bucket + +import ( + "testing" +) + +func TestDefaultEndpoint(t *testing.T) { + region := "eu01" + expected := "https://object.storage.eu01.onstackit.cloud" + if result := defaultEndpoint(region); result != expected { + t.Fatalf("defaultEndpoint(%q) = %q, want %q", region, result, expected) + } +} + +func TestClientConfigResolvedEndpoint(t *testing.T) { + tests := []struct { + description string + cfg ClientConfig + expected string + isValid bool + }{ + { + description: "explicit endpoint wins", + cfg: ClientConfig{ + Region: "eu01", + Endpoint: "https://custom.endpoint.example", + }, + expected: "https://custom.endpoint.example", + isValid: true, + }, + { + description: "derived from region", + cfg: ClientConfig{ + Region: "eu01", + }, + expected: "https://object.storage.eu01.onstackit.cloud", + isValid: true, + }, + { + description: "no endpoint and no region", + cfg: ClientConfig{}, + isValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + result, err := tt.cfg.ResolvedEndpoint() + if (err != nil) != !tt.isValid { + t.Fatalf("ResolvedEndpoint() error = %v, wantErr %v", err, !tt.isValid) + } + if tt.isValid && result != tt.expected { + t.Fatalf("ResolvedEndpoint() = %q, want %q", result, tt.expected) + } + }) + } +} + +func TestResolveFlagValue(t *testing.T) { + // A set flag value always wins + if result := resolveFlagValue("flag-value", "object_storage_access_key_id"); result != "flag-value" { + t.Fatalf("resolveFlagValue() = %q, want %q", result, "flag-value") + } + // An empty flag value falls back to the config (empty in tests) + if result := resolveFlagValue("", "object_storage_access_key_id"); result != "" { + t.Fatalf("resolveFlagValue() = %q, want empty", result) + } +} + +func TestClientConfigFromReference(t *testing.T) { + tests := []struct { + description string + flags ClientFlags + region string + ref Reference + expected ClientConfig + }{ + { + description: "plain reference, no endpoint from ref", + flags: ClientFlags{}, + region: "eu01", + ref: Reference{Bucket: "my-bucket"}, + expected: ClientConfig{Region: "eu01"}, + }, + { + description: "https reference provides endpoint and region", + flags: ClientFlags{}, + region: "eu01", + ref: Reference{ + Bucket: "my-bucket", + Endpoint: "https://object.storage.eu02.onstackit.cloud", + Region: "eu02", + }, + expected: ClientConfig{ + Region: "eu02", + Endpoint: "https://object.storage.eu02.onstackit.cloud", + }, + }, + { + description: "explicit endpoint flag wins over reference", + flags: ClientFlags{ + Endpoint: "https://custom.endpoint.example", + }, + region: "eu01", + ref: Reference{ + Bucket: "my-bucket", + Endpoint: "https://object.storage.eu02.onstackit.cloud", + Region: "eu02", + }, + expected: ClientConfig{ + Region: "eu02", + Endpoint: "https://custom.endpoint.example", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + result := ClientConfigFromReference(tt.flags, tt.region, tt.ref) + if result.Region != tt.expected.Region { + t.Fatalf("Region = %q, want %q", result.Region, tt.expected.Region) + } + if result.Endpoint != tt.expected.Endpoint { + t.Fatalf("Endpoint = %q, want %q", result.Endpoint, tt.expected.Endpoint) + } + }) + } +} diff --git a/internal/pkg/services/object-storage/bucket/errors.go b/internal/pkg/services/object-storage/bucket/errors.go new file mode 100644 index 000000000..cb08f1baa --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/errors.go @@ -0,0 +1,52 @@ +package bucket + +import ( + "encoding/xml" + "errors" + "fmt" +) + +var ErrNotFound = errors.New("not found") + +type APIError struct { + StatusCode int + Code string + Message string + RequestID string +} + +func (e *APIError) Error() string { + if e.Code != "" { + return fmt.Sprintf("Object Storage API error %d: %s: %s", e.StatusCode, e.Code, e.Message) + } + return fmt.Sprintf("Object Storage API error %d: %s", e.StatusCode, e.Message) +} + +func IsNotFound(err error) bool { + var apiErr *APIError + if errors.As(err, &apiErr) { + return apiErr.Code == "NoSuchKey" || apiErr.Code == "NoSuchBucket" || apiErr.StatusCode == 404 + } + return errors.Is(err, ErrNotFound) +} + +func parseAPIError(op string, statusCode int, body []byte) error { + var apiErr struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + RequestID string `xml:"RequestId"` + } + if err := xml.Unmarshal(body, &apiErr); err == nil && apiErr.Code != "" { + return &APIError{ + StatusCode: statusCode, + Code: apiErr.Code, + Message: apiErr.Message, + RequestID: apiErr.RequestID, + } + } + return &APIError{ + StatusCode: statusCode, + Message: fmt.Sprintf("%s: unexpected response", op), + } +} diff --git a/internal/pkg/services/object-storage/bucket/errors_test.go b/internal/pkg/services/object-storage/bucket/errors_test.go new file mode 100644 index 000000000..b02669a36 --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/errors_test.go @@ -0,0 +1,96 @@ +package bucket + +import ( + "errors" + "fmt" + "testing" +) + +func TestParseAPIError(t *testing.T) { + t.Run("valid error body", func(t *testing.T) { + body := []byte(`NoSuchKeyThe specified key does not exist.ABC123`) + err := parseAPIError("GetObject", 404, body) + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error type = %T, want *APIError", err) + } + if apiErr.StatusCode != 404 || apiErr.Code != "NoSuchKey" || apiErr.RequestID != "ABC123" { + t.Fatalf("APIError = %+v", apiErr) + } + if want := "Object Storage API error 404: NoSuchKey: The specified key does not exist."; apiErr.Error() != want { + t.Fatalf("Error() = %q, want %q", apiErr.Error(), want) + } + }) + + t.Run("invalid error body falls back to generic message", func(t *testing.T) { + err := parseAPIError("GetObject", 502, []byte("bad gateway")) + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error type = %T, want *APIError", err) + } + if apiErr.Code != "" || apiErr.Message != "GetObject: unexpected response" { + t.Fatalf("APIError = %+v", apiErr) + } + }) +} + +func TestIsNotFound(t *testing.T) { + tests := []struct { + description string + err error + expected bool + }{ + { + description: "NoSuchKey code", + err: &APIError{StatusCode: 404, Code: "NoSuchKey"}, + expected: true, + }, + { + description: "NoSuchBucket code", + err: &APIError{StatusCode: 404, Code: "NoSuchBucket"}, + expected: true, + }, + { + description: "404 without code", + err: &APIError{StatusCode: 404}, + expected: true, + }, + { + description: "AccessDenied is not not-found", + err: &APIError{StatusCode: 403, Code: "AccessDenied"}, + expected: false, + }, + { + description: "wrapped APIError", + err: fmt.Errorf("check object: %w", &APIError{StatusCode: 404, Code: "NoSuchKey"}), + expected: true, + }, + { + description: "error mentioning the code but not wrapping it", + err: errors.New("outer: Object Storage API error 404: NoSuchKey: The specified key does not exist."), + expected: false, + }, + { + description: "ErrNotFound sentinel", + err: ErrNotFound, + expected: true, + }, + { + description: "unrelated error", + err: errors.New("boom"), + expected: false, + }, + { + description: "nil", + err: nil, + expected: false, + }, + } + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + if got := IsNotFound(tt.err); got != tt.expected { + t.Fatalf("IsNotFound = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/internal/pkg/services/object-storage/bucket/signer.go b/internal/pkg/services/object-storage/bucket/signer.go new file mode 100644 index 000000000..93c0cda9b --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/signer.go @@ -0,0 +1,225 @@ +package bucket + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +const ( + signingAlgorithm = "AWS4-HMAC-SHA256" + signingService = "s3" + terminalType = "aws4_request" + credentialScopePrefix = "AWS4" + + emptyPayloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + unsignedPayload = "UNSIGNED-PAYLOAD" +) + +type Signer struct { + AccessKeyID string + SecretAccessKey string + Region string + Service string +} + +func NewSigner(accessKeyID, secretAccessKey, region string) *Signer { + return &Signer{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + Region: region, + Service: signingService, + } +} + +func (s *Signer) credentialScope(dateStamp string) string { + return dateStamp + "/" + s.Region + "/" + s.Service + "/" + terminalType +} + +func (s *Signer) signingKey(dateStamp string) []byte { + kDate := hmacSHA256([]byte(credentialScopePrefix+s.SecretAccessKey), dateStamp) + kRegion := hmacSHA256(kDate, s.Region) + kService := hmacSHA256(kRegion, s.Service) + return hmacSHA256(kService, terminalType) +} + +func canonicalPath(req *http.Request) string { + path := req.URL.Path + if path == "" { + return "/" + } + segments := strings.Split(path, "/") + for i, segment := range segments { + segments[i] = queryEncode(segment) + } + return strings.Join(segments, "/") +} + +func (s *Signer) Sign(req *http.Request, now time.Time) error { + now = now.UTC() + amzDate := now.Format("20060102T150405Z") + dateStamp := now.Format("20060102") + req.Header.Set("X-Amz-Date", amzDate) + + canonicalHeaders, signedHeaders := canonicalHeaders(req) + canonicalRequest := buildCanonicalRequest( + req.Method, + canonicalPath(req), + canonicalQueryString(req.URL.Query()), + canonicalHeaders, + signedHeaders, + payloadHash(req), + ) + signature := s.signature(canonicalRequest, amzDate, dateStamp) + req.Header.Set("Authorization", fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", + signingAlgorithm, s.AccessKeyID, s.credentialScope(dateStamp), signedHeaders, signature)) + return nil +} + +func (s *Signer) Presign(req *http.Request, expires time.Duration, now time.Time, payloadHash string) (string, error) { + now = now.UTC() + amzDate := now.Format("20060102T150405Z") + dateStamp := now.Format("20060102") + + query := req.URL.Query() + query.Set("X-Amz-Algorithm", signingAlgorithm) + query.Set("X-Amz-Credential", s.AccessKeyID+"/"+s.credentialScope(dateStamp)) + query.Set("X-Amz-Date", amzDate) + query.Set("X-Amz-Expires", strconv.FormatInt(int64(expires.Seconds()), 10)) + query.Set("X-Amz-SignedHeaders", "host") + + host := hostOf(req) + canonicalRequest := buildCanonicalRequest( + req.Method, + canonicalPath(req), + canonicalQueryString(query), + "host:"+host+"\n", + "host", + payloadHash, + ) + signature := s.signature(canonicalRequest, amzDate, dateStamp) + // X-Amz-Signature is not part of the canonical query string; append it last + // to match the SDK wire format + req.URL.RawQuery = canonicalQueryString(query) + "&X-Amz-Signature=" + signature + return req.URL.String(), nil +} + +func (s *Signer) signature(canonicalRequest, amzDate, dateStamp string) string { + stringToSign := strings.Join([]string{ + signingAlgorithm, + amzDate, + s.credentialScope(dateStamp), + sha256Hex([]byte(canonicalRequest)), + }, "\n") + return hex.EncodeToString(hmacSHA256(s.signingKey(dateStamp), stringToSign)) +} + +func buildCanonicalRequest(method, canonicalURI, canonicalQuery, canonicalHeaders, signedHeaders, payloadHash string) string { + return strings.Join([]string{method, canonicalURI, canonicalQuery, canonicalHeaders, signedHeaders, payloadHash}, "\n") +} + +func canonicalHeaders(req *http.Request) (headersBlock, signedHeaders string) { + names := make(map[string]string) + if host := hostOf(req); host != "" { + names["host"] = host + } + for name, values := range req.Header { + lower := strings.ToLower(name) + if !strings.HasPrefix(lower, "x-amz-") && lower != "content-type" { + continue + } + names[lower] = strings.Join(values, ",") + } + sorted := make([]string, 0, len(names)) + for name := range names { + sorted = append(sorted, name) + } + sort.Strings(sorted) + + var builder strings.Builder + for _, name := range sorted { + builder.WriteString(name) + builder.WriteByte(':') + builder.WriteString(strings.TrimSpace(names[name])) + builder.WriteByte('\n') + } + return builder.String(), strings.Join(sorted, ";") +} + +func canonicalQueryString(values url.Values) string { + type kv struct{ key, value string } + pairs := make([]kv, 0, len(values)) + for key, vals := range values { + for _, value := range vals { + pairs = append(pairs, kv{key: key, value: value}) + } + } + sort.Slice(pairs, func(i, j int) bool { + if pairs[i].key != pairs[j].key { + return pairs[i].key < pairs[j].key + } + return pairs[i].value < pairs[j].value + }) + parts := make([]string, 0, len(pairs)) + for _, pair := range pairs { + parts = append(parts, queryEncode(pair.key)+"="+queryEncode(pair.value)) + } + return strings.Join(parts, "&") +} + +func payloadHash(req *http.Request) string { + if v := req.Header.Get("X-Amz-Content-Sha256"); v != "" { + return v + } + return emptyPayloadHash +} + +func hostOf(req *http.Request) string { + if req.Host != "" { + return req.Host + } + return req.URL.Host +} + +func queryEncode(s string) string { + var builder strings.Builder + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'): + builder.WriteByte(c) + case c == '-' || c == '_' || c == '.' || c == '~': + builder.WriteByte(c) + default: + fmt.Fprintf(&builder, "%%%02X", c) + } + } + return builder.String() +} + +func escapeKey(key string) string { + segments := strings.Split(key, "/") + for i, segment := range segments { + segments[i] = queryEncode(segment) + } + return strings.Join(segments, "/") +} + +func hmacSHA256(key []byte, data string) []byte { + mac := hmac.New(sha256.New, key) + mac.Write([]byte(data)) + return mac.Sum(nil) +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/pkg/services/object-storage/bucket/signer_test.go b/internal/pkg/services/object-storage/bucket/signer_test.go new file mode 100644 index 000000000..374b68781 --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/signer_test.go @@ -0,0 +1,179 @@ +package bucket + +import ( + "net/http" + "testing" + "time" +) + +var ( + testTime = time.Date(2015, 8, 30, 12, 36, 0, 0, time.UTC) + testSigner = Signer{ //nolint:gosec // public example credentials from the AWS documentation + AccessKeyID: "AKIDEXAMPLE", + SecretAccessKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + Region: "us-east-1", + Service: "service", + } +) + +func TestSign(t *testing.T) { + // Reference computed with botocore (pinned timestamp): GET + // https://example.amazonaws.com/?Host=example.amazonaws.com, service "service", + // region us-east-1, date 20150830T123600Z + req, err := http.NewRequest("GET", "https://example.amazonaws.com/?Host=example.amazonaws.com", http.NoBody) + if err != nil { + t.Fatalf("build request: %v", err) + } + if err := testSigner.Sign(req, testTime); err != nil { + t.Fatalf("Sign: %v", err) + } + wantAuth := "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=ca564d2dbd30b429f5d637740c18b0039343ae449cf2bd0ba0b947298053f3c8" + if got := req.Header.Get("Authorization"); got != wantAuth { + t.Fatalf("Authorization = %q, want %q", got, wantAuth) + } + if got := req.Header.Get("X-Amz-Date"); got != "20150830T123600Z" { + t.Fatalf("X-Amz-Date = %q, want 20150830T123600Z", got) + } +} + +func TestSignSortsQueryParameters(t *testing.T) { + // Reference computed with botocore (pinned timestamp): the request query is + // out of order and must be canonicalized + req, err := http.NewRequest("GET", "https://example.amazonaws.com/?Param2=2&Param1=1", http.NoBody) + if err != nil { + t.Fatalf("build request: %v", err) + } + if err := testSigner.Sign(req, testTime); err != nil { + t.Fatalf("Sign: %v", err) + } + wantAuth := "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=afce2c564917f25d63492b2cf7c08495be06c0a160aba74e53bacf4caffe854c" + if got := req.Header.Get("Authorization"); got != wantAuth { + t.Fatalf("Authorization = %q, want %q", got, wantAuth) + } +} + +func TestPresign(t *testing.T) { + // Reference computed with botocore's S3SigV4QueryAuth (pinned timestamp): + // presigned download with UNSIGNED-PAYLOAD, signing only the host header + client := &Client{ + endpoint: "https://object.storage.eu01.onstackit.cloud", + region: "eu01", + signer: NewSigner("AKID", "SECRET", "eu01"), + now: func() time.Time { return time.Date(2024, 5, 1, 12, 0, 0, 0, time.UTC) }, + } + got, err := client.PresignGetObject("my-bucket", "dir/file.txt", 15*time.Minute) + if err != nil { + t.Fatalf("PresignGetObject: %v", err) + } + want := "https://object.storage.eu01.onstackit.cloud/my-bucket/dir/file.txt" + + "?X-Amz-Algorithm=AWS4-HMAC-SHA256" + + "&X-Amz-Credential=AKID%2F20240501%2Feu01%2Fs3%2Faws4_request" + + "&X-Amz-Date=20240501T120000Z" + + "&X-Amz-Expires=900" + + "&X-Amz-SignedHeaders=host" + + "&X-Amz-Signature=c58f23addabe8c12ea43bc65ef7e8e51c8f5bae00e0db74d7adb283448f39d65" + if got != want { + t.Fatalf("presigned URL = %s, want %s", got, want) + } +} + +func TestPresignEncodesKeyFully(t *testing.T) { + // Vector verified byte-for-byte against the AWS SDK presigner: a key with + // spaces, & and parentheses, UNSIGNED-PAYLOAD, and an extra query parameter + req, err := http.NewRequest("GET", "https://object.storage.eu01.onstackit.cloud/my-bucket/dir/file with space & (parens).txt", http.NoBody) + if err != nil { + t.Fatalf("build request: %v", err) + } + query := req.URL.Query() + query.Set("x-id", "GetObject") + req.URL.RawQuery = query.Encode() + req.URL.RawPath = canonicalPath(req) + + signer := NewSigner("AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "eu01") + got, err := signer.Presign(req, 15*time.Minute, time.Date(2026, 11, 16, 17, 6, 13, 0, time.UTC), unsignedPayload) + if err != nil { + t.Fatalf("Presign: %v", err) + } + want := "https://object.storage.eu01.onstackit.cloud/my-bucket/dir/file%20with%20space%20%26%20%28parens%29.txt" + + "?X-Amz-Algorithm=AWS4-HMAC-SHA256" + + "&X-Amz-Credential=AKIDEXAMPLE%2F20261116%2Feu01%2Fs3%2Faws4_request" + + "&X-Amz-Date=20261116T170613Z" + + "&X-Amz-Expires=900" + + "&X-Amz-SignedHeaders=host" + + "&x-id=GetObject" + + "&X-Amz-Signature=a9e8de480dde798cb296f1f81433ac4bf6183f03c49711646c0f99ebdcea908c" + if got != want { + t.Fatalf("presigned URL = %s, want %s", got, want) + } +} + +func TestCanonicalPath(t *testing.T) { + tests := []struct { + description string + rawURL string + expected string + }{ + {description: "root", rawURL: "https://example.com", expected: "/"}, + {description: "simple", rawURL: "https://example.com/bucket/key.txt", expected: "/bucket/key.txt"}, + {description: "reserved characters are fully encoded", rawURL: "https://example.com/bucket/a b&c(d).txt", expected: "/bucket/a%20b%26c%28d%29.txt"}, + {description: "slash stays unencoded", rawURL: "https://example.com/bucket/dir/file.txt", expected: "/bucket/dir/file.txt"}, + {description: "unreserved characters stay as-is", rawURL: "https://example.com/bucket/a-b_c.d~e.txt", expected: "/bucket/a-b_c.d~e.txt"}, + } + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + req, err := http.NewRequest("GET", tt.rawURL, http.NoBody) + if err != nil { + t.Fatalf("build request: %v", err) + } + if got := canonicalPath(req); got != tt.expected { + t.Fatalf("canonicalPath = %q, want %q", got, tt.expected) + } + }) + } +} + +func TestQueryEncode(t *testing.T) { + tests := []struct { + description string + value string + expected string + }{ + {description: "unreserved", value: "AZaz09-_.~", expected: "AZaz09-_.~"}, + {description: "space", value: "a b", expected: "a%20b"}, + {description: "ampersand", value: "a&b", expected: "a%26b"}, + {description: "parens", value: "(a)", expected: "%28a%29"}, + {description: "slash is encoded in query values", value: "a/b", expected: "a%2Fb"}, + {description: "plus", value: "a+b", expected: "a%2Bb"}, + {description: "empty", value: "", expected: ""}, + } + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + if got := queryEncode(tt.value); got != tt.expected { + t.Fatalf("queryEncode(%q) = %q, want %q", tt.value, got, tt.expected) + } + }) + } +} + +func TestCanonicalQueryString(t *testing.T) { + req, err := http.NewRequest("GET", "https://example.com/?list-type=2&prefix=dir/sub&prefix=a b&z=1&a=2", http.NoBody) + if err != nil { + t.Fatalf("build request: %v", err) + } + got := canonicalQueryString(req.URL.Query()) + want := "a=2&list-type=2&prefix=a%20b&prefix=dir%2Fsub&z=1" + if got != want { + t.Fatalf("canonicalQueryString = %q, want %q", got, want) + } +} + +func TestPayloadHash(t *testing.T) { + req, _ := http.NewRequest("GET", "https://example.com/", http.NoBody) + if got := payloadHash(req); got != emptyPayloadHash { + t.Fatalf("payloadHash without header = %q, want %q", got, emptyPayloadHash) + } + req.Header.Set("X-Amz-Content-Sha256", unsignedPayload) + if got := payloadHash(req); got != unsignedPayload { + t.Fatalf("payloadHash with header = %q, want %q", got, unsignedPayload) + } +} diff --git a/internal/pkg/services/object-storage/bucket/transfer.go b/internal/pkg/services/object-storage/bucket/transfer.go new file mode 100644 index 000000000..1466e1b34 --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/transfer.go @@ -0,0 +1,304 @@ +package bucket + +import ( + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +const maxDeleteBatchSize = 1000 + +type BucketInfo struct { + Name string `json:"name"` + CreationDate time.Time `json:"creation_date"` +} + +type ObjectInfo struct { + Key string `json:"key"` + Size int64 `json:"size"` + LastModified time.Time `json:"last_modified"` +} + +func ListBuckets(ctx context.Context, client *Client) ([]BucketInfo, error) { + return client.ListBuckets(ctx) +} + +func ListObjects(ctx context.Context, client *Client, ref Reference) ([]ObjectInfo, error) { + if ref.IsBucket() { + return nil, fmt.Errorf("cannot list objects of a bare bucket reference %q without a key", ref.String()) + } + objects := make([]ObjectInfo, 0) + token := "" + for { + page, err := client.ListObjectsV2(ctx, ref.Bucket, ref.Key, "", token) + if err != nil { + return nil, fmt.Errorf("list objects in %q: %w", ref.String(), err) + } + objects = append(objects, page.Objects...) + if !page.IsTruncated { + break + } + token = page.NextContinuationToken + } + return objects, nil +} + +func ListPrefixes(ctx context.Context, client *Client, ref Reference) (prefixes []string, objects []ObjectInfo, err error) { + page, err := client.ListObjectsV2(ctx, ref.Bucket, ref.Key, "/", "") + if err != nil { + return nil, nil, fmt.Errorf("list objects in %q: %w", ref.String(), err) + } + prefixes = append(prefixes, page.CommonPrefixes...) + objects = append(objects, page.Objects...) + return prefixes, objects, nil +} + +func HeadObjectExists(ctx context.Context, client *Client, ref Reference) (bool, error) { + if ref.IsBucket() { + return false, fmt.Errorf("cannot head a bare bucket reference %q without a key", ref.String()) + } + err := client.HeadObject(ctx, ref.Bucket, ref.Key) + if IsNotFound(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("check object %q: %w", ref.String(), err) + } + return true, nil +} + +func DownloadFile(ctx context.Context, client *Client, ref Reference, localPath string) error { + exists, err := HeadObjectExists(ctx, client, ref) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("object %q does not exist: if it is a prefix, use --recursive", ref.String()) + } + return downloadObject(ctx, client, ref, localPath) +} + +func downloadObject(ctx context.Context, client *Client, ref Reference, localPath string) error { + body, err := client.GetObject(ctx, ref.Bucket, ref.Key) + if err != nil { + return fmt.Errorf("download object %q: %w", ref.String(), err) + } + defer func() { + if cerr := body.Close(); cerr != nil { + fmt.Fprintf(os.Stderr, "close object body: %v\n", cerr) + } + }() + + if err := os.MkdirAll(filepath.Dir(localPath), 0o750); err != nil { + return fmt.Errorf("create local directory for %q: %w", localPath, err) + } + // Write to a temp file and rename, so a failed copy never leaves a + // partial file at the destination + file, err := os.CreateTemp(filepath.Dir(localPath), filepath.Base(localPath)+".tmp-*") + if err != nil { + return fmt.Errorf("create local file %q: %w", localPath, err) + } + tmpPath := file.Name() + defer func() { + // No-op after a successful rename; removes the temp file on failure + _ = os.Remove(tmpPath) + }() + if _, err := io.Copy(file, body); err != nil { + if cerr := file.Close(); cerr != nil { + fmt.Fprintf(os.Stderr, "close local file: %v\n", cerr) + } + return fmt.Errorf("write local file %q: %w", localPath, err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close local file %q: %w", localPath, err) + } + if err := os.Rename(tmpPath, localPath); err != nil { + return fmt.Errorf("write local file %q: %w", localPath, err) + } + return nil +} + +func DownloadPrefix(ctx context.Context, client *Client, ref Reference, localDir string) (int, error) { + objects, err := ListObjects(ctx, client, ref) + if err != nil { + return 0, err + } + for _, obj := range objects { + relPath := strings.TrimPrefix(obj.Key, ref.Key) + relPath = strings.TrimPrefix(relPath, "/") + if relPath == "" { + continue + } + localPath := filepath.Join(localDir, relPath) + if err := downloadObject(ctx, client, Reference{Bucket: ref.Bucket, Key: obj.Key}, localPath); err != nil { + return 0, err + } + } + return len(objects), nil +} + +func UploadFile(ctx context.Context, client *Client, localPath string, ref Reference) error { + file, err := os.Open(localPath) + if err != nil { + return fmt.Errorf("open local file %q: %w", localPath, err) + } + defer func() { + if cerr := file.Close(); cerr != nil { + fmt.Fprintf(os.Stderr, "close local file: %v\n", cerr) + } + }() + if err := client.PutObject(ctx, ref.Bucket, ref.Key, file); err != nil { + return fmt.Errorf("upload %q to %q: %w", localPath, ref.String(), err) + } + return nil +} + +func UploadDirectory(ctx context.Context, client *Client, localDir string, ref Reference) (int, error) { + count := 0 + prefix := strings.TrimSuffix(ref.Key, "/") + err := filepath.WalkDir(localDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return fmt.Errorf("walk %q: %w", path, walkErr) + } + if d.IsDir() { + return nil + } + relPath, err := filepath.Rel(localDir, path) + if err != nil { + return fmt.Errorf("resolve relative path of %q: %w", path, err) + } + key := filepath.ToSlash(relPath) + if prefix != "" { + key = prefix + "/" + key + } + if err := UploadFile(ctx, client, path, Reference{Bucket: ref.Bucket, Key: key}); err != nil { + return err + } + count++ + return nil + }) + if err != nil { + return 0, err + } + return count, nil +} + +func DeleteObjects(ctx context.Context, client *Client, refs []Reference) (int, error) { + if len(refs) == 0 { + return 0, nil + } + deleted := 0 + for start := 0; start < len(refs); start += maxDeleteBatchSize { + end := min(start+maxDeleteBatchSize, len(refs)) + batch := refs[start:end] + keys := make([]string, 0, len(batch)) + for _, ref := range batch { + keys = append(keys, ref.Key) + } + n, err := client.DeleteObjects(ctx, batch[0].Bucket, keys) + if err != nil { + return deleted, fmt.Errorf("delete %d objects from %q: %w", len(batch), batch[0].Bucket, err) + } + deleted += n + } + return deleted, nil +} + +func ResolveObjectRefs(ctx context.Context, client *Client, ref Reference, recursive bool) ([]Reference, error) { + if !recursive { + exists, err := HeadObjectExists(ctx, client, ref) + if err != nil { + return nil, err + } + if !exists { + return nil, fmt.Errorf("object %q does not exist: if it is a prefix, use --recursive", ref.String()) + } + return []Reference{ref}, nil + } + objects, err := ListObjects(ctx, client, ref) + if err != nil { + return nil, err + } + if len(objects) == 0 { + return nil, fmt.Errorf("no objects found under %q", ref.String()) + } + refs := make([]Reference, 0, len(objects)) + for _, obj := range objects { + refs = append(refs, Reference{Bucket: ref.Bucket, Key: obj.Key}) + } + return refs, nil +} + +func PresignGetObject(_ context.Context, client *Client, ref Reference, expires time.Duration) (string, error) { + url, err := client.PresignGetObject(ref.Bucket, ref.Key, expires) + if err != nil { + return "", fmt.Errorf("presign URL for %q: %w", ref.String(), err) + } + return url, nil +} + +func CopyObject(ctx context.Context, client *Client, src, dst Reference) error { + if err := client.CopyObject(ctx, src, dst); err != nil { + return fmt.Errorf("copy %q to %q: %w", src.String(), dst.String(), err) + } + return nil +} + +func localDestinationForObject(dst, key string) string { + base := filepath.Base(key) + if dst == "" { + return base + } + if strings.HasSuffix(dst, "/") || strings.HasSuffix(dst, string(filepath.Separator)) { + return filepath.Join(dst, base) + } + if stat, err := os.Stat(dst); err == nil && stat.IsDir() { + return filepath.Join(dst, base) + } + return dst +} + +func Download(ctx context.Context, client *Client, ref Reference, localPath string, recursive bool) (int, error) { + if ref.IsBucket() { + return 0, fmt.Errorf("cannot download a whole bucket %q: specify an object or prefix", ref.String()) + } + if recursive { + // Normalize to a path prefix so that "dir" does not also match + // "dir-other/..." and map to a bogus local path + ref.Key = strings.TrimSuffix(ref.Key, "/") + "/" + return DownloadPrefix(ctx, client, ref, localPath) + } + local := localDestinationForObject(localPath, ref.Key) + err := DownloadFile(ctx, client, ref, local) + if err != nil { + return 0, err + } + return 1, nil +} + +func Upload(ctx context.Context, client *Client, localPath string, ref Reference, recursive bool) (int, error) { + stat, err := os.Stat(localPath) + if err != nil { + return 0, fmt.Errorf("stat local path %q: %w", localPath, err) + } + if stat.IsDir() { + if !recursive { + return 0, fmt.Errorf("local path %q is a directory: use --recursive", localPath) + } + return UploadDirectory(ctx, client, localPath, ref) + } + key := ref.Key + if key == "" || strings.HasSuffix(key, "/") { + key += filepath.Base(localPath) + } + err = UploadFile(ctx, client, localPath, Reference{Bucket: ref.Bucket, Key: key}) + if err != nil { + return 0, err + } + return 1, nil +} diff --git a/internal/pkg/services/object-storage/bucket/transfer_test.go b/internal/pkg/services/object-storage/bucket/transfer_test.go new file mode 100644 index 000000000..6a274becc --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/transfer_test.go @@ -0,0 +1,603 @@ +package bucket + +import ( + "context" + "encoding/xml" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +type fakeStorage struct { + mu sync.Mutex + objects map[string][]byte + pageSize int + listCalls int + deleteCalls [][]string + copySources map[string]string +} + +func newFakeStorage(pageSize int) *fakeStorage { + return &fakeStorage{ + objects: map[string][]byte{}, + pageSize: pageSize, + copySources: map[string]string{}, + } +} + +func (f *fakeStorage) put(bucket, key string, body []byte) { + f.objects[bucket+"/"+key] = body +} + +func (f *fakeStorage) handler(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + + parts := strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2) + bucket := parts[0] + key := "" + if len(parts) == 2 { + key = parts[1] + } + + switch { + case r.Method == http.MethodGet && key == "" && r.URL.Query().Get("list-type") == "2": + f.listCalls++ + f.writeList(w, r, bucket) + case r.Method == http.MethodHead: + if _, ok := f.objects[bucket+"/"+key]; ok { + w.WriteHeader(http.StatusOK) + } else { + writeNoSuchKey(w) + } + case r.Method == http.MethodGet: + if body, ok := f.objects[bucket+"/"+key]; ok { + _, _ = w.Write(body) + } else { + writeNoSuchKey(w) + } + case r.Method == http.MethodPut: + if src := r.Header.Get("X-Amz-Copy-Source"); src != "" { + f.copySources[bucket+"/"+key] = src + w.WriteHeader(http.StatusOK) + } else { + body, _ := io.ReadAll(r.Body) + f.objects[bucket+"/"+key] = body + w.WriteHeader(http.StatusOK) + } + case r.Method == http.MethodPost && r.URL.Query().Has("delete"): + body, _ := io.ReadAll(r.Body) + var req deleteRequest + if err := xml.Unmarshal(body, &req); err != nil { + http.Error(w, "bad delete request: "+err.Error(), http.StatusBadRequest) + return + } + keys := make([]string, 0, len(req.Objects)) + for _, obj := range req.Objects { + keys = append(keys, obj.Key) + delete(f.objects, bucket+"/"+obj.Key) + } + f.deleteCalls = append(f.deleteCalls, keys) + _, _ = w.Write([]byte(``)) + default: + http.Error(w, "unexpected request: "+r.Method+" "+r.URL.Path, http.StatusNotImplemented) + } +} + +func (f *fakeStorage) writeList(w http.ResponseWriter, r *http.Request, bucket string) { + query := r.URL.Query() + prefix := query.Get("prefix") + delimiter := query.Get("delimiter") + + keys := make([]string, 0) + for key := range f.objects { + if !strings.HasPrefix(key, bucket+"/") { + continue + } + rel := strings.TrimPrefix(key, bucket+"/") + if strings.HasPrefix(rel, prefix) { + keys = append(keys, rel) + } + } + sort.Strings(keys) + + type entry struct { + full string + isPrefix bool + } + var entries []entry + seen := map[string]bool{} + for _, key := range keys { + rel := strings.TrimPrefix(key, prefix) + if delimiter != "" { + if i := strings.Index(rel, delimiter); i >= 0 { + common := prefix + rel[:i+len(delimiter)] + if !seen[common] { + seen[common] = true + entries = append(entries, entry{full: common, isPrefix: true}) + } + continue + } + } + entries = append(entries, entry{full: key}) + } + + offset, _ := strconv.Atoi(query.Get("continuation-token")) + end := len(entries) + if f.pageSize > 0 && offset+f.pageSize < len(entries) { + end = offset + f.pageSize + } + page := entries[offset:end] + + var b strings.Builder + b.WriteString(``) + if end < len(entries) { + b.WriteString(`true`) + b.WriteString(strconv.Itoa(end)) + b.WriteString(``) + } else { + b.WriteString(`false`) + } + for _, e := range page { + if e.isPrefix { + b.WriteString(``) + b.WriteString(xmlEscape(e.full)) + b.WriteString(``) + continue + } + b.WriteString(``) + b.WriteString(xmlEscape(e.full)) + b.WriteString(``) + b.WriteString(strconv.Itoa(len(f.objects[bucket+"/"+e.full]))) + b.WriteString(`2024-05-01T12:00:00Z`) + } + b.WriteString(``) + _, _ = w.Write([]byte(b.String())) +} + +func writeNoSuchKey(w http.ResponseWriter) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`NoSuchKeyThe specified key does not exist.`)) +} + +func xmlEscape(s string) string { + return strings.NewReplacer("&", "&", "<", "<", ">", ">", `"`, """).Replace(s) +} + +func newFakeClient(t *testing.T, fake *fakeStorage) *Client { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(fake.handler)) + t.Cleanup(server.Close) + client, err := NewClient(ClientConfig{ + Region: "eu01", + Endpoint: server.URL, + AccessKeyId: "AKIDEXAMPLE", + SecretAccessKey: "secret", + }) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + return client +} + +func TestListObjectsPaginates(t *testing.T) { + fake := newFakeStorage(2) + for i := 0; i < 5; i++ { + fake.put("my-bucket", "p/file-"+strconv.Itoa(i)+".txt", []byte("x")) + } + client := newFakeClient(t, fake) + + objects, err := ListObjects(context.Background(), client, Reference{Bucket: "my-bucket", Key: "p/"}) + if err != nil { + t.Fatalf("ListObjects: %v", err) + } + if len(objects) != 5 { + t.Fatalf("ListObjects returned %d objects, want 5", len(objects)) + } + if objects[0].Key != "p/file-0.txt" || objects[4].Key != "p/file-4.txt" { + t.Fatalf("unexpected object order: %+v", objects) + } + if fake.listCalls != 3 { + t.Fatalf("list calls = %d, want 3", fake.listCalls) + } +} + +func TestListObjectsBareBucket(t *testing.T) { + fake := newFakeStorage(0) + client := newFakeClient(t, fake) + + _, err := ListObjects(context.Background(), client, Reference{Bucket: "my-bucket"}) + if err == nil { + t.Fatal("expected an error for a bare bucket reference") + } +} + +func TestListPrefixes(t *testing.T) { + fake := newFakeStorage(0) + fake.put("my-bucket", "a/1.txt", []byte("x")) + fake.put("my-bucket", "a/2.txt", []byte("x")) + fake.put("my-bucket", "b/3.txt", []byte("x")) + fake.put("my-bucket", "top.txt", []byte("x")) + client := newFakeClient(t, fake) + + prefixes, objects, err := ListPrefixes(context.Background(), client, Reference{Bucket: "my-bucket"}) + if err != nil { + t.Fatalf("ListPrefixes: %v", err) + } + wantPrefixes := []string{"a/", "b/"} + if len(prefixes) != len(wantPrefixes) || prefixes[0] != wantPrefixes[0] || prefixes[1] != wantPrefixes[1] { + t.Fatalf("prefixes = %v, want %v", prefixes, wantPrefixes) + } + if len(objects) != 1 || objects[0].Key != "top.txt" { + t.Fatalf("objects = %+v, want [top.txt]", objects) + } +} + +func TestHeadObjectExists(t *testing.T) { + fake := newFakeStorage(0) + fake.put("my-bucket", "there.txt", []byte("x")) + client := newFakeClient(t, fake) + + tests := []struct { + description string + ref Reference + expected bool + isValid bool + }{ + {description: "exists", ref: Reference{Bucket: "my-bucket", Key: "there.txt"}, expected: true, isValid: true}, + {description: "missing", ref: Reference{Bucket: "my-bucket", Key: "nowhere.txt"}, expected: false, isValid: true}, + {description: "bare bucket", ref: Reference{Bucket: "my-bucket"}, isValid: false}, + } + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + exists, err := HeadObjectExists(context.Background(), client, tt.ref) + if (err != nil) != !tt.isValid { + t.Fatalf("HeadObjectExists error = %v, wantErr %v", err, !tt.isValid) + } + if tt.isValid && exists != tt.expected { + t.Fatalf("HeadObjectExists = %v, want %v", exists, tt.expected) + } + }) + } +} + +func TestDownloadFile(t *testing.T) { + fake := newFakeStorage(0) + fake.put("my-bucket", "dir/file.txt", []byte("hello world")) + client := newFakeClient(t, fake) + localPath := filepath.Join(t.TempDir(), "out", "sub", "file.txt") + + err := DownloadFile(context.Background(), client, Reference{Bucket: "my-bucket", Key: "dir/file.txt"}, localPath) + if err != nil { + t.Fatalf("DownloadFile: %v", err) + } + content, err := os.ReadFile(localPath) + if err != nil { + t.Fatalf("read downloaded file: %v", err) + } + if string(content) != "hello world" { + t.Fatalf("downloaded content = %q, want %q", content, "hello world") + } +} + +func TestDownloadFileMissingObject(t *testing.T) { + fake := newFakeStorage(0) + client := newFakeClient(t, fake) + + err := DownloadFile(context.Background(), client, Reference{Bucket: "my-bucket", Key: "nope.txt"}, filepath.Join(t.TempDir(), "nope.txt")) + if err == nil || !strings.Contains(err.Error(), "does not exist") { + t.Fatalf("error = %v, want it to mention the missing object", err) + } +} + +func TestDownloadPrefix(t *testing.T) { + fake := newFakeStorage(0) + fake.put("my-bucket", "p/a.txt", []byte("a")) + fake.put("my-bucket", "p/nested/b.txt", []byte("b")) + fake.put("my-bucket", "p/c.txt", []byte("c")) + fake.put("my-bucket", "other/d.txt", []byte("d")) + client := newFakeClient(t, fake) + localDir := t.TempDir() + + count, err := DownloadPrefix(context.Background(), client, Reference{Bucket: "my-bucket", Key: "p/"}, localDir) + if err != nil { + t.Fatalf("DownloadPrefix: %v", err) + } + if count != 3 { + t.Fatalf("count = %d, want 3", count) + } + for _, rel := range []string{"a.txt", "nested/b.txt", "c.txt"} { + if _, err := os.Stat(filepath.Join(localDir, rel)); err != nil { + t.Fatalf("expected downloaded file %q: %v", rel, err) + } + } +} + +func TestDownloadPrefixExcludesSiblingPrefixes(t *testing.T) { + fake := newFakeStorage(0) + fake.put("my-bucket", "dir/a.txt", []byte("a")) + fake.put("my-bucket", "dir-other/x.txt", []byte("x")) + client := newFakeClient(t, fake) + localDir := t.TempDir() + + // A prefix without a trailing slash must behave like a path prefix + count, err := Download(context.Background(), client, Reference{Bucket: "my-bucket", Key: "dir"}, localDir, true) + if err != nil { + t.Fatalf("Download: %v", err) + } + if count != 1 { + t.Fatalf("count = %d, want 1", count) + } + if _, err := os.Stat(filepath.Join(localDir, "a.txt")); err != nil { + t.Fatalf("expected downloaded file a.txt: %v", err) + } + if _, err := os.Stat(filepath.Join(localDir, "-other")); !os.IsNotExist(err) { + t.Fatalf("did not expect a local path for the sibling prefix dir-other") + } +} + +func TestUploadFile(t *testing.T) { + fake := newFakeStorage(0) + client := newFakeClient(t, fake) + localPath := filepath.Join(t.TempDir(), "source.txt") + if err := os.WriteFile(localPath, []byte("upload me"), 0o600); err != nil { + t.Fatalf("write local file: %v", err) + } + + err := UploadFile(context.Background(), client, localPath, Reference{Bucket: "my-bucket", Key: "dest/file.txt"}) + if err != nil { + t.Fatalf("UploadFile: %v", err) + } + if string(fake.objects["my-bucket/dest/file.txt"]) != "upload me" { + t.Fatalf("server received %q, want %q", fake.objects["my-bucket/dest/file.txt"], "upload me") + } +} + +func TestUploadDirectory(t *testing.T) { + fake := newFakeStorage(0) + client := newFakeClient(t, fake) + localDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(localDir, "nested"), 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(localDir, "a.txt"), []byte("a"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + if err := os.WriteFile(filepath.Join(localDir, "nested", "b.txt"), []byte("b"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + + count, err := UploadDirectory(context.Background(), client, localDir, Reference{Bucket: "my-bucket", Key: "pre/"}) + if err != nil { + t.Fatalf("UploadDirectory: %v", err) + } + if count != 2 { + t.Fatalf("count = %d, want 2", count) + } + if _, ok := fake.objects["my-bucket/pre/a.txt"]; !ok { + t.Fatalf("missing uploaded key pre/a.txt: %v", fake.objects) + } + if _, ok := fake.objects["my-bucket/pre/nested/b.txt"]; !ok { + t.Fatalf("missing uploaded key pre/nested/b.txt: %v", fake.objects) + } +} + +func TestUploadDirectoryToBareBucket(t *testing.T) { + fake := newFakeStorage(0) + client := newFakeClient(t, fake) + localDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(localDir, "nested"), 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(localDir, "a.txt"), []byte("a"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + if err := os.WriteFile(filepath.Join(localDir, "nested", "b.txt"), []byte("b"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + + count, err := UploadDirectory(context.Background(), client, localDir, Reference{Bucket: "my-bucket"}) + if err != nil { + t.Fatalf("UploadDirectory: %v", err) + } + if count != 2 { + t.Fatalf("count = %d, want 2", count) + } + if _, ok := fake.objects["my-bucket/a.txt"]; !ok { + t.Fatalf("missing uploaded key a.txt: %v", fake.objects) + } + if _, ok := fake.objects["my-bucket/nested/b.txt"]; !ok { + t.Fatalf("missing uploaded key nested/b.txt: %v", fake.objects) + } + for key := range fake.objects { + if strings.HasPrefix(key, "my-bucket//") { + t.Fatalf("unexpected leading-slash key %q", key) + } + } +} + +func TestDeleteObjectsBatchesInGroupsOf1000(t *testing.T) { + fake := newFakeStorage(0) + client := newFakeClient(t, fake) + refs := make([]Reference, 0, 2500) + for i := range 2500 { + refs = append(refs, Reference{Bucket: "my-bucket", Key: "k" + strconv.Itoa(i)}) + } + + deleted, err := DeleteObjects(context.Background(), client, refs) + if err != nil { + t.Fatalf("DeleteObjects: %v", err) + } + if deleted != 2500 { + t.Fatalf("deleted = %d, want 2500", deleted) + } + if len(fake.deleteCalls) != 3 { + t.Fatalf("delete calls = %d, want 3", len(fake.deleteCalls)) + } + if len(fake.deleteCalls[0]) != 1000 || len(fake.deleteCalls[1]) != 1000 || len(fake.deleteCalls[2]) != 500 { + t.Fatalf("batch sizes = %d/%d/%d, want 1000/1000/500", + len(fake.deleteCalls[0]), len(fake.deleteCalls[1]), len(fake.deleteCalls[2])) + } +} + +func TestDeleteObjectsEmpty(t *testing.T) { + fake := newFakeStorage(0) + client := newFakeClient(t, fake) + + deleted, err := DeleteObjects(context.Background(), client, nil) + if err != nil { + t.Fatalf("DeleteObjects: %v", err) + } + if deleted != 0 { + t.Fatalf("deleted = %d, want 0", deleted) + } + if len(fake.deleteCalls) != 0 { + t.Fatalf("delete calls = %d, want 0", len(fake.deleteCalls)) + } +} + +func TestCopyObject(t *testing.T) { + fake := newFakeStorage(0) + fake.put("my-bucket", "src.txt", []byte("data")) + client := newFakeClient(t, fake) + + err := CopyObject(context.Background(), client, + Reference{Bucket: "my-bucket", Key: "src.txt"}, + Reference{Bucket: "my-bucket", Key: "dst.txt"}) + if err != nil { + t.Fatalf("CopyObject: %v", err) + } + if got := fake.copySources["my-bucket/dst.txt"]; got != "my-bucket/src.txt" { + t.Fatalf("copy source = %q, want %q", got, "my-bucket/src.txt") + } +} + +func TestPresignGetObject(t *testing.T) { + fake := newFakeStorage(0) + client := newFakeClient(t, fake) + + url, err := PresignGetObject(context.Background(), client, Reference{Bucket: "my-bucket", Key: "dir/file.txt"}, 15*time.Minute) + if err != nil { + t.Fatalf("PresignGetObject: %v", err) + } + if !strings.Contains(url, "my-bucket/dir/file.txt") || !strings.Contains(url, "X-Amz-Signature=") { + t.Fatalf("presigned URL %q is missing the key or the signature", url) + } +} + +func TestLocalDestinationForObject(t *testing.T) { + existingDir := t.TempDir() + tests := []struct { + description string + dst string + key string + expected string + }{ + {description: "empty dst uses the base name", dst: "", key: "dir/file.txt", expected: "file.txt"}, + {description: "dst with trailing slash appends the base name", dst: "/tmp/x/", key: "dir/file.txt", expected: "/tmp/x/file.txt"}, + {description: "existing directory appends the base name", dst: existingDir, key: "dir/file.txt", expected: filepath.Join(existingDir, "file.txt")}, + {description: "file path is used as-is", dst: "/tmp/out.txt", key: "dir/file.txt", expected: "/tmp/out.txt"}, + } + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + if got := localDestinationForObject(tt.dst, tt.key); got != tt.expected { + t.Fatalf("localDestinationForObject = %q, want %q", got, tt.expected) + } + }) + } +} + +func TestDownload(t *testing.T) { + fake := newFakeStorage(0) + fake.put("my-bucket", "file.txt", []byte("single")) + fake.put("my-bucket", "p/a.txt", []byte("a")) + fake.put("my-bucket", "p/b.txt", []byte("b")) + client := newFakeClient(t, fake) + + t.Run("bare bucket is rejected", func(t *testing.T) { + _, err := Download(context.Background(), client, Reference{Bucket: "my-bucket"}, t.TempDir(), false) + if err == nil { + t.Fatal("expected an error for a bare bucket reference") + } + }) + + t.Run("single object", func(t *testing.T) { + localPath := filepath.Join(t.TempDir(), "out.txt") + count, err := Download(context.Background(), client, Reference{Bucket: "my-bucket", Key: "file.txt"}, localPath, false) + if err != nil { + t.Fatalf("Download: %v", err) + } + if count != 1 { + t.Fatalf("count = %d, want 1", count) + } + content, err := os.ReadFile(localPath) + if err != nil { + t.Fatalf("read downloaded file: %v", err) + } + if string(content) != "single" { + t.Fatalf("content = %q, want %q", content, "single") + } + }) + + t.Run("recursive prefix", func(t *testing.T) { + localDir := t.TempDir() + count, err := Download(context.Background(), client, Reference{Bucket: "my-bucket", Key: "p/"}, localDir, true) + if err != nil { + t.Fatalf("Download: %v", err) + } + if count != 2 { + t.Fatalf("count = %d, want 2", count) + } + }) +} + +func TestUpload(t *testing.T) { + fake := newFakeStorage(0) + client := newFakeClient(t, fake) + localFile := filepath.Join(t.TempDir(), "source.txt") + if err := os.WriteFile(localFile, []byte("data"), 0o600); err != nil { + t.Fatalf("write local file: %v", err) + } + + t.Run("missing local path", func(t *testing.T) { + _, err := Upload(context.Background(), client, filepath.Join(t.TempDir(), "missing.txt"), Reference{Bucket: "my-bucket"}, false) + if err == nil { + t.Fatal("expected an error for a missing local path") + } + }) + + t.Run("directory without recursive flag", func(t *testing.T) { + _, err := Upload(context.Background(), client, t.TempDir(), Reference{Bucket: "my-bucket"}, false) + if err == nil || !strings.Contains(err.Error(), "--recursive") { + t.Fatalf("error = %v, want it to mention --recursive", err) + } + }) + + t.Run("trailing slash key appends the base name", func(t *testing.T) { + _, err := Upload(context.Background(), client, localFile, Reference{Bucket: "my-bucket", Key: "pre/"}, false) + if err != nil { + t.Fatalf("Upload: %v", err) + } + if _, ok := fake.objects["my-bucket/pre/source.txt"]; !ok { + t.Fatalf("missing uploaded key pre/source.txt: %v", fake.objects) + } + }) + + t.Run("empty key appends the base name", func(t *testing.T) { + _, err := Upload(context.Background(), client, localFile, Reference{Bucket: "my-bucket"}, false) + if err != nil { + t.Fatalf("Upload: %v", err) + } + if _, ok := fake.objects["my-bucket/source.txt"]; !ok { + t.Fatalf("missing uploaded key source.txt: %v", fake.objects) + } + }) +} diff --git a/internal/pkg/services/object-storage/bucket/uri.go b/internal/pkg/services/object-storage/bucket/uri.go new file mode 100644 index 000000000..af3368488 --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/uri.go @@ -0,0 +1,151 @@ +package bucket + +import ( + "fmt" + "net" + "net/url" + "regexp" + "strings" +) + +// URIPrefix is the de-facto standard object storage URI scheme, accepted in +// addition to STACKIT Object Storage URLs +const URIPrefix = "s3://" + +type Reference struct { + Bucket string + Key string + // Endpoint is the data-plane base endpoint (scheme + host) extracted from a + // STACKIT https bucket URL. Empty for s3:// references, where it is derived + // from the region. + Endpoint string + // Region is extracted from a STACKIT https bucket URL. Empty for s3:// references. + Region string +} + +var ( + bucketNameRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`) + virtualHostRegex = regexp.MustCompile(`^([a-z0-9][a-z0-9.-]{1,61}[a-z0-9])\.object\.storage\.([a-z0-9-]+)\.onstackit\.cloud$`) + pathStyleHostRegex = regexp.MustCompile(`^object\.storage\.([a-z0-9-]+)\.onstackit\.cloud$`) +) + +func IsBucketURI(s string) bool { + return strings.HasPrefix(s, URIPrefix) +} + +func IsObjectRef(s string) bool { + return IsBucketURI(s) || strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") +} + +func (r Reference) IsBucket() bool { + return r.Key == "" +} + +func (r Reference) String() string { + base := URIPrefix + r.Bucket + if r.Endpoint != "" { + base = r.Endpoint + "/" + r.Bucket + } + if r.IsBucket() { + return base + } + return base + "/" + r.Key +} + +func ParseBucketURI(uri string) (Reference, error) { + switch { + case IsBucketURI(uri): + return parseBucketSchemeURI(uri) + case strings.HasPrefix(uri, "http://"), strings.HasPrefix(uri, "https://"): + return parseBucketURL(uri) + default: + return Reference{}, fmt.Errorf("%q is not a bucket URI (expected the %q prefix) or a STACKIT Object Storage URL (https://...)", uri, URIPrefix) + } +} + +func parseBucketSchemeURI(uri string) (Reference, error) { + rest := strings.TrimPrefix(uri, URIPrefix) + if rest == "" { + return Reference{}, fmt.Errorf("missing bucket in URI %q", uri) + } + parts := strings.SplitN(rest, "/", 2) + bucket := parts[0] + if err := ValidateBucketName(bucket); err != nil { + return Reference{}, fmt.Errorf("invalid bucket in URI %q: %w", uri, err) + } + key := "" + if len(parts) == 2 { + key = strings.TrimLeft(parts[1], "/") + } + return Reference{Bucket: bucket, Key: key}, nil +} + +func parseBucketURL(uri string) (Reference, error) { + u, err := url.Parse(uri) + if err != nil { + return Reference{}, fmt.Errorf("parse URL %q: %w", uri, err) + } + host := u.Hostname() + if host == "" { + return Reference{}, fmt.Errorf("missing host in URL %q", uri) + } + base := u.Scheme + "://" + host + path := strings.TrimLeft(u.Path, "/") + + // Virtual hosted style: https://.object.storage..onstackit.cloud[/] + // The client always uses path style, so the bucket label is stripped from the host. + if m := virtualHostRegex.FindStringSubmatch(host); m != nil { + bucket := m[1] + if err := ValidateBucketName(bucket); err != nil { + return Reference{}, fmt.Errorf("invalid bucket in URL %q: %w", uri, err) + } + return Reference{ + Bucket: bucket, + Key: path, + Endpoint: u.Scheme + "://" + host[len(bucket)+1:], + Region: m[2], + }, nil + } + + // Path style: https://object.storage..onstackit.cloud/[/] + if m := pathStyleHostRegex.FindStringSubmatch(host); m != nil { + parts := strings.SplitN(path, "/", 2) + if parts[0] == "" { + return Reference{}, fmt.Errorf("missing bucket in URL %q", uri) + } + if err := ValidateBucketName(parts[0]); err != nil { + return Reference{}, fmt.Errorf("invalid bucket in URL %q: %w", uri, err) + } + key := "" + if len(parts) == 2 { + key = parts[1] + } + return Reference{ + Bucket: parts[0], + Key: key, + Endpoint: base, + Region: m[1], + }, nil + } + + return Reference{}, fmt.Errorf("%q is not a valid STACKIT Object Storage URL (expected https://BUCKET.object.storage.REGION.onstackit.cloud[/KEY] or https://object.storage.REGION.onstackit.cloud/BUCKET[/KEY])", uri) +} + +func ValidateBucketName(name string) error { + if len(name) < 3 || len(name) > 63 { + return fmt.Errorf("bucket name must be between 3 and 63 characters (got %d)", len(name)) + } + if !bucketNameRegex.MatchString(name) { + return fmt.Errorf("bucket name %q is invalid (must contain only lowercase letters, digits, dots and hyphens, and start and end with a lowercase letter or digit)", name) + } + if strings.Contains(name, "..") { + return fmt.Errorf("bucket name %q must not contain consecutive dots", name) + } + if strings.Contains(name, ".-") || strings.Contains(name, "-.") { + return fmt.Errorf("bucket name %q must not contain dots and hyphens next to each other", name) + } + if ip := net.ParseIP(name); ip != nil && ip.To4() != nil { + return fmt.Errorf("bucket name %q must not look like an IP address", name) + } + return nil +} diff --git a/internal/pkg/services/object-storage/bucket/uri_test.go b/internal/pkg/services/object-storage/bucket/uri_test.go new file mode 100644 index 000000000..5e1d4b6cb --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/uri_test.go @@ -0,0 +1,231 @@ +package bucket + +import ( + "testing" +) + +func TestParseBucketURI(t *testing.T) { + tests := []struct { + description string + uri string + expected Reference + isValid bool + }{ + { + description: "bucket URI", + uri: "s3://my-bucket", + expected: Reference{Bucket: "my-bucket"}, + isValid: true, + }, + { + description: "bucket URI with trailing slash", + uri: "s3://my-bucket/", + expected: Reference{Bucket: "my-bucket"}, + isValid: true, + }, + { + description: "object URI", + uri: "s3://my-bucket/key/to/object.txt", + expected: Reference{Bucket: "my-bucket", Key: "key/to/object.txt"}, + isValid: true, + }, + { + description: "prefix URI with trailing slash", + uri: "s3://my-bucket/prefix/", + expected: Reference{Bucket: "my-bucket", Key: "prefix/"}, + isValid: true, + }, + { + description: "object URI with leading slash in key", + uri: "s3://my-bucket//key", + expected: Reference{Bucket: "my-bucket", Key: "key"}, + isValid: true, + }, + { + description: "virtual hosted bucket URL", + uri: "https://my-bucket.object.storage.eu01.onstackit.cloud", + expected: Reference{ + Bucket: "my-bucket", + Endpoint: "https://object.storage.eu01.onstackit.cloud", + Region: "eu01", + }, + isValid: true, + }, + { + description: "virtual hosted bucket URL with trailing slash", + uri: "https://my-bucket.object.storage.eu01.onstackit.cloud/", + expected: Reference{ + Bucket: "my-bucket", + Endpoint: "https://object.storage.eu01.onstackit.cloud", + Region: "eu01", + }, + isValid: true, + }, + { + description: "virtual hosted object URL", + uri: "https://my-bucket.object.storage.eu01.onstackit.cloud/dir/file.txt", + expected: Reference{ + Bucket: "my-bucket", + Key: "dir/file.txt", + Endpoint: "https://object.storage.eu01.onstackit.cloud", + Region: "eu01", + }, + isValid: true, + }, + { + description: "virtual hosted bucket URL with dotted bucket name", + uri: "https://my.bucket.name.object.storage.de-fra.onstackit.cloud/key", + expected: Reference{ + Bucket: "my.bucket.name", + Key: "key", + Endpoint: "https://object.storage.de-fra.onstackit.cloud", + Region: "de-fra", + }, + isValid: true, + }, + { + description: "path style bucket URL", + uri: "https://object.storage.eu01.onstackit.cloud/my-bucket", + expected: Reference{ + Bucket: "my-bucket", + Endpoint: "https://object.storage.eu01.onstackit.cloud", + Region: "eu01", + }, + isValid: true, + }, + { + description: "path style object URL", + uri: "https://object.storage.eu01.onstackit.cloud/my-bucket/dir/file.txt", + expected: Reference{ + Bucket: "my-bucket", + Key: "dir/file.txt", + Endpoint: "https://object.storage.eu01.onstackit.cloud", + Region: "eu01", + }, + isValid: true, + }, + { + description: "wrong scheme", + uri: "ftp://my-bucket/key", + isValid: false, + }, + { + description: "unrecognized host", + uri: "https://example.com/my-bucket", + isValid: false, + }, + { + description: "path style URL without bucket", + uri: "https://object.storage.eu01.onstackit.cloud", + isValid: false, + }, + { + description: "missing bucket", + uri: "s3://", + isValid: false, + }, + { + description: "empty bucket", + uri: "s3:///key", + isValid: false, + }, + { + description: "invalid bucket name uppercase", + uri: "s3://My-Bucket", + isValid: false, + }, + { + description: "invalid bucket name too short", + uri: "s3://ab", + isValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + result, err := ParseBucketURI(tt.uri) + if (err != nil) != !tt.isValid { + t.Fatalf("ParseBucketURI(%q) error = %v, wantErr %v", tt.uri, err, !tt.isValid) + } + if tt.isValid && result != tt.expected { + t.Fatalf("ParseBucketURI(%q) = %+v, want %+v", tt.uri, result, tt.expected) + } + }) + } +} + +func TestReferenceString(t *testing.T) { + tests := []struct { + description string + reference Reference + expected string + }{ + { + description: "bucket", + reference: Reference{Bucket: "my-bucket"}, + expected: "s3://my-bucket", + }, + { + description: "object", + reference: Reference{Bucket: "my-bucket", Key: "key/to/object.txt"}, + expected: "s3://my-bucket/key/to/object.txt", + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + if result := tt.reference.String(); result != tt.expected { + t.Fatalf("String() = %q, want %q", result, tt.expected) + } + }) + } +} + +func TestIsBucketURI(t *testing.T) { + tests := []struct { + description string + value string + expected bool + }{ + {description: "valid s3 uri", value: "s3://bucket", expected: true}, + {description: "local path", value: "./local/file.txt", expected: false}, + {description: "empty", value: "", expected: false}, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + if result := IsBucketURI(tt.value); result != tt.expected { + t.Fatalf("IsBucketURI(%q) = %v, want %v", tt.value, result, tt.expected) + } + }) + } +} + +func TestValidateBucketName(t *testing.T) { + tests := []struct { + description string + name string + isValid bool + }{ + {description: "valid", name: "my-bucket", isValid: true}, + {description: "valid with dots", name: "my.bucket.name", isValid: true}, + {description: "valid with digits", name: "bucket-123", isValid: true}, + {description: "too short", name: "ab", isValid: false}, + {description: "too long", name: "a234567890123456789012345678901234567890123456789012345678901234", isValid: false}, + {description: "uppercase", name: "My-Bucket", isValid: false}, + {description: "starts with hyphen", name: "-bucket", isValid: false}, + {description: "ends with dot", name: "bucket.", isValid: false}, + {description: "consecutive dots", name: "bu..cket", isValid: false}, + {description: "dot and hyphen adjacent", name: "buc-ket.", isValid: false}, + {description: "looks like ip", name: "192.168.1.1", isValid: false}, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + err := ValidateBucketName(tt.name) + if (err != nil) != !tt.isValid { + t.Fatalf("ValidateBucketName(%q) error = %v, wantErr %v", tt.name, err, !tt.isValid) + } + }) + } +} diff --git a/internal/pkg/services/object-storage/bucket/xml.go b/internal/pkg/services/object-storage/bucket/xml.go new file mode 100644 index 000000000..f2e02d5c9 --- /dev/null +++ b/internal/pkg/services/object-storage/bucket/xml.go @@ -0,0 +1,65 @@ +package bucket + +import ( + "encoding/xml" + "time" +) + +type listAllMyBucketsResult struct { + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Buckets []bucketEntry `xml:"Buckets>Bucket"` +} + +type bucketEntry struct { + Name string `xml:"Name"` + CreationDate time.Time `xml:"CreationDate"` +} + +type listBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken"` + Contents []objectEntry `xml:"Contents"` + CommonPrefixes []commonPrefix `xml:"CommonPrefixes"` +} + +type objectEntry struct { + Key string `xml:"Key"` + Size int64 `xml:"Size"` + LastModified time.Time `xml:"LastModified"` +} + +type commonPrefix struct { + Prefix string `xml:"Prefix"` +} + +type deleteRequest struct { + XMLName xml.Name `xml:"Delete"` + Quiet bool `xml:"Quiet"` + Objects []deleteEntry `xml:"Object"` +} + +type deleteEntry struct { + Key string `xml:"Key"` +} + +type deleteResult struct { + XMLName xml.Name `xml:"DeleteResult"` + Deleted []deletedEntry `xml:"Deleted"` + Errors []deleteEntryErr `xml:"Error"` +} + +type deletedEntry struct { + Key string `xml:"Key"` +} + +type deleteEntryErr struct { + Key string `xml:"Key"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} + +type createBucketConfiguration struct { + XMLName xml.Name `xml:"CreateBucketConfiguration"` + LocationConstraint string `xml:"LocationConstraint"` +}