From 7195124d5ee175c2e39d7fe85476364f52b3a8e5 Mon Sep 17 00:00:00 2001 From: Lanusse Morais Date: Tue, 8 Sep 2026 15:56:39 -0300 Subject: [PATCH 1/3] feat(s3): add object storage command group and bump the Go SDK --- .goreleaser.yaml | 5 +- README.md | 78 +- cli/cli.go | 43 +- cli/help_topics.go | 123 +- cli/operating_systems.go | 2 +- cli/project_flag.go | 42 + cli/regions.go | 2 +- cli/server_wait.go | 13 +- cli/teams_members.go | 4 +- cmd/build_s3.go | 31 + cmd/build_storage_filesystems.go | 9 +- cmd/build_storage_objects.go | 25 - cmd/elasticips/create.go | 4 +- cmd/elasticips/elasticips_test.go | 4 +- cmd/firewalls/assignments.go | 14 +- cmd/firewalls/create.go | 6 +- cmd/firewalls/firewall.go | 15 +- cmd/firewalls/firewalls_test.go | 4 +- cmd/firewalls/list.go | 2 +- cmd/projectsshkeys/create.go | 4 +- cmd/projectsshkeys/projectsshkeys_test.go | 6 +- cmd/s3/accesskeys.go | 42 + cmd/s3/accesskeys_create.go | 554 +++++++ cmd/s3/accesskeys_create_test.go | 257 +++ cmd/s3/accesskeys_delete.go | 188 +++ cmd/s3/accesskeys_forget.go | 74 + cmd/s3/accesskeys_get.go | 197 +++ cmd/s3/accesskeys_import.go | 224 +++ cmd/s3/accesskeys_list.go | 316 ++++ cmd/s3/accesskeys_list_test.go | 241 +++ cmd/s3/accesskeys_rotate.go | 280 ++++ cmd/s3/accesskeys_shared.go | 964 +++++++++++ cmd/s3/accesskeys_shared_test.go | 457 ++++++ cmd/s3/accesskeys_test.go | 158 ++ cmd/s3/bucket_render.go | 118 ++ cmd/s3/configure.go | 959 +++++++++++ cmd/s3/configure_export.go | 375 +++++ cmd/s3/configure_export_test.go | 255 +++ cmd/s3/configure_test.go | 254 +++ cmd/s3/cp.go | 53 + cmd/s3/e2e_test.go | 791 ++++++++++ cmd/s3/lifecycle.go | 385 +++++ cmd/s3/lifecycle_create.go | 156 ++ cmd/s3/lifecycle_create_test.go | 109 ++ cmd/s3/lifecycle_delete.go | 133 ++ cmd/s3/lifecycle_get.go | 43 + cmd/s3/lifecycle_list.go | 49 + cmd/s3/lifecycle_test.go | 159 ++ cmd/s3/lifecycle_update.go | 210 +++ cmd/s3/lifecycle_update_test.go | 112 ++ cmd/s3/ls.go | 453 ++++++ cmd/s3/ls_test.go | 485 ++++++ cmd/s3/mb.go | 613 +++++++ cmd/s3/mb_test.go | 379 +++++ cmd/s3/metrics.go | 370 +++++ cmd/s3/metrics_test.go | 249 +++ cmd/s3/mv.go | 36 + cmd/s3/petname.go | 56 + cmd/s3/presign.go | 219 +++ cmd/s3/presign_test.go | 261 +++ cmd/s3/rb.go | 358 +++++ cmd/s3/rb_test.go | 309 ++++ cmd/s3/rm.go | 435 +++++ cmd/s3/rm_test.go | 662 ++++++++ cmd/s3/s3.go | 478 ++++++ cmd/s3/stat.go | 289 ++++ cmd/s3/stat_test.go | 284 ++++ cmd/s3/sync.go | 237 +++ cmd/s3/transfer.go | 1405 +++++++++++++++++ cmd/s3/transfer_test.go | 1194 ++++++++++++++ cmd/s3/usage.go | 489 ++++++ cmd/s3/usage_test.go | 411 +++++ cmd/servers/rescue.go | 13 +- cmd/servers/servers_test.go | 11 +- cmd/servers/wait.go | 11 +- cmd/sshkeys/create.go | 4 +- cmd/sshkeys/sshkeys_test.go | 6 +- cmd/sshkeys/update.go | 2 +- cmd/storage_filesystems/update.go | 2 +- cmd/storage_filesystems/update_test.go | 6 +- cmd/storage_objects/create.go | 134 -- cmd/storage_objects/create_test.go | 81 - cmd/storage_objects/delete.go | 62 - cmd/storage_objects/get.go | 53 - cmd/storage_objects/list.go | 66 - cmd/storage_objects/render_test.go | 86 - cmd/storage_objects/storage_objects.go | 96 -- cmd/tags/create.go | 4 +- cmd/virtualmachines/get.go | 2 +- cmd/virtualmachines/list.go | 2 +- cmd/virtualmachines/wait.go | 5 +- go.mod | 46 +- go.sum | 106 +- internal/config/objectstorage.go | 158 ++ internal/config/profile.go | 4 + internal/exitcode/exitcode.go | 89 ++ internal/objectstorage/apiclient.go | 93 ++ internal/objectstorage/apiclient_test.go | 21 + internal/objectstorage/client.go | 138 ++ internal/objectstorage/client_test.go | 138 ++ internal/objectstorage/creds.go | 330 ++++ internal/objectstorage/creds_test.go | 124 ++ internal/objectstorage/errors.go | 327 ++++ internal/objectstorage/filter.go | 120 ++ internal/objectstorage/filter_test.go | 64 + internal/objectstorage/output.go | 250 +++ internal/objectstorage/prompt.go | 228 +++ internal/objectstorage/prompt_test.go | 63 + internal/objectstorage/region.go | 60 + internal/objectstorage/region_test.go | 41 + internal/objectstorage/resolver.go | 506 ++++++ .../objectstorage/resolver_filter_test.go | 45 + internal/objectstorage/s3test/server.go | 808 ++++++++++ internal/objectstorage/storageclass.go | 39 + internal/objectstorage/uri.go | 197 +++ internal/objectstorage/uri_test.go | 105 ++ internal/output/table/utils.go | 43 +- internal/renderer/format.go | 15 +- internal/renderer/main.go | 6 +- internal/renderer/text.go | 132 ++ internal/renderer/text_test.go | 146 ++ internal/tui/confirm.go | 5 +- internal/tui/list.go | 7 +- internal/tui/multiselect.go | 145 ++ internal/tui/multiselect_test.go | 100 ++ internal/tui/numberinput.go | 5 +- internal/tui/textinput.go | 5 +- internal/wait/server.go | 22 +- internal/wait/status.go | 29 + internal/wait/vm.go | 19 +- internal/wait/wait_test.go | 28 +- main.go | 6 +- 132 files changed, 22891 insertions(+), 789 deletions(-) create mode 100644 cmd/build_s3.go delete mode 100644 cmd/build_storage_objects.go create mode 100644 cmd/s3/accesskeys.go create mode 100644 cmd/s3/accesskeys_create.go create mode 100644 cmd/s3/accesskeys_create_test.go create mode 100644 cmd/s3/accesskeys_delete.go create mode 100644 cmd/s3/accesskeys_forget.go create mode 100644 cmd/s3/accesskeys_get.go create mode 100644 cmd/s3/accesskeys_import.go create mode 100644 cmd/s3/accesskeys_list.go create mode 100644 cmd/s3/accesskeys_list_test.go create mode 100644 cmd/s3/accesskeys_rotate.go create mode 100644 cmd/s3/accesskeys_shared.go create mode 100644 cmd/s3/accesskeys_shared_test.go create mode 100644 cmd/s3/accesskeys_test.go create mode 100644 cmd/s3/bucket_render.go create mode 100644 cmd/s3/configure.go create mode 100644 cmd/s3/configure_export.go create mode 100644 cmd/s3/configure_export_test.go create mode 100644 cmd/s3/configure_test.go create mode 100644 cmd/s3/cp.go create mode 100644 cmd/s3/e2e_test.go create mode 100644 cmd/s3/lifecycle.go create mode 100644 cmd/s3/lifecycle_create.go create mode 100644 cmd/s3/lifecycle_create_test.go create mode 100644 cmd/s3/lifecycle_delete.go create mode 100644 cmd/s3/lifecycle_get.go create mode 100644 cmd/s3/lifecycle_list.go create mode 100644 cmd/s3/lifecycle_test.go create mode 100644 cmd/s3/lifecycle_update.go create mode 100644 cmd/s3/lifecycle_update_test.go create mode 100644 cmd/s3/ls.go create mode 100644 cmd/s3/ls_test.go create mode 100644 cmd/s3/mb.go create mode 100644 cmd/s3/mb_test.go create mode 100644 cmd/s3/metrics.go create mode 100644 cmd/s3/metrics_test.go create mode 100644 cmd/s3/mv.go create mode 100644 cmd/s3/petname.go create mode 100644 cmd/s3/presign.go create mode 100644 cmd/s3/presign_test.go create mode 100644 cmd/s3/rb.go create mode 100644 cmd/s3/rb_test.go create mode 100644 cmd/s3/rm.go create mode 100644 cmd/s3/rm_test.go create mode 100644 cmd/s3/s3.go create mode 100644 cmd/s3/stat.go create mode 100644 cmd/s3/stat_test.go create mode 100644 cmd/s3/sync.go create mode 100644 cmd/s3/transfer.go create mode 100644 cmd/s3/transfer_test.go create mode 100644 cmd/s3/usage.go create mode 100644 cmd/s3/usage_test.go delete mode 100644 cmd/storage_objects/create.go delete mode 100644 cmd/storage_objects/create_test.go delete mode 100644 cmd/storage_objects/delete.go delete mode 100644 cmd/storage_objects/get.go delete mode 100644 cmd/storage_objects/list.go delete mode 100644 cmd/storage_objects/render_test.go delete mode 100644 cmd/storage_objects/storage_objects.go create mode 100644 internal/config/objectstorage.go create mode 100644 internal/exitcode/exitcode.go create mode 100644 internal/objectstorage/apiclient.go create mode 100644 internal/objectstorage/apiclient_test.go create mode 100644 internal/objectstorage/client.go create mode 100644 internal/objectstorage/client_test.go create mode 100644 internal/objectstorage/creds.go create mode 100644 internal/objectstorage/creds_test.go create mode 100644 internal/objectstorage/errors.go create mode 100644 internal/objectstorage/filter.go create mode 100644 internal/objectstorage/filter_test.go create mode 100644 internal/objectstorage/output.go create mode 100644 internal/objectstorage/prompt.go create mode 100644 internal/objectstorage/prompt_test.go create mode 100644 internal/objectstorage/region.go create mode 100644 internal/objectstorage/region_test.go create mode 100644 internal/objectstorage/resolver.go create mode 100644 internal/objectstorage/resolver_filter_test.go create mode 100644 internal/objectstorage/s3test/server.go create mode 100644 internal/objectstorage/storageclass.go create mode 100644 internal/objectstorage/uri.go create mode 100644 internal/objectstorage/uri_test.go create mode 100644 internal/renderer/text.go create mode 100644 internal/renderer/text_test.go create mode 100644 internal/tui/multiselect.go create mode 100644 internal/tui/multiselect_test.go create mode 100644 internal/wait/status.go diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 612d290..b4c90d5 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -21,8 +21,11 @@ builds: - env: - CGO_ENABLED=0 + flags: + - -trimpath ldflags: - - -X "github.com/latitudesh/lsh/internal/version.Version={{ .Tag }}" + # -s -w strips symbol and DWARF tables (~30% smaller binaries). + - -s -w -X "github.com/latitudesh/lsh/internal/version.Version={{ .Tag }}" goos: - linux - windows diff --git a/README.md b/README.md index 7d28f26..368420d 100755 --- a/README.md +++ b/README.md @@ -132,6 +132,77 @@ sudo lsh volume mount --id vol_abc123 - The CLI automatically finds your credentials when you run commands with sudo - Volume mount needs sudo for nvme-cli installation and NVMe operations +`lsh volumes` and `lsh filesystems` are aliases of `lsh volume` and `lsh storage-filesystems`. + +## Object storage + +`lsh s3` manages buckets, objects, access keys and lifecycle rules, addressing +buckets and objects as `s3:///`. Bucket administration goes through the +Latitude API; object operations talk to the bucket's S3 endpoint directly, so +endpoint and signing region never have to be configured by hand. + +`` accepts the display name, the `bkt_` ID or the backend bucket name. If +the same display name exists in more than one place, the command lists the +candidates and you narrow it with `--project`, `-c`/`--storage-class`, `--site` +(e.g. `lsh s3 stat s3://backups -c high_performance --site TYO4`), or the `bkt_` ID. + +Create a bucket, upload, list and delete (in a terminal, `mb` offers to create +an S3 access key and saves it to your profile): + +```bash +lsh s3 mb s3://backups --region DAL --project +lsh s3 cp ./dump.sql s3://backups/2026/09/ +lsh s3 ls s3://backups/2026/09/ --human-readable --summarize +lsh s3 cp s3://backups/2026/09/dump.sql ./restore/ +lsh s3 rm s3://backups/2026/09/dump.sql +``` + +Give an application or CI job its own scoped access key (the secret is shown +once; `-o text --query` extracts it for a secret store): + +```bash +lsh s3 access-keys create --bucket backups=rw --bucket logs=readonly --name ci-deploy +lsh s3 access-keys create --bucket backups=rw --name ci-deploy -o text --query "[0].secret_access_key" | gh secret set LSH_S3_SECRET_ACCESS_KEY +lsh s3 access-keys list +lsh s3 access-keys rotate ci-deploy --delete-old +``` + +Expire objects automatically with lifecycle rules: + +```bash +lsh s3 lifecycle create s3://logs --prefix tmp/ --expiration-days 7 +lsh s3 lifecycle list s3://logs +lsh s3 lifecycle delete s3://logs expire-7d-tmp +``` + +Use the same buckets from rclone, mc, s3cmd or any other S3 client: + +```bash +lsh s3 configure export s3://backups --format env # also: aws | rclone | mc | s3cmd | process +``` + +Clean up safely (`--dry-run` only reads; multi-object deletes ask for +confirmation in a terminal and require `--yes` in CI): + +```bash +lsh s3 rm s3://logs/tmp/ --recursive --dry-run +lsh s3 rm s3://logs/tmp/ --recursive --yes +lsh s3 rb s3://logs --force --yes +``` + +In CI, object commands authenticate with an S3 access key from the environment +instead of a saved profile: + +| Variable | Purpose | +| --- | --- | +| `LSH_S3_ACCESS_KEY_ID` / `LSH_S3_SECRET_ACCESS_KEY` | S3 access key used by `cp`, `ls`, `rm`, `stat` and `presign` (both required). | +| `LSH_S3_ENDPOINT_URL` | Talk to this S3 endpoint without the API; buckets are then addressed by their backend `bucket_name`. | +| `LSH_S3_SIGNING_REGION` | SigV4 signing region when it cannot be derived from the endpoint. | +| `LSH_S3_USE_AWS_ENV` | Set to `1` to reuse `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`. | + +`lsh help exit-codes` documents the exit codes (0-7, 130) that `lsh s3` returns +for scripts. + ## Output formats & automation Every `list` command can render its results in different formats, so the output @@ -143,11 +214,12 @@ lsh servers list -o table # human-readable table (default) lsh servers list -o json # JSON lsh servers list -o yaml # YAML lsh servers list -o csv # CSV (header + one row per item) +lsh servers list -o text # raw values, tab-separated lsh servers list --json # shortcut for -o json ``` Filter the structured output with a [JMESPath](https://jmespath.org/) expression -via `--query` (works with json/yaml/csv): +via `--query` (works with json/yaml/csv/text): ```bash lsh servers list --query "[?status=='on'].id" -o json @@ -165,8 +237,8 @@ lsh servers list --no-paginate # first page only; next page printed to std | Variable | Purpose | | --- | --- | -| `LSH_OUTPUT` | Default output format (`table`/`json`/`yaml`/`csv`). Precedence: `--output` flag > `LSH_OUTPUT` > config file > default. | -| `LSH_CLASSIC_OUTPUT` | Set to `true` to force the legacy plain-ASCII table. An explicit `-o json/yaml/csv` still wins over it. | +| `LSH_OUTPUT` | Default output format (`table`/`json`/`yaml`/`csv`/`text`). Precedence: `--output` flag > `LSH_OUTPUT` > config file > default. | +| `LSH_CLASSIC_OUTPUT` | Set to `true` to force the legacy plain-ASCII table. An explicit `-o json/yaml/csv/text` still wins over it. | | `LATITUDESH_TOKEN` | API token; bypasses any stored profile (see `lsh help authentication`). | | `LSH_PROFILE` | Use the named profile for the command. | | `LSH_PROJECT` | Pre-fill `--project` so list commands don't prompt. | diff --git a/cli/cli.go b/cli/cli.go index b9f7c94..cf72234 100755 --- a/cli/cli.go +++ b/cli/cli.go @@ -8,6 +8,7 @@ import ( "github.com/latitudesh/lsh/client" "github.com/latitudesh/lsh/cmd/lsh" servers "github.com/latitudesh/lsh/cmd/servers" + "github.com/latitudesh/lsh/internal/exitcode" "github.com/latitudesh/lsh/internal/pagination" "github.com/latitudesh/lsh/internal/renderer" "github.com/latitudesh/lsh/internal/version" @@ -70,10 +71,14 @@ func MakeRootCmd(rootCmd *cobra.Command) (*cobra.Command, error) { // Dedicated group so help topics show up clearly in `lsh --help`. rootCmd.AddGroup(&cobra.Group{ID: helpTopicsGroupID, Title: "Help topics:"}) + // Storage products (object storage, filesystems, volumes) share a section + // so they stay discoverable together in `lsh --help`. + rootCmd.AddGroup(&cobra.Group{ID: StorageGroupID, Title: "Storage:"}) rootCmd.AddCommand(makeHelpAuthenticationCmd()) rootCmd.AddCommand(makeHelpProfilesCmd()) rootCmd.AddCommand(makeHelpAutomationCmd()) rootCmd.AddCommand(makeHelpOutputFormatsCmd()) + rootCmd.AddCommand(makeHelpExitCodesCmd()) // Re-resolve the active profile once flags have been parsed so that // `--profile ` overrides LSH_PROFILE / default_profile for the @@ -88,11 +93,20 @@ func MakeRootCmd(rootCmd *cobra.Command) (*cobra.Command, error) { // Validate output/query/pagination selection up front so commands fail // fast with an actionable message (and a non-zero exit) instead of // silently falling back or clamping. + // These are command-line errors. Groups that opted into the documented + // exit codes report them as usage (2); the older groups keep exiting 1 + // for every failure, so their scripts are unaffected. + usageCode := func(err error) error { + if err == nil || !usesExitCodes(cmd) { + return err + } + return exitcode.New(exitcode.Usage, err) + } if err := renderer.ValidateOutputSelection(); err != nil { - return err + return usageCode(err) } if err := pagination.Validate(); err != nil { - return err + return usageCode(err) } // Hydrate the active profile into viper for commands that authenticate // against the API. Skip the login/auth/profile subtree: there --profile @@ -119,7 +133,7 @@ func MakeRootCmd(rootCmd *cobra.Command) (*cobra.Command, error) { viper.BindPFlag("base_path", rootCmd.PersistentFlags().Lookup("base-path")) var outputFlag string - rootCmd.PersistentFlags().StringVarP(&outputFlag, "output", "o", "table", "output format: table | json | yaml | csv") + rootCmd.PersistentFlags().StringVarP(&outputFlag, "output", "o", "table", "output format: table | json | yaml | csv | text") viper.BindPFlag("output", rootCmd.PersistentFlags().Lookup("output")) // LSH_OUTPUT sets a per-user default format. viper precedence is // flag > env > config > default, which is exactly what PD-6072 requires. @@ -131,7 +145,7 @@ func MakeRootCmd(rootCmd *cobra.Command) (*cobra.Command, error) { // Global automation controls. --query post-processes structured output with // a JMESPath expression; the pagination flags govern every `list` command. - rootCmd.PersistentFlags().String("query", "", "filter json/yaml/csv output with a JMESPath expression (see 'lsh help output-formats')") + rootCmd.PersistentFlags().String("query", "", "filter json/yaml/csv/text output with a JMESPath expression (see 'lsh help output-formats')") viper.BindPFlag("query", rootCmd.PersistentFlags().Lookup("query")) rootCmd.PersistentFlags().Int64("page-size", pagination.DefaultPageSize, "items to request per API page") @@ -561,9 +575,11 @@ func makeOperationGroupVirtualNetworksCmd() (*cobra.Command, error) { func makeOperationGroupVolumeCmd() (*cobra.Command, error) { operationGroupVolumeCmd := &cobra.Command{ - Use: "volume", - Short: "Manage volumes", - Long: `Commands to manage volume operations such as listing, mounting, creating, and deleting volumes`, + Use: "volume", + Aliases: []string{"volumes"}, + GroupID: StorageGroupID, + Short: "Manage volumes", + Long: `Commands to manage volume operations such as listing, mounting, creating, and deleting volumes`, } operationVolumeListCmd, err := makeOperationVolumeListCmd() @@ -598,3 +614,16 @@ func makeOperationGroupVolumeCmd() (*cobra.Command, error) { return operationGroupVolumeCmd, nil } + +// usesExitCodes reports whether cmd belongs to a subtree that opted into the +// documented exit codes (see exitcode.OptInAnnotation). The annotation is set +// on every command of such a group, but the walk keeps it working if only the +// group root carries it. +func usesExitCodes(cmd *cobra.Command) bool { + for c := cmd; c != nil; c = c.Parent() { + if c.Annotations[exitcode.OptInAnnotation] == "true" { + return true + } + } + return false +} diff --git a/cli/help_topics.go b/cli/help_topics.go index 7e511c5..c323324 100644 --- a/cli/help_topics.go +++ b/cli/help_topics.go @@ -6,6 +6,12 @@ import "github.com/spf13/cobra" // commands under a dedicated section of `lsh --help`. const helpTopicsGroupID = "help-topics" +// StorageGroupID groups the storage command groups (s3, storage-filesystems, +// volume) under a "Storage:" heading in `lsh --help`. Commands are added in +// init() before MakeRootCmd registers the group, which Cobra allows as long as +// the group exists by the time Execute runs. +const StorageGroupID = "storage" + // newHelpTopic returns a Cobra command that exists purely to host // long-form documentation. Running it (with or without `help`) prints // the topic content; it has no subcommands and no side effects. @@ -124,6 +130,54 @@ error: "--project is required ...". The caller can recover by listing projects first and retrying: lsh --no-input projects list -o json + +Confirmations (--yes / --no-input) + +Destructive object storage commands (lsh s3 rm --recursive, rb --force, +access-keys delete, lifecycle delete --all) ask for confirmation when run +in a terminal. The contract for automation is: + + --yes skip the confirmation and proceed + --no-input never prompt; a command that would have asked fails with + exit 7 instead of hanging (stdin not being a TTY has the + same effect) + +Prompts and progress go to stderr; stdout carries only data, so '-o json' +and '-o text' remain pipeable. A declined or refused confirmation never +exits 0. See 'lsh help exit-codes' for the full table. + +Object storage credentials + +Object commands (lsh s3 cp/ls/rm/stat/presign) authenticate with an S3 +access key, not with the API token. In CI, export one of the following: + + LSH_S3_ACCESS_KEY_ID S3 access key id (both variables are required; + LSH_S3_SECRET_ACCESS_KEY setting only one fails with exit 4) + LSH_S3_ENDPOINT_URL talk to this S3 endpoint without the API; + buckets are then addressed by their backend + bucket_name (see 'lsh s3 configure export') + LSH_S3_SIGNING_REGION SigV4 signing region when it cannot be derived + from the endpoint + LSH_S3_USE_AWS_ENV=1 opt in to reuse AWS_ACCESS_KEY_ID and + AWS_SECRET_ACCESS_KEY (ignored when + AWS_SESSION_TOKEN is set) + + # Resolve s3:// through the API, sign with the env key + LATITUDESH_TOKEN=ak_xxx LSH_S3_ACCESS_KEY_ID=... LSH_S3_SECRET_ACCESS_KEY=... \ + lsh s3 cp ./dump.sql s3://backups/2026/09/ + + # No API at all: endpoint + backend bucket name + LSH_S3_ENDPOINT_URL=https://s3.us-central-1.storage.sh \ + LSH_S3_ACCESS_KEY_ID=... LSH_S3_SECRET_ACCESS_KEY=... \ + lsh s3 ls s3://backups-7f3a/ + + # Hand a scoped key to another job (the secret is printed once) + lsh s3 access-keys create --bucket backups=rw --name ci -o text \ + --query "[0].secret_access_key" | gh secret set LSH_S3_SECRET_ACCESS_KEY + +Precedence: LSH_S3_* environment > --access-key > the best +saved key of the active profile ('lsh s3 configure'). Without any of them +the command exits 4 with the commands that fix it. `, ) } @@ -131,7 +185,7 @@ projects first and retrying: func makeHelpOutputFormatsCmd() *cobra.Command { return newHelpTopic( "output-formats", - "Render results as table, JSON, YAML or CSV (with JMESPath queries)", + "Render results as table, JSON, YAML, CSV or text (with JMESPath queries)", `lsh — Output formats By default, lsh prints a human-readable table. Use --output (or -o) to switch @@ -141,6 +195,7 @@ to a machine-readable format for scripts, pipelines and AI agents: lsh servers list -o json # raw JSON lsh servers list -o yaml # YAML lsh servers list -o csv # CSV (header + one row per item) + lsh servers list -o text # raw values, tab-separated lsh servers list --json # shortcut for -o json # Set a per-user default without passing -o every time. @@ -149,13 +204,13 @@ to a machine-readable format for scripts, pipelines and AI agents: lsh servers list # prints JSON # Force the legacy plain-ASCII table (e.g. CI that parses fixed columns). - # An explicit -o json/yaml/csv still wins over this. + # An explicit -o json/yaml/csv/text still wins over this. LSH_CLASSIC_OUTPUT=true lsh servers list Filtering with --query (JMESPath) -The --query flag post-processes structured output (json/yaml/csv) with a -JMESPath expression — no jq or extra tooling required: +The --query flag post-processes structured output (json/yaml/csv/text) +with a JMESPath expression — no jq or extra tooling required: # Only the IDs of servers that are powered on lsh servers list --query '[?status==`+"`on`"+`].id' -o json @@ -163,7 +218,21 @@ JMESPath expression — no jq or extra tooling required: # A projection of selected fields lsh servers list --query '[].{id: id, host: hostname}' -o yaml ---query requires a structured format; combine it with -o json, yaml or csv. + # A single raw value, ready for another tool (no quotes, no JSON) + lsh s3 access-keys create --bucket backups --name ci -o text \ + --query "[0].secret_access_key" | gh secret set LSH_S3_SECRET_ACCESS_KEY + +--query requires a structured format; combine it with -o json, yaml, csv or +text. + +The text format + +-o text prints values without quotes or structure: +a scalar on one line; a list of scalars one per line; a list of objects as +one tab-separated row per item with keys in sorted order; a single object +as keyvalue lines. Nested values are JSON-encoded so a row never spans +several lines. Use it with --query to extract exactly one field for a shell +variable or a pipe. Pagination @@ -179,3 +248,47 @@ List commands fetch every page by default. These flags give you control: `, ) } + +func makeHelpExitCodesCmd() *cobra.Command { + return newHelpTopic( + "exit-codes", + "Process exit codes for scripts and CI", + `lsh — Exit codes + +The object storage commands ('lsh s3' and its subcommands) attach a +specific exit code to every failure so scripts can tell "not found" from +"no credentials" from "refused for safety" without parsing stderr. + + Code Meaning + ---- ----------------------------------------------------------- + 0 success (including an empty listing) + 1 generic error, or one or more transfers failed + 2 invalid usage: bad URI or flag, ambiguous bucket, + --recursive on a whole bucket without --all + 3 not found: bucket, object, access key or lifecycle rule + 4 credentials missing or invalid (no S3 access key, + InvalidAccessKeyId, SignatureDoesNotMatch) + 5 permission denied (403 from the API or the S3 endpoint) + 6 partial success: some objects failed in rm --recursive + or rb --force; the remaining ones are listed on stderr + 7 refused for safety: non-empty bucket without --force, + --max-delete exceeded, object lock retention, prompt + declined, or a confirmation needed without a TTY and + without --yes + 130 interrupted (Ctrl-C / SIGINT) after in-flight work was + aborted + +Errors are always printed to stderr; stdout carries only data, so +'-o json' and '-o text' output stays pipeable even when a command fails. + + lsh s3 cp ./dump.sql s3://backups/ || case $? in + 4) echo "configure an access key: lsh s3 configure" ;; + 7) echo "refused; add --yes in CI" ;; + esac + +Older command groups (servers, projects, plans, ...) predate this table +and still exit 1 for every error. Their behaviour is unchanged; only +'lsh s3' uses the codes above. +`, + ) +} diff --git a/cli/operating_systems.go b/cli/operating_systems.go index c9247c1..8ecf5e2 100644 --- a/cli/operating_systems.go +++ b/cli/operating_systems.go @@ -78,7 +78,7 @@ func runOperatingSystemsList(_ *cobra.Command, _ []string) error { pageSize := page.PageSize pageNumber := int64(1) - resp, err := client.OperatingSystems.ListPlans(ctx, &pageSize, &pageNumber) + resp, err := client.OperatingSystems.ListPlans(ctx, &pageSize, &pageNumber, nil) if err != nil { stopSpinner() utils.PrintError(err) diff --git a/cli/project_flag.go b/cli/project_flag.go index 8604391..d5d4ed7 100644 --- a/cli/project_flag.go +++ b/cli/project_flag.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/latitudesh/lsh/internal/exitcode" "github.com/latitudesh/lsh/internal/prompt" "github.com/latitudesh/lsh/internal/util" "github.com/spf13/cobra" @@ -86,3 +87,44 @@ func resolveProjectFlag(cmd *cobra.Command) error { } return cmd.Flags().Set("project", selected) } + +// PickProjectForList resolves the project scope for a "list" command that can +// also run across every project. Precedence: an explicit --project flag, +// LSH_PROJECT, --all-projects, then — in an interactive terminal — a project +// picker that includes an "All projects" entry. +// +// A non-interactive session (or --no-input) has no picker to show, so it keeps +// the behaviour these listings always had and covers every project. Failing +// there instead would break existing scripts (including the ones calling the +// legacy `storage-objects list`) for a prompt they could never have answered. +// +// It returns the chosen project (id or slug, empty when "all") and whether the +// user opted into all projects. The command owns the flags "project", +// "all-projects" and "no-input". +func PickProjectForList(cmd *cobra.Command) (project string, allProjects bool, err error) { + if v, _ := cmd.Flags().GetString("project"); v != "" { + return v, false, nil + } + if env := os.Getenv("LSH_PROJECT"); env != "" { + return env, false, nil + } + if all, _ := cmd.Flags().GetBool("all-projects"); all { + return "", true, nil + } + noInput, _ := cmd.Flags().GetBool("no-input") + if noInput || !isInteractive() { + return "", true, nil + } + token := viper.GetString("Authorization") + if token == "" { + return "", false, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login' first") + } + selected, err := prompt.SelectProject(cmd.Context(), newAuthClient(), token, true) + if err != nil { + return "", false, err + } + if selected == prompt.AllProjectsSentinel { + return "", true, nil + } + return selected, false, nil +} diff --git a/cli/regions.go b/cli/regions.go index b8924f5..4d5921b 100644 --- a/cli/regions.go +++ b/cli/regions.go @@ -77,7 +77,7 @@ func runRegionsList(_ *cobra.Command, _ []string) error { pageSize := page.PageSize pageNumber := int64(1) - resp, err := client.Regions.Get(ctx, &pageSize, &pageNumber) + resp, err := client.Regions.Get(ctx, operations.GetRegionsRequest{PageSize: &pageSize, PageNumber: &pageNumber}) if err != nil { stopSpinner() utils.PrintError(err) diff --git a/cli/server_wait.go b/cli/server_wait.go index 6b8dfcf..0a9b690 100644 --- a/cli/server_wait.go +++ b/cli/server_wait.go @@ -6,7 +6,6 @@ import ( "fmt" "os" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" "github.com/latitudesh/lsh/client/servers" "github.com/latitudesh/lsh/cmd/lsh" @@ -21,7 +20,7 @@ import ( // The create/reinstall calls still go through the legacy client; the wait loop // polls via the SDK (Servers.Get). Progress and outcome are written to stderr // so they never corrupt structured (-o json) output on stdout. -func waitForServerState(cmd *cobra.Command, serverID string, want, fail []components.ServerDataStatus) error { +func waitForServerState(cmd *cobra.Command, serverID string, want, fail []wait.ServerStatus) error { o := wait.OptionsFrom(cmd) if !o.Enabled { if cmd.Flags().Changed("timeout") { @@ -85,10 +84,10 @@ func renderServerState(cmd *cobra.Command, serverID string) { // Provisioning is "done" once the server settles into a stable power state — // it may finish either powered on or off — and "failed" on a failed deployment. // The in-progress states (deploying, disk_erasing) keep the wait polling. -func serverProvisionTargets() (want, fail []components.ServerDataStatus) { - return []components.ServerDataStatus{ - components.ServerDataStatusOn, - components.ServerDataStatusOff, +func serverProvisionTargets() (want, fail []wait.ServerStatus) { + return []wait.ServerStatus{ + wait.ServerStatusOn, + wait.ServerStatusOff, }, - []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} + []wait.ServerStatus{wait.ServerStatusFailedDeployment} } diff --git a/cli/teams_members.go b/cli/teams_members.go index 53f9139..b8d2af6 100644 --- a/cli/teams_members.go +++ b/cli/teams_members.go @@ -128,7 +128,7 @@ func runTeamMembersList(_ *cobra.Command, _ []string) error { pageSize := page.PageSize pageNumber := int64(1) - resp, err := client.Teams.Members.GetTeamMembers(ctx, &pageSize, &pageNumber) + resp, err := client.Teams.Members.GetTeamMembers(ctx, &pageSize, &pageNumber, nil) if err != nil { stopSpinner() utils.PrintError(err) @@ -339,7 +339,7 @@ func selectTeamMember(ctx context.Context, client *latitudeshgosdk.Latitudesh) ( // output-pagination flags (--max-items / --no-paginate), so it walks all // pages itself using the default page size. pageSize := pagination.DefaultPageSize - resp, err := client.Teams.Members.GetTeamMembers(ctx, &pageSize, nil) + resp, err := client.Teams.Members.GetTeamMembers(ctx, &pageSize, nil, nil) if err != nil { return teamMemberChoice{}, err } diff --git a/cmd/build_s3.go b/cmd/build_s3.go new file mode 100644 index 0000000..5e93163 --- /dev/null +++ b/cmd/build_s3.go @@ -0,0 +1,31 @@ +package cmd + +import ( + s3 "github.com/latitudesh/lsh/cmd/s3" +) + +// s3Cmd is the object storage group (`lsh s3`, aliases `buckets` and the +// legacy `storage-objects`). It replaces cmd/storage_objects. +var s3Cmd = s3.NewGroupCmd() + +func init() { + s3Cmd.AddCommand(s3.NewLsCmd()) + s3Cmd.AddCommand(s3.NewMbCmd()) + s3Cmd.AddCommand(s3.NewRbCmd()) + s3Cmd.AddCommand(s3.NewStatCmd()) + s3Cmd.AddCommand(s3.NewCpCmd()) + s3Cmd.AddCommand(s3.NewMvCmd()) + s3Cmd.AddCommand(s3.NewRmCmd()) + s3Cmd.AddCommand(s3.NewSyncCmd()) + s3Cmd.AddCommand(s3.NewPresignCmd()) + s3Cmd.AddCommand(s3.NewConfigureCmd()) + s3Cmd.AddCommand(s3.NewMetricsCmd()) + s3Cmd.AddCommand(s3.NewUsageCmd()) + s3Cmd.AddCommand(s3.NewAccessKeysCmd()) + s3Cmd.AddCommand(s3.NewLifecycleCmd()) + // Install the shared error/exit-code contract on the whole tree now that + // every subcommand (and its late-assigned RunE/PreRunE) is in place. + s3.Finalize(s3Cmd) + + rootCmd.AddCommand(s3Cmd) +} diff --git a/cmd/build_storage_filesystems.go b/cmd/build_storage_filesystems.go index 0e64617..078c66b 100644 --- a/cmd/build_storage_filesystems.go +++ b/cmd/build_storage_filesystems.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/latitudesh/lsh/cli" storage_filesystems "github.com/latitudesh/lsh/cmd/storage_filesystems" cobra "github.com/spf13/cobra" ) @@ -14,9 +15,11 @@ func init() { } var storageFilesystemsCmd = &cobra.Command{ - Use: "storage-filesystems", - Short: "Manage filesystem storage", - Long: "Manage filesystem storage: list, resize and delete filesystems.", + Use: "storage-filesystems", + Aliases: []string{"filesystems"}, + GroupID: cli.StorageGroupID, + Short: "Manage filesystem storage", + Long: "Manage filesystem storage: list, resize and delete filesystems.", Example: ` lsh storage-filesystems list --project my-project lsh storage-filesystems update fs_xxxxxxxx --size 3000 lsh storage-filesystems delete fs_xxxxxxxx`, diff --git a/cmd/build_storage_objects.go b/cmd/build_storage_objects.go deleted file mode 100644 index ae1f2cb..0000000 --- a/cmd/build_storage_objects.go +++ /dev/null @@ -1,25 +0,0 @@ -package cmd - -import ( - storage_objects "github.com/latitudesh/lsh/cmd/storage_objects" - cobra "github.com/spf13/cobra" -) - -func init() { - storageObjectsCmd.AddCommand(storage_objects.NewListCmd()) - storageObjectsCmd.AddCommand(storage_objects.NewGetCmd()) - storageObjectsCmd.AddCommand(storage_objects.NewCreateCmd()) - storageObjectsCmd.AddCommand(storage_objects.NewDeleteCmd()) - - rootCmd.AddCommand(storageObjectsCmd) -} - -var storageObjectsCmd = &cobra.Command{ - Use: "storage-objects", - Short: "Manage object storage buckets", - Long: "Manage object storage buckets: list, get, create and delete S3-compatible buckets.", - Example: ` lsh storage-objects list --project my-project - lsh storage-objects create --project my-project --name my-bucket --region SAO2 - lsh storage-objects get bucket_xxxxxxxx - lsh storage-objects delete bucket_xxxxxxxx`, -} diff --git a/cmd/elasticips/create.go b/cmd/elasticips/create.go index ded49c7..e191216 100644 --- a/cmd/elasticips/create.go +++ b/cmd/elasticips/create.go @@ -49,9 +49,9 @@ func buildCreateRequest(cmd *cobra.Command) (components.CreateElasticIP, error) request := components.CreateElasticIP{ Data: components.CreateElasticIPData{ Type: components.CreateElasticIPTypeElasticIps, - Attributes: components.CreateElasticIPAttributes{ + Attributes: &components.CreateElasticIPAttributes{ ProjectID: project, - ServerID: server, + ServerID: &server, }, }, } diff --git a/cmd/elasticips/elasticips_test.go b/cmd/elasticips/elasticips_test.go index 09ea094..e748165 100644 --- a/cmd/elasticips/elasticips_test.go +++ b/cmd/elasticips/elasticips_test.go @@ -71,8 +71,8 @@ func TestBuildCreateRequest(t *testing.T) { if request.Data.Attributes.ProjectID != "my-project" { t.Errorf("project_id = %q, want my-project", request.Data.Attributes.ProjectID) } - if request.Data.Attributes.ServerID != "sv_x" { - t.Errorf("server_id = %q, want sv_x", request.Data.Attributes.ServerID) + if request.Data.Attributes.ServerID == nil || *request.Data.Attributes.ServerID != "sv_x" { + t.Errorf("server_id = %v, want sv_x", request.Data.Attributes.ServerID) } } diff --git a/cmd/firewalls/assignments.go b/cmd/firewalls/assignments.go index bf0349d..f133bb4 100644 --- a/cmd/firewalls/assignments.go +++ b/cmd/firewalls/assignments.go @@ -91,7 +91,7 @@ func (o *ListAssignmentsOperation) run(cmd *cobra.Command, args []string) error value, _ := cmd.Flags().GetString("server") filterServer = &value } - response, err := client.Firewalls.GetAllFirewallAssignments(ctx, filterServer, nil, nil, operations.WithRetries(lsh.RetryConfig())) + response, err := client.Firewalls.GetAllFirewallAssignments(ctx, filterServer, nil, nil, nil, operations.WithRetries(lsh.RetryConfig())) if err != nil { utils.PrintError(err) return err @@ -149,7 +149,7 @@ func buildCreateAssignmentRequest(cmd *cobra.Command) (firewallID string, body o Data: operations.CreateFirewallAssignmentFirewallsAssignmentsData{ Type: operations.CreateFirewallAssignmentFirewallsAssignmentsTypeFirewallAssignments, Attributes: &operations.CreateFirewallAssignmentFirewallsAssignmentsAttributes{ - ServerID: serverID, + ServerID: &serverID, }, }, } @@ -189,11 +189,13 @@ func (o *CreateAssignmentOperation) run(cmd *cobra.Command, args []string) error // current SDK model does not map (its fields come back empty). Fall back // to the request inputs so the confirmation shows the association that // was created instead of a blank row. - if fs.Attributes == nil || getStr(fs.Attributes.FirewallID) == "" { + if fs.Data == nil || fs.Data.Attributes == nil || getStr(fs.Data.Attributes.FirewallID) == "" { serverID, _ := cmd.Flags().GetString("server") - fs.Attributes = &components.FirewallServerAttributes{ - FirewallID: &firewallID, - ServerID: &serverID, + fs.Data = &components.FirewallServerData{ + Attributes: &components.FirewallServerAttributes{ + FirewallID: &firewallID, + Server: &components.FirewallServerServer{ID: &serverID}, + }, } } assignment := FirewallServerAssignment{FirewallServer: fs} diff --git a/cmd/firewalls/create.go b/cmd/firewalls/create.go index 2371875..eae91e0 100644 --- a/cmd/firewalls/create.go +++ b/cmd/firewalls/create.go @@ -119,9 +119,9 @@ func buildCreateFirewallRequest(cmd *cobra.Command) (*operations.CreateFirewallF } request := operations.CreateFirewallFirewallsRequestBody{ - Data: operations.CreateFirewallData{ - Type: operations.CreateFirewallTypeFirewalls, - Attributes: &operations.CreateFirewallAttributes{ + Data: operations.CreateFirewallFirewallsData{ + Type: operations.CreateFirewallFirewallsTypeFirewalls, + Attributes: &operations.CreateFirewallFirewallsAttributes{ Name: name, Project: project, Rules: rules, diff --git a/cmd/firewalls/firewall.go b/cmd/firewalls/firewall.go index 87dc59f..2792bae 100644 --- a/cmd/firewalls/firewall.go +++ b/cmd/firewalls/firewall.go @@ -171,16 +171,21 @@ func (m *FirewallServerAssignment) GetData() []renderer.ResponseData { } func (m *FirewallServerAssignment) TableRow() table.Row { - var firewallID, serverID string - if attr := m.Attributes; attr != nil { - firewallID = getStr(attr.FirewallID) - serverID = getStr(attr.ServerID) + var id, firewallID, serverID string + if d := m.Data; d != nil { + id = getStr(d.ID) + if attr := d.Attributes; attr != nil { + firewallID = getStr(attr.FirewallID) + if attr.Server != nil { + serverID = getStr(attr.Server.ID) + } + } } return table.Row{ "id": table.Cell{ Label: "ID", - Value: table.String(getStr(m.ID)), + Value: table.String(id), }, "firewall_id": table.Cell{ Label: "Firewall ID", diff --git a/cmd/firewalls/firewalls_test.go b/cmd/firewalls/firewalls_test.go index 0501899..03086c3 100644 --- a/cmd/firewalls/firewalls_test.go +++ b/cmd/firewalls/firewalls_test.go @@ -165,7 +165,7 @@ func TestBuildCreateFirewallRequest(t *testing.T) { t.Fatalf("buildCreateFirewallRequest returned error: %v", err) } - if request.Data.Type != operations.CreateFirewallTypeFirewalls { + if request.Data.Type != operations.CreateFirewallFirewallsTypeFirewalls { t.Errorf("type = %v, want firewalls", request.Data.Type) } if request.Data.Attributes.Name != "web" { @@ -262,7 +262,7 @@ func TestBuildCreateAssignmentRequest(t *testing.T) { if body.Data.Type != operations.CreateFirewallAssignmentFirewallsAssignmentsTypeFirewallAssignments { t.Errorf("type = %v, want firewall_assignments", body.Data.Type) } - if body.Data.Attributes == nil || body.Data.Attributes.ServerID != "sv_x" { + if body.Data.Attributes == nil || body.Data.Attributes.ServerID == nil || *body.Data.Attributes.ServerID != "sv_x" { t.Errorf("server_id = %v, want sv_x", body.Data.Attributes) } } diff --git a/cmd/firewalls/list.go b/cmd/firewalls/list.go index ce86902..e59f328 100644 --- a/cmd/firewalls/list.go +++ b/cmd/firewalls/list.go @@ -46,7 +46,7 @@ func (o *ListFirewallsOperation) run(cmd *cobra.Command, args []string) error { filterProject = &value } - response, err := client.Firewalls.List(ctx, filterProject, nil, nil, operations.WithRetries(lsh.RetryConfig())) + response, err := client.Firewalls.List(ctx, filterProject, nil, nil, nil, operations.WithRetries(lsh.RetryConfig())) if err != nil { utils.PrintError(err) return err diff --git a/cmd/projectsshkeys/create.go b/cmd/projectsshkeys/create.go index 05f0f30..9254c46 100644 --- a/cmd/projectsshkeys/create.go +++ b/cmd/projectsshkeys/create.go @@ -54,8 +54,8 @@ func (o *CreateProjectSSHKeyOperation) run(cmd *cobra.Command, args []string) er Data: operations.PostProjectSSHKeyProjectsSSHKeysData{ Type: operations.PostProjectSSHKeyProjectsSSHKeysTypeSSHKeys, Attributes: &operations.PostProjectSSHKeyProjectsSSHKeysAttributes{ - Name: &name, - PublicKey: &publicKey, + Name: name, + PublicKey: publicKey, }, }, } diff --git a/cmd/projectsshkeys/projectsshkeys_test.go b/cmd/projectsshkeys/projectsshkeys_test.go index 823237e..b37656c 100644 --- a/cmd/projectsshkeys/projectsshkeys_test.go +++ b/cmd/projectsshkeys/projectsshkeys_test.go @@ -66,15 +66,15 @@ func TestCreateBuildsRequestBody(t *testing.T) { Data: operations.PostProjectSSHKeyProjectsSSHKeysData{ Type: operations.PostProjectSSHKeyProjectsSSHKeysTypeSSHKeys, Attributes: &operations.PostProjectSSHKeyProjectsSSHKeysAttributes{ - Name: &name, - PublicKey: &publicKey, + Name: name, + PublicKey: publicKey, }, }, } if request.Data.Type != "ssh_keys" { t.Errorf("type = %q, want ssh_keys", request.Data.Type) } - if *request.Data.Attributes.Name != name { + if request.Data.Attributes.Name != name { t.Error("name not wired through") } } diff --git a/cmd/s3/accesskeys.go b/cmd/s3/accesskeys.go new file mode 100644 index 0000000..29a6a25 --- /dev/null +++ b/cmd/s3/accesskeys.go @@ -0,0 +1,42 @@ +package s3 + +import ( + cobra "github.com/spf13/cobra" +) + +// NewAccessKeysCmd builds `lsh s3 access-keys` (alias `keys`): the S3 +// credentials of a project, managed through the Latitude API and optionally +// saved in the local profile so the data-plane commands pick them up. +func NewAccessKeysCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "access-keys", + Aliases: []string{"keys", "access-key"}, + Short: "Manage S3 access keys", + Long: `Manage the S3 access keys of your object storage buckets. + +Access keys are separate from your API token: the API creates them, but the +secret is returned once and never again. A key is either fullaccess (every +bucket of the project in that storage class, and site for high_performance) +or limited_access (specific buckets with rw or readonly permission). + +Keys can be saved in the active lsh profile (--save, or --save-as ); +'lsh s3 cp/ls/rm' then pick the least-privileged saved key that covers the +bucket automatically. +Saved keys never leave this machine except towards the bucket's endpoint.`, + Example: ` lsh s3 access-keys list + lsh s3 access-keys list --saved + lsh s3 access-keys create --bucket backups=rw --bucket logs=readonly --name ci-deploy + lsh s3 access-keys create --all-buckets --storage-class standard --project my-project --save + lsh s3 access-keys rotate ci-deploy --delete-old + lsh s3 access-keys delete ci-deploy --yes + echo "$SECRET" | lsh s3 access-keys import --name legacy --access-key-id AKIA... --project my-project`, + } + cmd.AddCommand(newAccessKeysListCmd()) + cmd.AddCommand(newAccessKeysGetCmd()) + cmd.AddCommand(newAccessKeysCreateCmd()) + cmd.AddCommand(newAccessKeysDeleteCmd()) + cmd.AddCommand(newAccessKeysRotateCmd()) + cmd.AddCommand(newAccessKeysImportCmd()) + cmd.AddCommand(newAccessKeysForgetCmd()) + return cmd +} diff --git a/cmd/s3/accesskeys_create.go b/cmd/s3/accesskeys_create.go new file mode 100644 index 0000000..8c1aae3 --- /dev/null +++ b/cmd/s3/accesskeys_create.go @@ -0,0 +1,554 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysCreateCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "create", + Aliases: []string{"new", "add"}, + Short: "Create an access key for specific buckets or for all buckets of a class", + Long: `Create an S3 access key through the Latitude API. + +Scope is either specific buckets (--bucket, see its help for the grammar) or +every bucket of the project in one storage class (--all-buckets). All --bucket +values must share storage class, project and, for high_performance, site: one +key cannot span backends. Project, class and site are inferred from the buckets +when possible. + +The secret is returned once by the API and cannot be retrieved again: + - human output without --save prints it in clear, once; + - --save stores the key in the active profile (under the key name, or the + --save-as name) and does not print the secret (add --show-secret to print + it too); a name already holding another key gets a -2, -3… suffix; + - -o json|yaml|csv includes secret_access_key only when -o was passed + explicitly or with --show-secret (never because of LSH_OUTPUT/config). + +--like copies class, site, project and scope from a saved key so a +new bucket can be added to an existing scope (the API has no key update).`, + Example: ` lsh s3 access-keys create --bucket backups=rw --bucket logs=readonly --name ci-deploy + lsh s3 access-keys create --bucket backups --save + lsh s3 access-keys create --all-buckets --storage-class standard --project my-project --name ops --save-as ops + lsh s3 access-keys create --all-buckets --storage-class high_performance --region TYO4 --project my-project + lsh s3 access-keys create --bucket backups=rw --name ci-deploy -o json | jq -r .secret_access_key + lsh s3 access-keys create --like ci-deploy --bucket new-bucket=rw --name ci-deploy-v2`, + Args: cobra.NoArgs, + RunE: runAccessKeysCreate, + }) + addProjectFlag(cmd, true, "project of the key (ID or slug); inferred from --bucket when omitted") + cmd.Flags().StringArray("bucket", nil, bucketSpecUsage) + cmd.Flags().Bool("all-buckets", false, "grant access to every bucket of the project in the storage class (fullaccess)") + cmd.Flags().StringP("storage-class", "c", "", "storage class of the key: standard or high_performance (inferred from --bucket)") + cmd.Flags().String("region", "", "site of the key (e.g. TYO4); required for high_performance when it cannot be inferred") + cmd.Flags().String("name", "", "key name (default lsh--); normalized server-side") + cmd.Flags().Bool("save", false, "save the key in the active profile under the key name (the secret is then not printed)") + cmd.Flags().String("save-as", "", "save the key in the active profile under this name (implies --save)") + cmd.Flags().Bool("show-secret", false, "print the secret even when saving or when -o comes from the environment") + cmd.Flags().String("like", "", "copy storage class, site, project and scope from this saved key") + return cmd +} + +// createOptions are the parsed flags of create (also used by rotate/mb). +type createOptions struct { + BucketSpecs []bucketSpec + AllBuckets bool + StorageClass string + Site string + Project string + Name string + Save bool + SaveName string + ShowSecret bool + Like string +} + +// createPlan is everything needed to call the API and print the result. +type createPlan struct { + Request accessKeyRequest + Buckets []scopedBucket // resolved --bucket values (empty for fullaccess) + Endpoint string + Signing string + // ProjectID is the bkt_ project ID to store with the key when known. + ProjectID string +} + +// parseCreateOptions reads the flags. +func parseCreateOptions(cmd *cobra.Command) (createOptions, error) { + var o createOptions + values, _ := cmd.Flags().GetStringArray("bucket") + specs, err := parseBucketSpecs(values) + if err != nil { + return o, err + } + o.BucketSpecs = specs + o.AllBuckets, _ = cmd.Flags().GetBool("all-buckets") + if o.StorageClass, err = storageClassFlag(cmd); err != nil { + return o, err + } + if o.Site, err = regionFlag(cmd); err != nil { + return o, err + } + o.Project = projectFlag(cmd) + o.Name, _ = cmd.Flags().GetString("name") + o.Name = strings.TrimSpace(o.Name) + if err := validateAccessKeyName(o.Name); err != nil { + return o, err + } + // --save is a plain bool (so --save=false never saves); --save-as + // picks the saved name and implies --save. + o.Save, _ = cmd.Flags().GetBool("save") + o.SaveName, _ = cmd.Flags().GetString("save-as") + o.SaveName = strings.TrimSpace(o.SaveName) + if o.SaveName != "" { + o.Save = true + } + o.ShowSecret, _ = cmd.Flags().GetBool("show-secret") + o.Like, _ = cmd.Flags().GetString("like") + return o, nil +} + +// applyLike fills the options from a saved key template. Explicit flags win. +func applyLike(o *createOptions, like config.StoredAccessKey) error { + if like.Scope == config.ScopeUnknown { + return exitcode.Errorf(exitcode.Usage, "saved key %q has unknown scope and cannot be used as a template", o.Like) + } + if o.StorageClass == "" { + o.StorageClass = like.StorageClass + } + if o.Site == "" { + o.Site = strings.ToUpper(like.Site) + } + if o.Project == "" { + o.Project = like.ProjectID + } + if !o.AllBuckets && len(o.BucketSpecs) == 0 && like.Scope == config.ScopeFullAccess { + o.AllBuckets = true + } + if like.Scope == config.ScopeLimitedAccess && !o.AllBuckets { + seen := map[string]bool{} + for _, s := range o.BucketSpecs { + seen[s.Token] = true + } + ids := make([]string, 0, len(like.Buckets)) + for id := range like.Buckets { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + if !seen[id] { + o.BucketSpecs = append(o.BucketSpecs, bucketSpec{Token: id, Permission: like.Buckets[id]}) + } + } + } + return nil +} + +// buildCreatePlan resolves the buckets, validates that they belong to one +// backend and infers project, class and site. +func buildCreatePlan(ctx context.Context, r *objectstorage.Resolver, o createOptions) (*createPlan, error) { + if o.AllBuckets && len(o.BucketSpecs) > 0 { + return nil, exitcode.Errorf(exitcode.Usage, "--all-buckets and --bucket are mutually exclusive") + } + if !o.AllBuckets && len(o.BucketSpecs) == 0 { + return nil, exitcode.Errorf(exitcode.Usage, "choose the scope: --bucket [=rw|readonly] (repeatable) or --all-buckets --storage-class ") + } + plan := &createPlan{Request: accessKeyRequest{Project: o.Project, StorageClass: o.StorageClass, Site: o.Site, Name: o.Name}} + + if len(o.BucketSpecs) > 0 { + for _, spec := range o.BucketSpecs { + b, err := r.Resolve(ctx, spec.Token) + if err != nil { + return nil, err + } + if b.EndpointOverride { + return nil, exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url") + } + if err := r.FillSite(ctx, b); err != nil { + if b.StorageClass == objectstorage.ClassHighPerformance && o.Site == "" { + return nil, err + } + lsh.LogDebugf("[s3] site lookup failed for %s: %v", b.ID, err) + } + plan.Buckets = append(plan.Buckets, scopedBucket{Bucket: b, Permission: spec.Permission}) + } + if err := validateSameGroup(plan.Buckets); err != nil { + return nil, err + } + first := plan.Buckets[0].Bucket + plan.ProjectID = first.ProjectID + if plan.Request.Project == "" { + plan.Request.Project = first.ProjectRef() + } + if plan.Request.StorageClass == "" { + plan.Request.StorageClass = first.StorageClass + } else if plan.Request.StorageClass != first.StorageClass { + return nil, exitcode.Errorf(exitcode.Usage, "--storage-class %s does not match the buckets (%s)", plan.Request.StorageClass, first.StorageClass) + } + if plan.Request.Site == "" { + plan.Request.Site = strings.ToUpper(first.Site) + } + plan.Request.Scope = config.ScopeLimitedAccess + plan.Request.Buckets = map[string]string{} + for _, sb := range plan.Buckets { + plan.Request.Buckets[sb.Bucket.ID] = sb.Permission + } + plan.Endpoint, plan.Signing = first.Endpoint, first.SigningRegion + } else { + plan.Request.Scope = config.ScopeFullAccess + if plan.Request.Project == "" { + return nil, exitcode.Errorf(exitcode.Usage, "--project is required with --all-buckets (the key covers every bucket of one project)") + } + if err := inferFromProject(ctx, r, plan); err != nil { + return nil, err + } + } + + if plan.Request.Project == "" { + return nil, exitcode.Errorf(exitcode.Usage, "could not infer the project; pass --project ") + } + if plan.Request.StorageClass == objectstorage.ClassHighPerformance && plan.Request.Site == "" { + return nil, exitcode.Errorf(exitcode.Usage, "--region is required for high_performance keys and could not be inferred; pass the site slug of the VAST cluster (e.g. TYO4)") + } + if plan.Request.Site == "" { + return nil, exitcode.Errorf(exitcode.Usage, "--region is required and could not be inferred from the buckets; pass a Latitude site slug (e.g. DAL)") + } + return plan, nil +} + +// inferFromProject fills class, site, endpoint and project ID for a +// fullaccess key from the project's buckets. +func inferFromProject(ctx context.Context, r *objectstorage.Resolver, plan *createPlan) error { + scoped := *r + scoped.Project = plan.Request.Project + list, err := scoped.ListBuckets(ctx) + if err != nil { + return err + } + var buckets []*objectstorage.Bucket + classes := map[string]bool{} + for _, d := range list { + b := objectstorage.BucketFromData(d) + if plan.ProjectID == "" { + plan.ProjectID = b.ProjectID + } + classes[b.StorageClass] = true + if plan.Request.StorageClass == "" || b.StorageClass == plan.Request.StorageClass { + buckets = append(buckets, b) + } + } + if plan.Request.StorageClass == "" { + switch len(classes) { + case 1: + for c := range classes { + plan.Request.StorageClass = c + } + case 0: + return exitcode.Errorf(exitcode.Usage, "project %s has no buckets yet; pass --storage-class standard|high_performance (and --region )", plan.Request.Project) + default: + return exitcode.Errorf(exitcode.Usage, "project %s has buckets of several storage classes; pass --storage-class standard or high_performance", plan.Request.Project) + } + buckets = buckets[:0] + for _, d := range list { + b := objectstorage.BucketFromData(d) + if b.StorageClass == plan.Request.StorageClass { + buckets = append(buckets, b) + } + } + } + // Sites come from the raw API (the SDK model drops them). + if len(buckets) > 0 { + sites, err := objectstorage.RawBucketSites(ctx, "") + if err != nil { + lsh.LogDebugf("[s3] site lookup failed: %v", err) + } + distinct := map[string]bool{} + for _, b := range buckets { + if b.Site == "" { + b.Site = sites[b.ID] + } + if b.Site != "" { + distinct[strings.ToUpper(b.Site)] = true + } + } + if plan.Request.Site == "" { + if plan.Request.StorageClass == objectstorage.ClassHighPerformance && len(distinct) > 1 { + names := make([]string, 0, len(distinct)) + for s := range distinct { + names = append(names, s) + } + sort.Strings(names) + return exitcode.Errorf(exitcode.Usage, "project %s has high_performance buckets in several sites (%s); pass --region ", plan.Request.Project, strings.Join(names, ", ")) + } + for _, b := range buckets { + if b.Site != "" { + plan.Request.Site = strings.ToUpper(b.Site) + break + } + } + } + for _, b := range buckets { + if plan.Request.StorageClass == objectstorage.ClassHighPerformance && !strings.EqualFold(b.Site, plan.Request.Site) { + continue + } + if b.Endpoint != "" { + plan.Endpoint, plan.Signing = b.Endpoint, b.SigningRegion + break + } + } + } + return nil +} + +// defaultCreateName generates a pet name for the plan's storage class, the +// same style the dashboard suggests (key---). +func defaultCreateName(cmd *cobra.Command, plan *createPlan) string { + _ = cmd + return generateKeyName(plan.Request.StorageClass) +} + +// bucketPermView is one bucket in the JSON output. +type bucketPermView struct { + BucketName string `json:"bucket_name"` + Name string `json:"name,omitempty"` + ID string `json:"id,omitempty"` + Permission string `json:"permission"` +} + +// createdKeyView is the normalized JSON shape of a created key. +type createdKeyView struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key,omitempty"` + Username string `json:"username,omitempty"` + Status string `json:"status,omitempty"` + Scope string `json:"scope"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Endpoint string `json:"endpoint,omitempty"` + SigningRegion string `json:"signing_region,omitempty"` + Buckets []bucketPermView `json:"buckets"` + SavedAs string `json:"saved_as,omitempty"` + Profile string `json:"profile,omitempty"` + DryRun bool `json:"dry_run,omitempty"` +} + +func newCreatedKeyView(k *createdAccessKey, plan *createPlan, showSecret bool) createdKeyView { + v := createdKeyView{ + Name: k.Name, AccessKeyID: k.AccessKeyID, Username: k.Username, Status: k.Status, Scope: k.Scope, + StorageClass: k.StorageClass, Site: k.Site, Project: k.Project, Endpoint: k.Endpoint, SigningRegion: k.SigningRegion, + Buckets: []bucketPermView{}, + } + if showSecret { + v.SecretAccessKey = k.SecretAccessKey + } + for _, sb := range plan.Buckets { + v.Buckets = append(v.Buckets, bucketPermView{BucketName: sb.Bucket.BucketName, Name: sb.Bucket.Name, ID: sb.Bucket.ID, Permission: sb.Permission}) + } + return v +} + +func (v createdKeyView) TableRow() table.Row { + status := "created" + if v.DryRun { + status = "dryrun" + } + return table.Row{ + "name": {Label: "Name", Value: v.Name}, + "access_key_id": {Label: "Access Key ID", Value: v.AccessKeyID}, + "scope": {Label: "Scope", Value: v.Scope}, + "storage_class": {Label: "Class", Value: v.StorageClass}, + "region": {Label: "Site", Value: dash(v.Site)}, + "project": {Label: "Project", Value: v.Project}, + "status": {Label: "Status", Value: status}, + } +} + +// bucketsLine renders "backups-7f3a (rw), logs-91aa (readonly)" or the +// fullaccess description. +func (v createdKeyView) bucketsLine() string { + if v.Scope == config.ScopeFullAccess { + s := fmt.Sprintf("all %s buckets of project %s", v.StorageClass, v.Project) + if v.Site != "" { + s += " in " + v.Site + } + return s + } + parts := make([]string, 0, len(v.Buckets)) + for _, b := range v.Buckets { + parts = append(parts, fmt.Sprintf("%s (%s)", firstNonEmptyStr(b.BucketName, b.Name, b.ID), b.Permission)) + } + return strings.Join(parts, ", ") +} + +// printCreatedHuman prints the J3 block. The secret line is only present +// when showSecret is true; it is never masked. +func printCreatedHuman(w io.Writer, v createdKeyView, showSecret bool) { + if v.DryRun { + fmt.Fprintf(w, "(dryrun) create access key %q (no API call made):\n", v.Name) + } else if showSecret { + fmt.Fprintf(w, "Access key %q created. The secret is shown once and cannot be retrieved again:\n", v.Name) + } else { + fmt.Fprintf(w, "Access key %q created.\n", v.Name) + } + if v.AccessKeyID != "" { + fmt.Fprintf(w, " Access Key ID: %s\n", v.AccessKeyID) + } + if showSecret && v.SecretAccessKey != "" { + fmt.Fprintf(w, " Secret Access Key: %s\n", v.SecretAccessKey) + } + if v.Endpoint != "" { + fmt.Fprintf(w, " Endpoint: %s\n", v.Endpoint) + } else { + fmt.Fprintf(w, " Endpoint: (see 'lsh s3 stat s3://' for the bucket endpoint)\n") + } + if v.SigningRegion != "" { + fmt.Fprintf(w, " Signing region: %s\n", v.SigningRegion) + } + fmt.Fprintf(w, " Class: %s\n", v.StorageClass) + if v.Site != "" { + fmt.Fprintf(w, " Site: %s\n", v.Site) + } + fmt.Fprintf(w, " Project: %s\n", v.Project) + fmt.Fprintf(w, " Scope: %s\n", v.Scope) + fmt.Fprintf(w, " Buckets: %s\n", v.bucketsLine()) + if v.SavedAs != "" { + fmt.Fprintf(w, " Saved as: %q in profile %s\n", v.SavedAs, v.Profile) + } +} + +func runAccessKeysCreate(cmd *cobra.Command, _ []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + o, err := parseCreateOptions(cmd) + if err != nil { + return printErr(err) + } + if o.Like != "" { + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + like, ok := p.ObjectStorageKeys()[o.Like] + if !ok { + return printErr(exitcode.Errorf(exitcode.NotFound, "no saved access key named %q in profile %s; run 'lsh s3 access-keys list --saved'", o.Like, profileName)) + } + if err := applyLike(&o, like); err != nil { + return printErr(err) + } + } + // Fail before creating anything if the key could not be saved afterwards: + // the secret would be lost. + profileName := "" + if o.Save { + _, name, _, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(exitcode.Errorf(exitcode.Of(err), "--save needs an active profile: %v", err)) + } + profileName = name + } + + r := newResolver(cmd) + if r.EndpointURL != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url")) + } + plan, err := buildCreatePlan(ctx, r, o) + if err != nil { + return printErr(err) + } + if plan.Request.Name == "" { + plan.Request.Name = defaultCreateName(cmd, plan) + } + saveName := o.SaveName + if o.Save && saveName == "" { + saveName = plan.Request.Name + } + + decision := decideSecretDisplay(isHuman(), o.Save, o.ShowSecret, outputExplicit(cmd)) + + if dryRun() { + k := &createdAccessKey{ + Name: plan.Request.Name, StorageClass: plan.Request.StorageClass, Site: plan.Request.Site, + Project: plan.Request.Project, Scope: plan.Request.Scope, Endpoint: plan.Endpoint, SigningRegion: plan.Signing, + } + v := newCreatedKeyView(k, plan, false) + v.DryRun = true + if o.Save { + v.SavedAs, v.Profile = saveName, profileName + } + if isHuman() { + printCreatedHuman(os.Stdout, v, false) + } else { + render([]renderer.ResponseData{v}) + } + return nil + } + + // The API name is generated whenever --name was not given; --save-as only + // names the local alias, so it must not disable the collision re-roll. + created, err := createAccessKeyRetrying(ctx, cmd, plan.Request, o.Name == "") + if err != nil { + return printErr(err) + } + // A re-roll changed the API name: the alias has to follow it, or it would + // keep pointing at the name of another live key. + if o.SaveName == "" && created.Name != "" && created.Name != saveName { + saveName = created.Name + } + created.Endpoint, created.SigningRegion = plan.Endpoint, plan.Signing + if len(plan.Buckets) > 0 { + created.Buckets = map[string]string{} + for _, sb := range plan.Buckets { + created.Buckets[sb.Bucket.Name] = sb.Permission + } + } + v := newCreatedKeyView(created, plan, decision.Show) + + if o.Save { + stored := created.stored(plan.ProjectID, plan.Request.Buckets, config.KeySourceCreate) + // saveNewKey never overwrites another key: a taken name gets -2, -3… + finalName, savedProfile, err := saveNewKey(cmd, saveName, stored) + if err != nil { + // "it was saved in the profile" would be a lie now. + decision.Warning = "" + // Removing the key is preferred over printing its secret; the + // helper only prints when it cannot remove it (or --show-secret + // asked for it), and always on stderr, never on stdout. + reportUnsavedKey(ctx, os.Stderr, plan.Request, created, plan, saveName, o.ShowSecret, err) + // The command's contract with --save is create *and* persist, and + // the key may have been removed again: exiting 0 here would tell + // automation the credential is in the profile. + return printErr(exitcode.Errorf(exitcode.Generic, "access key %q was created but not saved in the profile", saveName)) + } else { + v.SavedAs, v.Profile = finalName, savedProfile + if finalName != saveName { + objectstorage.Hintf("a saved key named %q already exists; saved as %q instead", saveName, finalName) + } + objectstorage.Hintf("Saved as %q in profile %s. The CLI uses it automatically for the buckets it covers.", finalName, savedProfile) + } + } + if decision.Warning != "" { + objectstorage.Warnf("%s", decision.Warning) + } + if isHuman() { + printCreatedHuman(os.Stdout, v, decision.Show) + return nil + } + render([]renderer.ResponseData{v}) + return nil +} diff --git a/cmd/s3/accesskeys_create_test.go b/cmd/s3/accesskeys_create_test.go new file mode 100644 index 0000000..c29d3d0 --- /dev/null +++ b/cmd/s3/accesskeys_create_test.go @@ -0,0 +1,257 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "regexp" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func TestCreateStoredRoundTrip(t *testing.T) { + home := withTempProfile(t, nil) + f := newFakeAPI(t) + api := f.client() + + req := accessKeyRequest{ + Project: "proj_1", StorageClass: objectstorage.ClassStandard, Site: "DAL", Name: "CI Deploy", + Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw", "bkt_2": "readonly"}, + } + created, err := api.create(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if created.AccessKeyID != "WASABIKEYID" || created.SecretAccessKey != "wasabi-secret-value" || created.Name != "ci deploy" { + t.Fatalf("unexpected created key: id=%s name=%s", created.AccessKeyID, created.Name) + } + + // The request body carries the expected JSON:API shape. + reqs := f.Requests() + if len(reqs) != 1 || reqs[0].Method != "POST" || reqs[0].Path != "/storage/access_keys" { + t.Fatalf("unexpected requests: %+v", reqs) + } + var body map[string]interface{} + if err := json.Unmarshal([]byte(reqs[0].Body), &body); err != nil { + t.Fatal(err) + } + attrs := body["data"].(map[string]interface{})["attributes"].(map[string]interface{}) + if attrs["access_scope"] != "limited_access" || attrs["region"] != "DAL" || attrs["storage_class"] != "standard" || attrs["project"] != "proj_1" { + t.Fatalf("unexpected attributes: %v", attrs) + } + perms := attrs["bucket_permissions"].([]interface{}) + if len(perms) != 2 || perms[0].(map[string]interface{})["bucket_id"] != "bkt_1" { + t.Fatalf("unexpected bucket_permissions: %v", perms) + } + + // Save into the temp profile and read it back through the same helpers. + stored := created.stored("proj_1", req.Buckets, config.KeySourceCreate) + profileName, err := objectstorage.SaveKey("", "ci-deploy", stored) + if err != nil || profileName != "test" { + t.Fatalf("SaveKey: profile=%q err=%v", profileName, err) + } + _, _, p, err := objectstorage.ActiveProfile("") + if err != nil { + t.Fatal(err) + } + got, ok := p.ObjectStorageKeys()["ci-deploy"] + if !ok { + t.Fatalf("key not saved: %v", p.ObjectStorageKeys()) + } + if got.AccessKeyID != "WASABIKEYID" || got.SecretAccessKey != "wasabi-secret-value" || got.Scope != config.ScopeLimitedAccess || + got.Buckets["bkt_2"] != "readonly" || got.Username != "someone+ci deploy@example.com" || got.Source != config.KeySourceCreate || + got.StorageClass != "standard" || got.ProjectID != "proj_1" || got.CreatedAt.IsZero() { + t.Fatalf("stored key mismatch: %s scope=%s buckets=%v", got, got.Scope, got.Buckets) + } + raw := readConfigRaw(t, home) + if !strings.Contains(raw, `"secret_access_key": "wasabi-secret-value"`) || !strings.Contains(raw, `"scope": "limited_access"`) { + t.Fatalf("config.json content unexpected:\n%s", raw) + } + + // The credential resolver now picks the key for the bucket it covers. + cred, err := objectstorage.ResolveCredential(&objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a", Endpoint: "https://s3.us-central-1.storage.sh", StorageClass: "standard", ProjectID: "proj_1"}, objectstorage.CredentialOptions{Write: true}) + if err != nil || cred.Name != "ci-deploy" || cred.Secret() != "wasabi-secret-value" { + t.Fatalf("saved key not selected: %v err=%v", cred, err) + } + + // ForgetKeyByID removes it again. + _, removed, err := objectstorage.ForgetKeyByID("", "WASABIKEYID") + if err != nil || len(removed) != 1 || removed[0] != "ci-deploy" { + t.Fatalf("ForgetKeyByID: %v err=%v", removed, err) + } + if strings.Contains(readConfigRaw(t, home), "wasabi-secret-value") { + t.Fatal("secret still on disk after forget") + } +} + +func TestCreateVASTShapeAndAPIErrors(t *testing.T) { + f := newFakeAPI(t) + f.createShape = "vast" + api := f.client() + req := accessKeyRequest{Project: "proj_1", StorageClass: objectstorage.ClassHighPerformance, Site: "TYO4", Name: "fast", Scope: config.ScopeFullAccess} + created, err := api.create(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if created.AccessKeyID != "VASTKEYID" || created.SecretAccessKey != "vast-secret-value" || created.Site != "TYO4" || created.Buckets != nil { + t.Fatalf("unexpected created key: %+v", created.stored("", nil, "")) + } + + // Validation errors never reach the API. + before := len(f.Requests()) + bad := req + bad.Site = "" + if _, err := api.create(context.Background(), bad); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("missing site must be a usage error, got %v", err) + } + if len(f.Requests()) != before { + t.Fatal("invalid request must not hit the API") + } +} + +func TestCreatedKeyViewSecretHandling(t *testing.T) { + k := &createdAccessKey{ + Name: "ci-deploy", AccessKeyID: "XL68DDURVGUUOULWPCAE", SecretAccessKey: "the-secret-value", StorageClass: "standard", Site: "DAL", + Project: "my-project", Scope: config.ScopeLimitedAccess, Endpoint: "https://s3.us-central-1.storage.sh", SigningRegion: "us-central-1", + } + plan := &createPlan{Buckets: []scopedBucket{ + {Bucket: &objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a"}, Permission: "rw"}, + {Bucket: &objectstorage.Bucket{ID: "bkt_2", Name: "logs", BucketName: "logs-91aa"}, Permission: "readonly"}, + }} + + // Shown once, in clear, with the J3 layout. + var buf bytes.Buffer + printCreatedHuman(&buf, newCreatedKeyView(k, plan, true), true) + out := buf.String() + for _, want := range []string{ + `Access key "ci-deploy" created. The secret is shown once and cannot be retrieved again:`, + "Access Key ID: XL68DDURVGUUOULWPCAE", + "Secret Access Key: the-secret-value", + "Endpoint: https://s3.us-central-1.storage.sh", + "Signing region: us-central-1", + "Site: DAL", + "Buckets: backups-7f3a (rw), logs-91aa (readonly)", + } { + if !strings.Contains(out, want) { + t.Errorf("human output missing %q:\n%s", want, out) + } + } + + // Omitted: no secret at all, not even a fragment. + buf.Reset() + v := newCreatedKeyView(k, plan, false) + printCreatedHuman(&buf, v, false) + if strings.Contains(buf.String(), "secret") || strings.Contains(buf.String(), "the-") || strings.Contains(buf.String(), "value") { + t.Fatalf("secret (or a fragment) leaked:\n%s", buf.String()) + } + b, _ := json.Marshal(v) + if strings.Contains(string(b), "secret") { + t.Fatalf("JSON must omit secret_access_key entirely: %s", b) + } + if !strings.Contains(string(b), `"buckets":[{"bucket_name":"backups-7f3a","name":"backups","id":"bkt_1","permission":"rw"}`) { + t.Fatalf("JSON buckets shape unexpected: %s", b) + } + b, _ = json.Marshal(newCreatedKeyView(k, plan, true)) + if !strings.Contains(string(b), `"secret_access_key":"the-secret-value"`) || !strings.Contains(string(b), `"signing_region":"us-central-1"`) { + t.Fatalf("explicit JSON must include the secret and endpoint data: %s", b) + } + + // Fullaccess keys describe their coverage instead of listing buckets. + full := *k + full.Scope = config.ScopeFullAccess + fv := newCreatedKeyView(&full, &createPlan{}, false) + if got := fv.bucketsLine(); got != "all standard buckets of project my-project in DAL" { + t.Fatalf("fullaccess buckets line = %q", got) + } +} + +func TestApplyLike(t *testing.T) { + like := config.StoredAccessKey{StorageClass: "standard", Site: "", ProjectID: "proj_1", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw", "bkt_2": "readonly"}} + o := createOptions{Like: "ci", BucketSpecs: []bucketSpec{{Token: "bkt_3", Permission: "rw"}}} + if err := applyLike(&o, like); err != nil { + t.Fatal(err) + } + if o.Project != "proj_1" || o.StorageClass != "standard" || len(o.BucketSpecs) != 3 || o.BucketSpecs[1].Token != "bkt_1" || o.BucketSpecs[2].Permission != "readonly" { + t.Fatalf("applyLike merged wrongly: %+v", o) + } + full := config.StoredAccessKey{StorageClass: "high_performance", Site: "tyo4", ProjectID: "proj_1", Scope: config.ScopeFullAccess} + o = createOptions{Like: "hp"} + if err := applyLike(&o, full); err != nil || !o.AllBuckets || o.Site != "TYO4" { + t.Fatalf("fullaccess template must set --all-buckets and the site: %+v err=%v", o, err) + } + if err := applyLike(&createOptions{Like: "u"}, config.StoredAccessKey{Scope: config.ScopeUnknown}); err == nil { + t.Fatal("unknown scope template must fail") + } +} + +func TestBuildCreatePlanScopeErrors(t *testing.T) { + r := &objectstorage.Resolver{} + if _, err := buildCreatePlan(context.Background(), r, createOptions{}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("no scope must be a usage error, got %v", err) + } + if _, err := buildCreatePlan(context.Background(), r, createOptions{AllBuckets: true, BucketSpecs: []bucketSpec{{Token: "a"}}}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("both scopes must be a usage error, got %v", err) + } + if _, err := buildCreatePlan(context.Background(), r, createOptions{AllBuckets: true, StorageClass: "standard"}); err == nil || !strings.Contains(err.Error(), "--project") { + t.Fatalf("--all-buckets without project must ask for --project, got %v", err) + } +} + +// TestParseCreateOptionsSaveFlags covers F27: --save is a plain bool, +// --save-as implies --save, and --save=false never saves. +func TestParseCreateOptionsSaveFlags(t *testing.T) { + parse := func(args ...string) createOptions { + t.Helper() + cmd := newAccessKeysCreateCmd() + if err := cmd.ParseFlags(args); err != nil { + t.Fatalf("ParseFlags(%v): %v", args, err) + } + o, err := parseCreateOptions(cmd) + if err != nil { + t.Fatalf("parseCreateOptions(%v): %v", args, err) + } + return o + } + if o := parse("--bucket", "backups"); o.Save || o.SaveName != "" { + t.Fatalf("default must not save: %+v", o) + } + if o := parse("--bucket", "backups", "--save"); !o.Save || o.SaveName != "" { + t.Fatalf("--save must save under the key name: %+v", o) + } + if o := parse("--bucket", "backups", "--save=false"); o.Save { + t.Fatalf("--save=false must not save: %+v", o) + } + if o := parse("--bucket", "backups", "--save-as", "ops"); !o.Save || o.SaveName != "ops" { + t.Fatalf("--save-as must imply --save with that name: %+v", o) + } + if o := parse("--bucket", "backups", "--save", "--save-as", " ops "); !o.Save || o.SaveName != "ops" { + t.Fatalf("--save --save-as must trim the name: %+v", o) + } + cmd := newAccessKeysCreateCmd() + if err := cmd.ParseFlags([]string{"--save=ops"}); err == nil { + t.Fatal("--save= is no longer accepted; --save-as replaces it") + } +} + +func TestGenerateKeyName(t *testing.T) { + re := regexp.MustCompile(`^key-[a-z]+-[a-z]+-(std|hp)$`) + for _, class := range []string{"standard", "high_performance", ""} { + got := generateKeyName(class) + if !re.MatchString(got) { + t.Errorf("generateKeyName(%q) = %q, want key---", class, got) + } + if len(got) < 3 || len(got) > maxAccessKeyNameLen { + t.Errorf("generateKeyName(%q) = %q, length %d out of 3..%d", class, got, len(got), maxAccessKeyNameLen) + } + if err := validateAccessKeyName(got); err != nil { + t.Errorf("generated name %q rejected by validator: %v", got, err) + } + } + if tierAbbr("high_performance") != "hp" || tierAbbr("standard") != "std" || tierAbbr("") != "std" { + t.Error("tierAbbr mapping wrong") + } +} diff --git a/cmd/s3/accesskeys_delete.go b/cmd/s3/accesskeys_delete.go new file mode 100644 index 0000000..c265b5b --- /dev/null +++ b/cmd/s3/accesskeys_delete.go @@ -0,0 +1,188 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysDeleteCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "delete ", + Aliases: []string{"rm", "revoke"}, + Short: "Delete an access key on the API and forget it locally", + Long: `Delete an access key. The key is looked up by name, access key ID or username +in the API (every project unless --project is given) and among the saved keys. +When several keys match (same name in two classes, sites or projects) the +command lists them and asks for --storage-class, --region or --project. + +The key is revoked on the API and removed from the active profile. Anything +still using it stops working immediately.`, + Example: ` lsh s3 access-keys delete ci-deploy + lsh s3 access-keys delete XL68DDURVGUUOULWPCAE --yes + lsh s3 access-keys delete backup --storage-class high_performance --region TYO4 --project my-project`, + Args: cobra.ExactArgs(1), + RunE: runAccessKeysDelete, + }) + addProjectFlag(cmd, true, "project of the key (ID or slug); every project is searched when omitted") + cmd.Flags().StringP("storage-class", "c", "", "disambiguate by storage class: standard or high_performance") + cmd.Flags().String("region", "", "disambiguate by site (e.g. TYO4); required for high_performance keys when unknown") + addYesFlag(cmd) + return cmd +} + +// deleteResult is the structured output of delete. +type deleteKeyResult struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + Username string `json:"username,omitempty"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Deleted bool `json:"deleted"` + Forgotten []string `json:"forgotten,omitempty"` + DryRun bool `json:"dry_run,omitempty"` +} + +func (d deleteKeyResult) TableRow() table.Row { + status := "deleted" + if d.DryRun { + status = "dryrun" + } + return table.Row{ + "name": {Label: "Name", Value: d.Name}, + "access_key_id": {Label: "Access Key ID", Value: d.AccessKeyID}, + "storage_class": {Label: "Class", Value: d.StorageClass}, + "region": {Label: "Site", Value: dash(d.Site)}, + "project": {Label: "Project", Value: d.Project}, + "status": {Label: "Status", Value: status}, + } +} + +// resolveDeleteTarget picks the single key token refers to among the API +// keys and the saved keys. Saved keys resolve a name to an access key ID; +// when the API does not list the key, the saved metadata is used as target +// (the API still needs the username). +func resolveDeleteTarget(token string, keys []apiKey, saved map[string]config.StoredAccessKey, filter listFilter) (apiKey, error) { + matches := findAPIKeys(token, keys, filter) + savedName, savedKey, hasSaved := findSavedKey(token, saved) + if len(matches) == 0 && hasSaved && savedKey.AccessKeyID != "" { + matches = findAPIKeys(savedKey.AccessKeyID, keys, filter) + } + switch len(matches) { + case 1: + return filter.withSite(matches[0]), nil + case 0: + if hasSaved && filter.matchSaved(savedKey) { + k := apiKey{ + Name: savedName, Username: savedKey.Username, AccessKeyID: savedKey.AccessKeyID, + StorageClass: savedKey.StorageClass, Site: savedKey.Site, Project: firstNonEmptyStr(filter.Project, savedKey.ProjectID), + Access: savedKey.Scope, + } + if filter.Site != "" { + k.Site = filter.Site + } + return k, nil + } + return apiKey{}, exitcode.Errorf(exitcode.NotFound, "access key %q not found; run 'lsh s3 access-keys list' (add --project to search one project)", token) + } + return apiKey{}, ambiguousKeys(token, matches) +} + +func runAccessKeysDelete(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + token := strings.TrimSpace(args[0]) + + class, err := storageClassFlag(cmd) + if err != nil { + return printErr(err) + } + site, err := regionFlag(cmd) + if err != nil { + return printErr(err) + } + yes, _ := cmd.Flags().GetBool(flagYes) + filter := listFilter{StorageClass: class, Site: site, Project: projectFlag(cmd)} + _, _, savedKeys := savedKeysByID(cmd) + _, savedKey, hasSaved := findSavedKey(token, savedKeys) + + r := newResolver(cmd) + if r.EndpointURL != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url")) + } + api := newKeysAPI() + projects := []string{filter.Project} + if filter.Project == "" { + if hasSaved && savedKey.ProjectID != "" { + projects = []string{savedKey.ProjectID} + } else { + projects, _, err = teamProjects(ctx, r) + if err != nil { + return printErr(err) + } + } + } + keys, err := api.listProjects(ctx, projects) + if err != nil { + return printErr(err) + } + target, err := resolveDeleteTarget(token, keys, savedKeys, filter) + if err != nil { + return printErr(err) + } + if target.StorageClass == "" { + return printErr(exitcode.Errorf(exitcode.Usage, "the storage class of %q is unknown; pass --storage-class standard|high_performance", token)) + } + + res := deleteKeyResult{ + Name: target.Name, AccessKeyID: target.AccessKeyID, Username: target.Username, + StorageClass: target.StorageClass, Site: strings.ToUpper(target.Site), Project: target.Project, DryRun: dryRun(), + } + objectstorage.Hintf("access key %q id=%s class=%s site=%s project=%s", dash(res.Name), res.AccessKeyID, res.StorageClass, dash(res.Site), res.Project) + + if !dryRun() { + if err := objectstorage.ConfirmOrRefuse(cmd, yes, fmt.Sprintf("Delete access key %q (%s)? Anything using it stops working", dash(res.Name), res.AccessKeyID)); err != nil { + return printErr(err) + } + if err := api.delete(ctx, target.Username, target.StorageClass, target.Project, target.Site); err != nil { + return printErr(err) + } + res.Deleted = true + profileName, removed, err := objectstorage.ForgetKeyByID(profileFlag(cmd), target.AccessKeyID) + if err != nil { + lsh.LogDebugf("[s3] could not update the profile after deleting %s: %v", target.AccessKeyID, err) + } + res.Forgotten = removed + if len(removed) > 0 { + objectstorage.Hintf("removed saved key(s) %s from profile %s", strings.Join(quoteAll(removed), ", "), profileName) + } + } + + if isHuman() { + prefix := "" + if res.DryRun { + prefix = "(dryrun) " + } + fmt.Printf("%sdelete_access_key: %s (%s)\n", prefix, dash(res.Name), res.AccessKeyID) + return nil + } + render([]renderer.ResponseData{res}) + return nil +} + +func quoteAll(values []string) []string { + out := make([]string, 0, len(values)) + for _, v := range values { + out = append(out, fmt.Sprintf("%q", v)) + } + return out +} diff --git a/cmd/s3/accesskeys_forget.go b/cmd/s3/accesskeys_forget.go new file mode 100644 index 0000000..b6fd273 --- /dev/null +++ b/cmd/s3/accesskeys_forget.go @@ -0,0 +1,74 @@ +package s3 + +import ( + "fmt" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysForgetCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "forget ", + Aliases: []string{"unsave"}, + Short: "Remove a saved access key from this profile (the key stays valid on the API)", + Long: `Remove an access key from the active profile only. The key keeps working for +whoever else has it; use 'lsh s3 access-keys delete' to revoke it on the API.`, + Example: ` lsh s3 access-keys forget ci-deploy + lsh s3 access-keys forget ci-deploy --profile acme`, + Args: cobra.ExactArgs(1), + RunE: runAccessKeysForget, + }) + return cmd +} + +// forgetResult is the structured output of forget. +type forgetResult struct { + Name string `json:"name"` + Profile string `json:"profile"` + Removed bool `json:"removed"` + DryRun bool `json:"dry_run,omitempty"` +} + +func (f forgetResult) TableRow() table.Row { + status := "removed" + if f.DryRun { + status = "dryrun" + } + return table.Row{ + "name": {Label: "Name", Value: f.Name}, + "profile": {Label: "Profile", Value: f.Profile}, + "status": {Label: "Status", Value: status}, + } +} + +func runAccessKeysForget(cmd *cobra.Command, args []string) error { + name := strings.TrimSpace(args[0]) + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + if _, ok := p.ObjectStorageKeys()[name]; !ok { + return printErr(exitcode.Errorf(exitcode.NotFound, "no saved access key named %q in profile %s; run 'lsh s3 access-keys list --saved'", name, profileName)) + } + res := forgetResult{Name: name, Profile: profileName, Removed: true, DryRun: dryRun()} + if !dryRun() { + if _, _, err := objectstorage.ForgetKey(profileFlag(cmd), name); err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + } + if isHuman() { + prefix := "" + if res.DryRun { + prefix = "(dryrun) " + } + fmt.Printf("%sforget: access key %q removed from profile %s\n", prefix, name, profileName) + return nil + } + render([]renderer.ResponseData{res}) + return nil +} diff --git a/cmd/s3/accesskeys_get.go b/cmd/s3/accesskeys_get.go new file mode 100644 index 0000000..c58f50a --- /dev/null +++ b/cmd/s3/accesskeys_get.go @@ -0,0 +1,197 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysGetCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "get ", + Aliases: []string{"show", "describe"}, + Short: "Show one access key", + Long: `Show one access key by name, access key ID or username. The API is searched +first (every project unless --project is given), then the keys saved in the +active profile. The secret is never shown: it only exists in the create output.`, + Example: ` lsh s3 access-keys get ci-deploy + lsh s3 access-keys get XL68DDURVGUUOULWPCAE -o json`, + Args: cobra.ExactArgs(1), + RunE: runAccessKeysGet, + }) + addProjectFlag(cmd, true, "search this project only (ID or slug)") + cmd.Flags().StringP("storage-class", "c", "", "disambiguate by storage class: standard or high_performance") + cmd.Flags().String("region", "", "disambiguate by site (e.g. DAL, TYO4)") + return cmd +} + +// findAPIKeys returns the API keys matching token by name, access key ID or +// username, narrowed by the filter. +func findAPIKeys(token string, keys []apiKey, filter listFilter) []apiKey { + var out []apiKey + for _, k := range keys { + if !filter.matchAPI(k) { + continue + } + if k.AccessKeyID == token || k.Username == token || strings.EqualFold(k.Name, token) { + out = append(out, k) + } + } + return out +} + +// findSavedKey looks a saved key up by name or access key ID. +func findSavedKey(token string, keys map[string]config.StoredAccessKey) (string, config.StoredAccessKey, bool) { + if k, ok := keys[token]; ok { + return token, k, true + } + for name, k := range keys { + if k.AccessKeyID == token { + return name, k, true + } + } + return "", config.StoredAccessKey{}, false +} + +// ambiguousKeys formats the exit-2 error for several matching keys. +func ambiguousKeys(token string, keys []apiKey) error { + lines := make([]string, 0, len(keys)) + for _, k := range keys { + lines = append(lines, fmt.Sprintf(" %-22s class=%s site=%s project=%s", k.AccessKeyID, k.StorageClass, dash(strings.ToUpper(k.Site)), k.Project)) + } + return exitcode.Errorf(exitcode.Usage, "%q matches %d access keys:\n%s\ndisambiguate with the access key ID or --storage-class / --region / --project", token, len(keys), strings.Join(lines, "\n")) +} + +func runAccessKeysGet(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + token := strings.TrimSpace(args[0]) + + class, err := storageClassFlag(cmd) + if err != nil { + return printErr(err) + } + site, err := regionFlag(cmd) + if err != nil { + return printErr(err) + } + filter := listFilter{StorageClass: class, Site: site, Project: projectFlag(cmd)} + savedByID, profileName, savedKeys := savedKeysByID(cmd) + + // Saved keys let a name or ID resolve to a project even without --project. + savedName, savedKey, hasSaved := findSavedKey(token, savedKeys) + + r := newResolver(cmd) + api := newKeysAPI() + projects := []string{filter.Project} + slugs := map[string]string{} + if filter.Project == "" { + if hasSaved && savedKey.ProjectID != "" { + projects = []string{savedKey.ProjectID} + } else { + projects, slugs, err = teamProjects(ctx, r) + if err != nil { + return printErr(err) + } + } + } + keys, listErr := api.listProjects(ctx, projects) + if listErr != nil && !hasSaved { + return printErr(listErr) + } + if listErr != nil { + lsh.LogDebugf("[s3] API listing failed, falling back to the saved key: %v", listErr) + } + + matches := findAPIKeys(token, keys, filter) + if len(matches) == 0 && hasSaved && savedKey.AccessKeyID != "" { + matches = findAPIKeys(savedKey.AccessKeyID, keys, filter) + } + switch { + case len(matches) > 1: + return printErr(ambiguousKeys(token, matches)) + case len(matches) == 1: + row := newKeyRow(filter.withSite(matches[0]), savedByID, slugs) + if isHuman() { + printKeyDetails(os.Stdout, row) + return nil + } + render([]renderer.ResponseData{row}) + return nil + } + + if !hasSaved { + return printErr(exitcode.Errorf(exitcode.NotFound, "access key %q not found in the API nor in profile %s; run 'lsh s3 access-keys list'", token, dash(profileName))) + } + row := newSavedKeyRow(savedName, savedKey, profileName) + if listErr == nil { + row.API = "missing on API" + objectstorage.Warnf("access key %q (%s) is saved in profile %s but the API does not list it; if it was deleted run 'lsh s3 access-keys forget %s'", savedName, savedKey.AccessKeyID, profileName, savedName) + } + if isHuman() { + printSavedKeyDetails(os.Stdout, row) + return nil + } + render([]renderer.ResponseData{row}) + return nil +} + +// printKeyDetails prints the human view of one API key as Label: value lines. +func printKeyDetails(w io.Writer, r keyRow) { + fmt.Fprintf(w, "Name: %s\n", r.Name) + fmt.Fprintf(w, "Access Key ID: %s\n", r.AccessKeyID) + fmt.Fprintf(w, "Scope: %s\n", r.scopeCell()) + buckets := r.bucketsCell() + if r.Access != config.ScopeFullAccess && len(r.Buckets) > 0 { + buckets = strings.Join(r.Buckets, ", ") + } + fmt.Fprintf(w, "Buckets: %s\n", buckets) + fmt.Fprintf(w, "Class: %s\n", r.StorageClass) + fmt.Fprintf(w, "Site: %s\n", dash(r.Site)) + fmt.Fprintf(w, "Project: %s\n", dash(r.Project)) + fmt.Fprintf(w, "Status: %s\n", dash(r.Status)) + if r.Username != "" { + fmt.Fprintf(w, "Username: %s\n", r.Username) + } + if r.CreatedAt != "" { + fmt.Fprintf(w, "Created: %s\n", r.CreatedAt) + } + saved := "no" + if r.Saved { + saved = fmt.Sprintf("yes (as %q)", r.SavedAs) + } + fmt.Fprintf(w, "Saved: %s\n", saved) +} + +// printSavedKeyDetails prints the human view of one saved key (no secret). +func printSavedKeyDetails(w io.Writer, r savedKeyRow) { + fmt.Fprintf(w, "Name: %s\n", r.Name) + fmt.Fprintf(w, "Access Key ID: %s\n", r.AccessKeyID) + fmt.Fprintf(w, "Scope: %s\n", dash(r.Scope)) + buckets := r.bucketsCell() + if r.Scope != config.ScopeFullAccess && len(r.Buckets) > 0 { + buckets = formatPerms(r.Buckets) + } + fmt.Fprintf(w, "Buckets: %s\n", buckets) + fmt.Fprintf(w, "Class: %s\n", dash(r.StorageClass)) + fmt.Fprintf(w, "Site: %s\n", dash(r.Site)) + fmt.Fprintf(w, "Project: %s\n", dash(r.ProjectID)) + if r.Username != "" { + fmt.Fprintf(w, "Username: %s\n", r.Username) + } + fmt.Fprintf(w, "Source: %s\n", dash(r.Source)) + fmt.Fprintf(w, "Created: %s\n", dash(r.CreatedAt)) + fmt.Fprintf(w, "Profile: %s\n", dash(r.Profile)) + if r.API != "" { + fmt.Fprintf(w, "API: %s\n", r.API) + } +} diff --git a/cmd/s3/accesskeys_import.go b/cmd/s3/accesskeys_import.go new file mode 100644 index 0000000..501ea68 --- /dev/null +++ b/cmd/s3/accesskeys_import.go @@ -0,0 +1,224 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysImportCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "import --name --access-key-id ", + Short: "Save an existing access key in this profile (secret from stdin or a prompt)", + Long: `Save an access key that already exists (created in the dashboard, by another +tool or on another machine) in the active profile. + +The secret is never passed as a flag: it is read from a prompt without echo on +a terminal, or from a single line on stdin otherwise, so +'echo "$SECRET" | lsh s3 access-keys import ...' works in scripts. + +When --project is known the key is matched against the API to record its scope +(class, site, buckets); otherwise --storage-class/--region/--bucket/--all-buckets +describe it. Keys with unknown scope are never selected automatically: they are +only used with --access-key .`, + Example: ` lsh s3 access-keys import --name legacy --access-key-id AKIA... --project my-project + echo "$SECRET" | lsh s3 access-keys import --name ci --access-key-id AKIA... --bucket backups=rw + lsh s3 access-keys import --name ops --access-key-id AKIA... --all-buckets --storage-class standard --project my-project`, + Args: cobra.NoArgs, + RunE: runAccessKeysImport, + }) + addProjectFlag(cmd, true, "project of the key (ID or slug); lets the CLI look its scope up on the API") + cmd.Flags().String("name", "", "name to save the key under (required)") + cmd.Flags().String("access-key-id", "", "the access key ID (required)") + cmd.Flags().StringP("storage-class", "c", "", "storage class of the key: standard or high_performance") + cmd.Flags().String("region", "", "site of the key (e.g. TYO4) for high_performance keys") + cmd.Flags().StringArray("bucket", nil, bucketSpecUsage) + cmd.Flags().Bool("all-buckets", false, "the key covers every bucket of the project in its storage class (fullaccess)") + _ = cmd.MarkFlagRequired("name") + _ = cmd.MarkFlagRequired("access-key-id") + return cmd +} + +// importScope resolves the metadata of an imported key: first from the API +// (when the project is known and the ID is listed), then from the flags. +func importScope(ctx context.Context, r *objectstorage.Resolver, accessKeyID string, o createOptions) (config.StoredAccessKey, error) { + k := config.StoredAccessKey{ + AccessKeyID: accessKeyID, + StorageClass: o.StorageClass, + Site: o.Site, + ProjectID: o.Project, + Scope: config.ScopeUnknown, + Source: config.KeySourceImport, + CreatedAt: time.Now().UTC(), + } + + if o.Project != "" && r.API != nil { + scopes, err := objectstorage.RawAccessKeyScopes(ctx, o.Project) + if err != nil { + lsh.LogDebugf("[s3] could not look the key up on the API: %v", err) + } else if s, ok := scopes[accessKeyID]; ok { + if k.StorageClass == "" { + k.StorageClass = s.StorageClass + } + if k.Site == "" { + k.Site = strings.ToUpper(s.Site) + } + k.Username = s.Username + switch s.Access { + case config.ScopeFullAccess: + k.Scope = config.ScopeFullAccess + case config.PermissionRW, config.PermissionReadOnly: + k.Scope = config.ScopeLimitedAccess + k.Buckets = map[string]string{} + scoped := *r + scoped.Project = o.Project + list, err := scoped.ListBuckets(ctx) + if err != nil { + return k, err + } + for _, d := range list { + b := objectstorage.BucketFromData(d) + for _, name := range s.Buckets { + if b.BucketName == name || b.Name == name { + k.Buckets[b.ID] = s.Access + if k.ProjectID == "" || !strings.HasPrefix(k.ProjectID, "proj_") { + k.ProjectID = b.ProjectID + } + } + } + } + } + if k.Scope != config.ScopeUnknown { + return k, nil + } + } + } + + switch { + case o.AllBuckets && len(o.BucketSpecs) > 0: + return k, exitcode.Errorf(exitcode.Usage, "--all-buckets and --bucket are mutually exclusive") + case o.AllBuckets: + k.Scope = config.ScopeFullAccess + if k.StorageClass == "" { + return k, exitcode.Errorf(exitcode.Usage, "--storage-class standard|high_performance is required with --all-buckets") + } + if k.StorageClass == objectstorage.ClassHighPerformance && k.Site == "" { + return k, exitcode.Errorf(exitcode.Usage, "--region is required for a high_performance key (e.g. TYO4)") + } + case len(o.BucketSpecs) > 0: + var scoped []scopedBucket + for _, spec := range o.BucketSpecs { + b, err := r.Resolve(ctx, spec.Token) + if err != nil { + return k, err + } + if b.EndpointOverride { + return k, exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url") + } + if err := r.FillSite(ctx, b); err != nil { + lsh.LogDebugf("[s3] site lookup failed for %s: %v", b.ID, err) + } + scoped = append(scoped, scopedBucket{Bucket: b, Permission: spec.Permission}) + } + if err := validateSameGroup(scoped); err != nil { + return k, err + } + first := scoped[0].Bucket + k.Scope = config.ScopeLimitedAccess + k.Buckets = map[string]string{} + for _, sb := range scoped { + k.Buckets[sb.Bucket.ID] = sb.Permission + } + if k.StorageClass == "" { + k.StorageClass = first.StorageClass + } + if k.Site == "" && first.StorageClass == objectstorage.ClassHighPerformance { + k.Site = strings.ToUpper(first.Site) + } + if k.ProjectID == "" || !strings.HasPrefix(k.ProjectID, "proj_") { + k.ProjectID = firstNonEmptyStr(first.ProjectID, k.ProjectID) + } + } + return k, nil +} + +func runAccessKeysImport(cmd *cobra.Command, _ []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + o, err := parseCreateOptions(cmd) + if err != nil { + return printErr(err) + } + accessKeyID, _ := cmd.Flags().GetString("access-key-id") + accessKeyID = strings.TrimSpace(accessKeyID) + if o.Name == "" || accessKeyID == "" { + return printErr(exitcode.Errorf(exitcode.Usage, "--name and --access-key-id are required")) + } + // Fail before reading the secret when there is no profile to save into. + _, profileName, _, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + + r := newResolver(cmd) + if r.EndpointURL != "" { + r.API = nil + } + stored, err := importScope(ctx, r, accessKeyID, o) + // keyMatchesBucket compares ProjectID with the bucket's proj_ ID, so a slug + // passed on --project (accepted everywhere else) would leave the imported + // key permanently unselectable. The --bucket paths already resolve it. + if err == nil && stored.ProjectID != "" && !strings.HasPrefix(stored.ProjectID, "proj_") { + stored.ProjectID = resolveProjectID(ctx, r, stored.ProjectID) + } + if err != nil { + return printErr(err) + } + if stored.Scope == config.ScopeUnknown { + objectstorage.Warnf("the scope of %s is unknown (not found on the API and no --bucket/--all-buckets given); the key is only used when selected explicitly with --access-key %s", accessKeyID, o.Name) + } + row := newSavedKeyRow(o.Name, stored, profileName) + + if dryRun() { + if isHuman() { + fmt.Printf("(dryrun) import: access key %s as %q in profile %s (class=%s site=%s scope=%s)\n", accessKeyID, o.Name, profileName, dash(stored.StorageClass), dash(stored.Site), stored.Scope) + return nil + } + render([]renderer.ResponseData{row}) + return nil + } + + secret, err := objectstorage.ReadSecret(cmd, fmt.Sprintf("Secret access key for %s: ", accessKeyID)) + if err != nil { + return printErr(exitcode.Errorf(exitcode.Usage, "could not read the secret: %v", err)) + } + if secret == "" { + return printErr(exitcode.Errorf(exitcode.Usage, "no secret provided; type it at the prompt or pipe it on stdin: echo \"$SECRET\" | lsh s3 access-keys import ...")) + } + stored.SecretAccessKey = secret + // Same clash policy as create/configure/mb: a name holding another key + // is never overwritten; the import is saved with a -2, -3… suffix. + finalName, profileName, err := saveNewKey(cmd, o.Name, stored) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + if finalName != o.Name { + objectstorage.Hintf("a saved key named %q already exists; saved as %q instead", o.Name, finalName) + row = newSavedKeyRow(finalName, stored, profileName) + } + if isHuman() { + fmt.Printf("import: access key %s saved as %q in profile %s (scope %s)\n", accessKeyID, finalName, profileName, stored.Scope) + return nil + } + render([]renderer.ResponseData{row}) + return nil +} diff --git a/cmd/s3/accesskeys_list.go b/cmd/s3/accesskeys_list.go new file mode 100644 index 0000000..b5cd45b --- /dev/null +++ b/cmd/s3/accesskeys_list.go @@ -0,0 +1,316 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + "text/tabwriter" + + "github.com/latitudesh/lsh/cli" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysListCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List access keys (API), or the keys saved in this profile (--saved)", + Long: `List the access keys of a project. Pick a project with --project (or +LSH_PROJECT), list every project with --all-projects, or — in a terminal — +choose from the project picker (which includes "All projects"). + +SCOPE is fullaccess, rw or readonly as the API reports it; BUCKETS are the +backend bucket names the key covers; SAVED tells whether the key (by access key +ID) is stored in the active profile. Secrets are never listed. + +--saved lists only the local profile, marking keys the API no longer knows as +"missing on API".`, + Example: ` lsh s3 access-keys list + lsh s3 access-keys list --project my-project -o json + lsh s3 access-keys list --bucket backups + lsh s3 access-keys list --saved`, + Args: cobra.NoArgs, + RunE: runAccessKeysList, + }) + addProjectFlag(cmd, true, "list the keys of this project only (ID or slug)") + cmd.Flags().Bool("all-projects", false, "list the keys of every project (skip the project picker)") + cmd.Flags().StringP("storage-class", "c", "", "only keys of this storage class: standard or high_performance") + cmd.Flags().String("region", "", "only keys of this site (e.g. DAL, TYO4)") + cmd.Flags().String("bucket", "", "only the keys covering this bucket (name or bkt_ ID)") + cmd.Flags().Bool("saved", false, "list the keys saved in the active profile instead of the API") + return cmd +} + +// listFilter narrows a key listing. +type listFilter struct { + StorageClass string + Site string + Project string +} + +// matchAPI applies the class and site filters. A key the API lists without +// a site (standard keys span sites) matches any --region: the flag then +// disambiguates rather than hides. +func (f listFilter) matchAPI(k apiKey) bool { + if f.StorageClass != "" && k.StorageClass != f.StorageClass { + return false + } + if f.Site != "" && k.Site != "" && !strings.EqualFold(k.Site, f.Site) { + return false + } + return true +} + +// withSite returns k with the --region value filled in when the API left the +// site empty, so a single match carries the site the user named. +func (f listFilter) withSite(k apiKey) apiKey { + if k.Site == "" && f.Site != "" { + k.Site = f.Site + } + return k +} + +func (f listFilter) matchSaved(k config.StoredAccessKey) bool { + if f.StorageClass != "" && k.StorageClass != f.StorageClass { + return false + } + // Same wildcard as matchAPI: a key stored without a site (standard keys + // span sites) matches any --region instead of being hidden by it. + if f.Site != "" && k.Site != "" && !strings.EqualFold(k.Site, f.Site) { + return false + } + // --project takes an ID or a slug, while the key stores whichever was + // known when it was saved. Comparing across the two kinds would hide keys + // instead of filtering them, so only like-for-like counts as a mismatch. + if f.Project != "" && k.ProjectID != "" && comparableProjects(k.ProjectID, f.Project) && k.ProjectID != f.Project { + return false + } + return true +} + +// comparableProjects reports whether two project references are of the same +// kind (both proj_ IDs or both slugs) and can therefore be compared. +func comparableProjects(a, b string) bool { + return strings.HasPrefix(a, "proj_") == strings.HasPrefix(b, "proj_") +} + +func runAccessKeysList(cmd *cobra.Command, _ []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + class, err := storageClassFlag(cmd) + if err != nil { + return printErr(err) + } + site, err := regionFlag(cmd) + if err != nil { + return printErr(err) + } + filter := listFilter{StorageClass: class, Site: site, Project: projectFlag(cmd)} + + if saved, _ := cmd.Flags().GetBool("saved"); saved { + return runAccessKeysListSaved(ctx, cmd, filter) + } + + api := newKeysAPI() + r := newResolver(cmd) + var keys []apiKey + slugs := map[string]string{} + multiProject := false + if bucket, _ := cmd.Flags().GetString("bucket"); bucket != "" { + b, err := r.Resolve(ctx, bucket) + if err != nil { + return printErr(err) + } + if b.EndpointOverride { + return printErr(exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url")) + } + if err := r.FillSite(ctx, b); err != nil { + lsh.LogDebugf("[s3] site lookup failed for %s: %v", b.ID, err) + } + keys, err = api.listForBucket(ctx, b) + if err != nil { + return printErr(err) + } + if b.ProjectSlug != "" { + slugs[b.ProjectRef()] = b.ProjectSlug + } + } else { + // Pick the project like `lsh s3 ls`: --project / LSH_PROJECT, or + // --all-projects, otherwise the interactive picker (with "All projects"). + project, allProjects, perr := cli.PickProjectForList(cmd) + if perr != nil { + return printErr(perr) + } + var projects []string + if allProjects { + projects, slugs, err = teamProjects(ctx, r) + if err != nil { + return printErr(err) + } + multiProject = len(projects) > 1 + } else { + projects = []string{project} + filter.Project = project + } + keys, err = api.listProjects(ctx, projects) + if err != nil { + return printErr(err) + } + } + + savedByID, _, _ := savedKeysByID(cmd) + var rows []keyRow + for _, k := range keys { + if !filter.matchAPI(k) { + continue + } + rows = append(rows, newKeyRow(k, savedByID, slugs)) + } + if isHuman() { + if len(rows) == 0 { + objectstorage.Hintf("no access keys found") + return nil + } + writeKeyTable(os.Stdout, rows, multiProject) + return nil + } + out := make([]renderer.ResponseData, 0, len(rows)) + for _, row := range rows { + out = append(out, row) + } + render(out) + return nil +} + +// writeKeyTable prints the human listing: an aligned table with a header, no +// borders, the way the plan shows it. +func writeKeyTable(w io.Writer, rows []keyRow, withProject bool) { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + header := "NAME\tACCESS KEY ID\tSCOPE\tBUCKETS\tCLASS\tSITE\tSTATUS\tSAVED" + if withProject { + header = "PROJECT\t" + header + } + fmt.Fprintln(tw, header) + for _, r := range rows { + line := strings.Join([]string{r.Name, r.AccessKeyID, r.scopeCell(), r.bucketsCell(), r.StorageClass, dash(r.Site), dash(r.Status), yesNo(r.Saved)}, "\t") + if withProject { + line = dash(r.Project) + "\t" + line + } + fmt.Fprintln(tw, line) + } + tw.Flush() +} + +// runAccessKeysListSaved lists the profile's keys, cross-checking the API +// when it is reachable. +func runAccessKeysListSaved(ctx context.Context, cmd *cobra.Command, filter listFilter) error { + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + keys := p.ObjectStorageKeys() + names := p.SortedObjectStorageKeyNames() + + // Best-effort API cross-check over the projects the saved keys mention. + known, queried := apiKeyIDs(ctx, cmd, keys, filter.Project) + + var rows []savedKeyRow + for _, name := range names { + k := keys[name] + if !filter.matchSaved(k) { + continue + } + row := newSavedKeyRow(name, k, profileName) + // A key whose project was never listed cannot be judged: leaving the + // column empty is honest, "missing on API" would not be. + if known != nil && k.ProjectID != "" && queried[k.ProjectID] { + if known[k.AccessKeyID] { + row.API = "ok" + } else { + row.API = "missing on API" + } + } + rows = append(rows, row) + } + if isHuman() { + if len(rows) == 0 { + objectstorage.Hintf("no access keys saved in profile %s; run 'lsh s3 configure' or 'lsh s3 access-keys create --save'", profileName) + return nil + } + writeSavedKeyTable(os.Stdout, rows, known != nil) + return nil + } + out := make([]renderer.ResponseData, 0, len(rows)) + for _, row := range rows { + out = append(out, row) + } + render(out) + return nil +} + +// apiKeyIDs returns the set of access key IDs the API knows for the projects +// referenced by the saved keys (or the --project filter), or nil when the API +// could not be consulted (not logged in, network error, no project known). +func apiKeyIDs(ctx context.Context, cmd *cobra.Command, keys map[string]config.StoredAccessKey, project string) (known, queried map[string]bool) { + if endpointOverride(cmd) != "" { + return nil, nil + } + seen := map[string]bool{} + var projects []string + if project != "" { + projects = []string{project} + } else { + for _, k := range keys { + if k.ProjectID != "" && !seen[k.ProjectID] { + seen[k.ProjectID] = true + projects = append(projects, k.ProjectID) + } + } + sort.Strings(projects) + } + if len(projects) == 0 { + return nil, nil + } + api := newKeysAPI() + known = map[string]bool{} + queried = map[string]bool{} + for _, p := range projects { + list, err := api.list(ctx, p) + if err != nil { + lsh.LogDebugf("[s3] could not cross-check saved keys with the API for project %s: %v", p, err) + return nil, nil + } + queried[p] = true + for _, k := range list { + known[k.AccessKeyID] = true + } + } + return known, queried +} + +// writeSavedKeyTable prints the --saved listing. +func writeSavedKeyTable(w io.Writer, rows []savedKeyRow, withAPI bool) { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + header := "NAME\tACCESS KEY ID\tCLASS\tSITE\tPROJECT\tSCOPE\tBUCKETS\tSOURCE\tCREATED" + if withAPI { + header += "\tAPI" + } + fmt.Fprintln(tw, header) + for _, r := range rows { + line := strings.Join([]string{r.Name, r.AccessKeyID, dash(r.StorageClass), dash(r.Site), dash(r.ProjectID), dash(r.Scope), r.bucketsCell(), dash(r.Source), dash(r.CreatedAt)}, "\t") + if withAPI { + line += "\t" + dash(r.API) + } + fmt.Fprintln(tw, line) + } + tw.Flush() +} diff --git a/cmd/s3/accesskeys_list_test.go b/cmd/s3/accesskeys_list_test.go new file mode 100644 index 0000000..1b62036 --- /dev/null +++ b/cmd/s3/accesskeys_list_test.go @@ -0,0 +1,241 @@ +package s3 + +import ( + "bytes" + "context" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func mustTime(t *testing.T, s string) time.Time { + t.Helper() + v, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatal(err) + } + return v +} + +func TestListMergesRawScopes(t *testing.T) { + f := newFakeAPI(t) + f.keys["proj_1"] = keysDocument + + keys, err := f.client().list(context.Background(), "proj_1") + if err != nil { + t.Fatal(err) + } + if len(keys) != 3 { + t.Fatalf("want 3 keys, got %d: %+v", len(keys), keys) + } + // One GET per project: the raw document already carries every field. + reqs := f.Requests() + if len(reqs) != 1 || reqs[0].Method != "GET" || reqs[0].Path != "/storage/access_keys" || reqs[0].Query.Get("project") != "proj_1" { + t.Fatalf("list must issue exactly one GET /storage/access_keys, got %+v", reqs) + } + if !strings.HasPrefix(reqs[0].Query.Get("project"), "proj_") { + t.Fatalf("project query wrong: %+v", reqs[0].Query) + } + byID := map[string]apiKey{} + for _, k := range keys { + byID[k.AccessKeyID] = k + } + ops := byID["OPSKEYID000000000001"] + if ops.Access != "fullaccess" || ops.StorageClass != "standard" || len(ops.Buckets) != 2 || ops.scope() != config.ScopeFullAccess { + t.Fatalf("ops key not merged with raw scope: %+v", ops) + } + ci := byID["CIKEYID0000000000002"] + if ci.Access != "rw" || ci.scope() != config.ScopeLimitedAccess || ci.Buckets[0] != "backups-7f3a" { + t.Fatalf("ci key not merged: %+v", ci) + } + hp := byID["HPKEYID0000000000003"] + if hp.StorageClass != "high_performance" || hp.Site != "TYO4" || hp.Access != "readonly" { + t.Fatalf("hp key wrong: %+v", hp) + } + + // Rows: SAVED comes from the profile match, SCOPE from the raw access. + rows := []keyRow{} + saved := map[string]string{"CIKEYID0000000000002": "ci-deploy"} + for _, k := range keys { + rows = append(rows, newKeyRow(k, saved, map[string]string{"proj_1": "my-project"})) + } + var buf bytes.Buffer + writeKeyTable(&buf, rows, false) + out := buf.String() + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) != 4 || !strings.HasPrefix(lines[0], "NAME") || !strings.Contains(lines[0], "SAVED") { + t.Fatalf("unexpected table:\n%s", out) + } + if !strings.Contains(out, "ci-deploy") || !strings.Contains(out, "yes") || !strings.Contains(out, "backups-7f3a") || !strings.Contains(out, "all") { + t.Fatalf("table missing columns:\n%s", out) + } + if strings.Contains(out, "secret") { + t.Fatalf("listing must never mention secrets:\n%s", out) + } + for _, r := range rows { + if r.Project != "my-project" { + t.Fatalf("project slug not applied: %+v", r) + } + } +} + +// TestListWithoutScopeFields replaces the former "raw scopes unavailable" +// test: listing is now a single raw request, so there is no second call to +// degrade from. Records without `access`/`buckets` (older backends) must +// still list and render '-' for the unknown scope. +func TestListWithoutScopeFields(t *testing.T) { + f := newFakeAPI(t) + f.keys["proj_1"] = `{"data":{"standard":[{"name":"plain","username":"p@example.com","access_key_id":"PLAINKEYID0000000001","status":"Active","created_at":"2026-09-01T10:00:00Z"}],"high_performance":[]}}` + + keys, err := f.client().list(context.Background(), "proj_1") + if err != nil { + t.Fatal(err) + } + if len(keys) != 1 || keys[0].Access != "" || keys[0].StorageClass != "standard" || keys[0].Site != "" { + t.Fatalf("listing must tolerate records without scope fields: %+v", keys) + } + row := newKeyRow(keys[0], nil, nil) + if row.scopeCell() != emptyCell || row.bucketsCell() != emptyCell { + t.Fatalf("unknown scope must render the shared placeholder: %+v", row) + } + if len(f.Requests()) != 1 { + t.Fatalf("want exactly one request, got %d", len(f.Requests())) + } +} + +func TestListProjectNotFound(t *testing.T) { + f := newFakeAPI(t) + _, err := f.client().list(context.Background(), "nope") + if err == nil || exitcode.Of(err) != exitcode.NotFound { + t.Fatalf("404 must map to exit 3, got %v", err) + } + // Without a token the raw call fails locally with exit 4 (no request made). + api := f.client() + api.token = "" + if _, err := api.list(context.Background(), "proj_1"); err == nil || exitcode.Of(err) != exitcode.Credentials { + t.Fatalf("missing token must be exit 4, got %v", err) + } +} + +// TestSiteFilterTreatsEmptyAPISiteAsWildcard covers F17: standard keys the +// API lists without a site must not vanish behind --region, and the single +// match gets the requested site filled in. +func TestSiteFilterTreatsEmptyAPISiteAsWildcard(t *testing.T) { + noSite := apiKey{Name: "ops", AccessKeyID: "A1", Username: "u1", StorageClass: "standard", Project: "proj_1"} + dal := apiKey{Name: "ci", AccessKeyID: "A2", Username: "u2", StorageClass: "standard", Site: "DAL", Project: "proj_1"} + tyo := apiKey{Name: "fast", AccessKeyID: "A3", Username: "u3", StorageClass: "high_performance", Site: "TYO4", Project: "proj_1"} + filter := listFilter{Site: "DAL"} + if !filter.matchAPI(noSite) || !filter.matchAPI(dal) || filter.matchAPI(tyo) { + t.Fatalf("empty API site must match any --region; a differing site must not") + } + if got := filter.withSite(noSite); got.Site != "DAL" { + t.Fatalf("withSite must fill the empty site, got %q", got.Site) + } + if got := filter.withSite(tyo); got.Site != "TYO4" { + t.Fatalf("withSite must not override an API site, got %q", got.Site) + } + keys := []apiKey{noSite, dal, tyo} + if got := findAPIKeys("ops", keys, filter); len(got) != 1 || got[0].AccessKeyID != "A1" { + t.Fatalf("get with --region must still find the standard key without a site: %+v", got) + } + target, err := resolveDeleteTarget("ops", keys, nil, filter) + if err != nil || target.AccessKeyID != "A1" || target.Site != "DAL" { + t.Fatalf("delete with --region must resolve the key and carry the site, got %+v err=%v", target, err) + } +} + +func TestResolveDeleteTarget(t *testing.T) { + keys := []apiKey{ + {Name: "backup", AccessKeyID: "A1", Username: "u1", StorageClass: "standard", Site: "DAL", Project: "proj_1"}, + {Name: "backup", AccessKeyID: "A2", Username: "u2", StorageClass: "high_performance", Site: "TYO4", Project: "proj_1"}, + {Name: "other", AccessKeyID: "A3", Username: "u3", StorageClass: "standard", Project: "proj_2"}, + } + saved := map[string]config.StoredAccessKey{ + "mine": {AccessKeyID: "A3", Username: "u3", StorageClass: "standard", ProjectID: "proj_2", Scope: config.ScopeFullAccess}, + "orphaned": {AccessKeyID: "ZZ", Username: "uz", StorageClass: "standard", ProjectID: "proj_9", Scope: config.ScopeFullAccess}, + } + + _, err := resolveDeleteTarget("backup", keys, saved, listFilter{}) + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--storage-class") { + t.Fatalf("ambiguous name must exit 2 with the disambiguating flags, got %v", err) + } + k, err := resolveDeleteTarget("backup", keys, saved, listFilter{StorageClass: "high_performance"}) + if err != nil || k.AccessKeyID != "A2" { + t.Fatalf("--storage-class must disambiguate, got %+v err=%v", k, err) + } + k, err = resolveDeleteTarget("backup", keys, saved, listFilter{Site: "dal"}) + if err != nil || k.AccessKeyID != "A1" { + t.Fatalf("--region must disambiguate, got %+v err=%v", k, err) + } + k, err = resolveDeleteTarget("u2", keys, saved, listFilter{}) + if err != nil || k.AccessKeyID != "A2" { + t.Fatalf("username must resolve, got %+v err=%v", k, err) + } + // A saved name resolves to its access key ID on the API. + k, err = resolveDeleteTarget("mine", keys, saved, listFilter{}) + if err != nil || k.AccessKeyID != "A3" || k.Username != "u3" { + t.Fatalf("saved name must resolve through the API, got %+v err=%v", k, err) + } + // A saved key the API no longer lists is deleted with its stored metadata. + k, err = resolveDeleteTarget("orphaned", keys, saved, listFilter{}) + if err != nil || k.AccessKeyID != "ZZ" || k.Username != "uz" || k.Project != "proj_9" { + t.Fatalf("orphaned saved key must resolve from the profile, got %+v err=%v", k, err) + } + if _, err := resolveDeleteTarget("missing", keys, saved, listFilter{}); err == nil || exitcode.Of(err) != exitcode.NotFound { + t.Fatalf("unknown token must exit 3, got %v", err) + } +} + +func TestDeleteSendsRegionOnlyForHighPerformance(t *testing.T) { + f := newFakeAPI(t) + api := f.client() + ctx := context.Background() + if err := api.delete(ctx, "u1", "standard", "proj_1", "DAL"); err != nil { + t.Fatal(err) + } + if err := api.delete(ctx, "u2", "high_performance", "proj_1", "TYO4"); err != nil { + t.Fatal(err) + } + if err := api.delete(ctx, "u2", "high_performance", "proj_1", ""); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("high_performance without site must fail with exit 2, got %v", err) + } + if err := api.delete(ctx, "", "standard", "proj_1", ""); err == nil { + t.Fatal("missing username must fail") + } + reqs := f.Requests() + if len(reqs) != 2 { + t.Fatalf("want 2 DELETE calls, got %d", len(reqs)) + } + if reqs[0].Path != "/storage/access_keys/u1/standard" || reqs[0].Query.Get("project") != "proj_1" || reqs[0].Query.Has("region") { + t.Fatalf("standard delete request wrong: %+v", reqs[0]) + } + if reqs[1].Path != "/storage/access_keys/u2/high_performance" || reqs[1].Query.Get("region") != "TYO4" { + t.Fatalf("high_performance delete request wrong: %+v", reqs[1]) + } + + f.deleteStatus = 404 + if err := api.delete(ctx, "gone", "standard", "proj_1", ""); err == nil || exitcode.Of(err) != exitcode.NotFound { + t.Fatalf("404 must map to exit 3, got %v", err) + } +} + +func TestSavedKeyRowNeverHoldsSecret(t *testing.T) { + k := config.StoredAccessKey{AccessKeyID: "ID", SecretAccessKey: "topsecret", StorageClass: "standard", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw"}, Source: "import", CreatedAt: mustTime(t, "2026-09-07T10:00:00Z")} + row := newSavedKeyRow("mine", k, "test") + var buf bytes.Buffer + writeSavedKeyTable(&buf, []savedKeyRow{row}, true) + if strings.Contains(buf.String(), "topsecret") { + t.Fatalf("secret leaked: %s", buf.String()) + } + if !strings.Contains(buf.String(), "bkt_1=rw") || !strings.Contains(buf.String(), "API") { + t.Fatalf("unexpected saved table:\n%s", buf.String()) + } + var out bytes.Buffer + printSavedKeyDetails(&out, row) + if strings.Contains(out.String(), "topsecret") || !strings.Contains(out.String(), "Access Key ID: ID") { + t.Fatalf("unexpected details:\n%s", out.String()) + } +} diff --git a/cmd/s3/accesskeys_rotate.go b/cmd/s3/accesskeys_rotate.go new file mode 100644 index 0000000..108ab0b --- /dev/null +++ b/cmd/s3/accesskeys_rotate.go @@ -0,0 +1,280 @@ +package s3 + +import ( + "context" + "fmt" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysRotateCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "rotate ", + Short: "Create a new key with the same scope as a saved key and save it in its place", + Long: `Rotate a saved access key: a new key with the same storage class, site, +project and scope (fullaccess or the same buckets and permissions) is created +and saved under the same name, replacing the old one on this machine. + +The old key keeps working until it is deleted: pass --delete-old to delete it +right away (after confirmation), or delete it later once every consumer has +the new secret.`, + Example: ` lsh s3 access-keys rotate ci-deploy + lsh s3 access-keys rotate ci-deploy --delete-old --yes + lsh s3 access-keys rotate ci-deploy --name ci-deploy-2026q4`, + Args: cobra.ExactArgs(1), + RunE: runAccessKeysRotate, + }) + addProjectFlag(cmd, true, "project of the key (ID or slug) when the saved key does not record it") + cmd.Flags().Bool("delete-old", false, "delete the old key on the API after the new one is saved") + cmd.Flags().String("name", "", "name of the new key (default: the saved name with a - suffix)") + cmd.Flags().String("region", "", "site of the key (e.g. TYO4) when the saved high_performance key does not record it") + addYesFlag(cmd) + return cmd +} + +// rotatedName appends the date suffix used for the new key by default, plus a +// sequence number from the second rotation of the same day on (the API rejects +// a duplicate name). The result always fits the API's name limit, so a long +// alias does not produce a request the API refuses. +func rotatedName(base string, now time.Time, seq int) string { + base = sanitizeKeyPart(base) + if base == "" { + base = "lsh-key" + } + suffix := "-" + now.UTC().Format("20060102") + if seq > 1 { + suffix += "-" + strconv.Itoa(seq) + } + if len(base)+len(suffix) > maxAccessKeyNameLen { + base = strings.Trim(base[:maxAccessKeyNameLen-len(suffix)], "-") + } + return base + suffix +} + +// rotateCreateKey creates the new key, working around a name that is already +// taken. Unlike create, rotate does not fall back to a random pet name: the +// name is meant to stay recognisable, so it grows a sequence number instead. +func rotateCreateKey(ctx context.Context, cmd *cobra.Command, req accessKeyRequest, base string, autoName bool) (*createdAccessKey, error) { + for seq := 2; ; seq++ { + created, err := createAccessKeyRetrying(ctx, cmd, req, false) + if err == nil || !autoName || !isNameConflict(err) || seq > 9 { + return created, err + } + req.Name = rotatedName(base, time.Now(), seq) + } +} + +// rotateResult is the structured output of rotate. +type rotateResult struct { + Name string `json:"name"` + SavedAs string `json:"saved_as"` + Profile string `json:"profile"` + OldAccessKeyID string `json:"old_access_key_id"` + NewAccessKeyID string `json:"new_access_key_id,omitempty"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Scope string `json:"scope"` + OldDeleted bool `json:"old_deleted"` + DryRun bool `json:"dry_run,omitempty"` +} + +func (r rotateResult) TableRow() table.Row { + status := "rotated" + if r.DryRun { + status = "dryrun" + } + return table.Row{ + "name": {Label: "Name", Value: r.Name}, + "old_access_key_id": {Label: "Old Access Key ID", Value: r.OldAccessKeyID}, + "new_access_key_id": {Label: "New Access Key ID", Value: dash(r.NewAccessKeyID)}, + "storage_class": {Label: "Class", Value: r.StorageClass}, + "region": {Label: "Site", Value: dash(r.Site)}, + "status": {Label: "Status", Value: status}, + } +} + +// rotateRequest builds the create request that mirrors a saved key. +func rotateRequest(k config.StoredAccessKey, project, site, newName string) (accessKeyRequest, error) { + if k.Scope == config.ScopeUnknown || k.Scope == "" { + return accessKeyRequest{}, exitcode.Errorf(exitcode.Usage, "the saved key has unknown scope, so its scope cannot be reproduced; create a new key with 'lsh s3 access-keys create' instead") + } + req := accessKeyRequest{ + Project: firstNonEmptyStr(k.ProjectID, project), + StorageClass: k.StorageClass, + Site: strings.ToUpper(firstNonEmptyStr(site, k.Site)), + Name: newName, + Scope: k.Scope, + } + if k.Scope == config.ScopeLimitedAccess { + req.Buckets = make(map[string]string, len(k.Buckets)) + for id, perm := range k.Buckets { + req.Buckets[id] = perm + } + } + if req.Project == "" { + return accessKeyRequest{}, exitcode.Errorf(exitcode.Usage, "the saved key does not record its project; pass --project ") + } + if req.StorageClass == "" { + return accessKeyRequest{}, exitcode.Errorf(exitcode.Usage, "the saved key does not record its storage class; re-import it with --storage-class or create a new key") + } + return req, nil +} + +func runAccessKeysRotate(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + savedName := strings.TrimSpace(args[0]) + + site, err := regionFlag(cmd) + if err != nil { + return printErr(err) + } + deleteOld, _ := cmd.Flags().GetBool("delete-old") + yes, _ := cmd.Flags().GetBool(flagYes) + newName, _ := cmd.Flags().GetString("name") + newName = strings.TrimSpace(newName) + + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + old, ok := p.ObjectStorageKeys()[savedName] + if !ok { + return printErr(exitcode.Errorf(exitcode.NotFound, "no saved access key named %q in profile %s; run 'lsh s3 access-keys list --saved'", savedName, profileName)) + } + autoName := newName == "" + if autoName { + newName = rotatedName(savedName, time.Now(), 1) + } else if err := validateAccessKeyName(newName); err != nil { + return printErr(err) + } + req, err := rotateRequest(old, projectFlag(cmd), site, newName) + if err != nil { + return printErr(err) + } + if err := req.validate(); err != nil { + return printErr(err) + } + // Do not create a key we would then refuse to finish handling. + if deleteOld && !yes && !dryRun() && !objectstorage.CanPrompt(cmd) { + return printErr(exitcode.Errorf(exitcode.Refused, "--delete-old asks for confirmation — pass --yes in a non-interactive session")) + } + + res := rotateResult{ + Name: newName, SavedAs: savedName, Profile: profileName, OldAccessKeyID: old.AccessKeyID, + StorageClass: req.StorageClass, Site: req.Site, Project: req.Project, Scope: req.Scope, DryRun: dryRun(), + } + deleteHint := fmt.Sprintf("lsh s3 access-keys delete %s --project %s --storage-class %s", old.AccessKeyID, req.Project, req.StorageClass) + if req.StorageClass == objectstorage.ClassHighPerformance { + deleteHint += " --region " + req.Site + } + + if dryRun() { + if isHuman() { + fmt.Printf("(dryrun) rotate: %s -> new key %q (class=%s site=%s project=%s scope=%s), saved as %q in profile %s\n", + old.AccessKeyID, newName, req.StorageClass, dash(req.Site), req.Project, req.Scope, savedName, profileName) + if deleteOld { + fmt.Printf("(dryrun) delete_access_key: %s\n", old.AccessKeyID) + } + return nil + } + render([]renderer.ResponseData{res}) + return nil + } + + api := newKeysAPI() + created, err := rotateCreateKey(ctx, cmd, req, savedName, autoName) + if err != nil { + return printErr(err) + } + res.Name = created.Name + res.NewAccessKeyID = created.AccessKeyID + // Record the project as a proj_ ID: the old key's when it has one, + // otherwise --project resolved through the project's buckets. + projectID := old.ProjectID + if projectID == "" { + projectID = resolveProjectID(ctx, newResolver(cmd), req.Project) + } + stored := created.stored(projectID, req.Buckets, config.KeySourceCreate) + // Rotate replaces the saved key in place (same name), unlike create. + if _, err := objectstorage.SaveKey(profileFlag(cmd), savedName, stored); err != nil { + // The old key is still live and still saved, so the new one can simply + // be removed: rotating again is cheaper than handling a printed secret. + // The view renders the scope from the plan, so the request's buckets + // have to be there for the fallback print to be complete. + created.Buckets = req.Buckets + reportUnsavedKey(ctx, os.Stderr, req, created, &createPlan{Request: req, Buckets: rotateScopedBuckets(req)}, savedName, false, err) + objectstorage.Hintf("the old key %s is untouched and still saved as %q", old.AccessKeyID, savedName) + return printErr(objectstorage.Humanize(err, nil, nil)) + } + objectstorage.Hintf("new key %s saved as %q in profile %s (replacing %s)", created.AccessKeyID, savedName, profileName, old.AccessKeyID) + + if deleteOld { + username := old.Username + if username == "" { + if keys, err := api.list(ctx, req.Project); err == nil { + for _, k := range keys { + if k.AccessKeyID == old.AccessKeyID { + username = k.Username + break + } + } + } + } + switch { + case username == "": + objectstorage.Warnf("could not find the username of the old key %s on the API; delete it later with: %s", old.AccessKeyID, deleteHint) + default: + err := objectstorage.ConfirmOrRefuse(cmd, yes, fmt.Sprintf("Delete the old access key %s now? Anything still using it stops working", old.AccessKeyID)) + if err != nil { + objectstorage.Warnf("old key %s kept; delete it later with: %s", old.AccessKeyID, deleteHint) + } else if err := api.delete(ctx, username, req.StorageClass, req.Project, req.Site); err != nil { + objectstorage.Warnf("could not delete the old key %s: %v — delete it later with: %s", old.AccessKeyID, err, deleteHint) + } else { + res.OldDeleted = true + } + } + } else { + objectstorage.Hintf("old key %s kept so existing consumers keep working; once they have the new secret run: %s", old.AccessKeyID, deleteHint) + } + + if isHuman() { + fmt.Printf("rotate: %s -> %s (saved as %q in profile %s)\n", old.AccessKeyID, created.AccessKeyID, savedName, profileName) + if res.OldDeleted { + fmt.Printf("delete_access_key: %s\n", old.AccessKeyID) + } + return nil + } + render([]renderer.ResponseData{res}) + return nil +} + +// rotateScopedBuckets rebuilds the plan's bucket list from the request. Rotate +// copies the old key's scope, which is stored as bkt_ IDs, so only the IDs and +// permissions are known — enough for the printed scope of an unsaved key. +func rotateScopedBuckets(req accessKeyRequest) []scopedBucket { + if len(req.Buckets) == 0 { + return nil + } + ids := make([]string, 0, len(req.Buckets)) + for id := range req.Buckets { + ids = append(ids, id) + } + sort.Strings(ids) + out := make([]scopedBucket, 0, len(ids)) + for _, id := range ids { + out = append(out, scopedBucket{Bucket: &objectstorage.Bucket{ID: id, Name: id}, Permission: req.Buckets[id]}) + } + return out +} diff --git a/cmd/s3/accesskeys_shared.go b/cmd/s3/accesskeys_shared.go new file mode 100644 index 0000000..65bcef4 --- /dev/null +++ b/cmd/s3/accesskeys_shared.go @@ -0,0 +1,964 @@ +package s3 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "sort" + "strings" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/version" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// accessKeyRequest describes an access key to create through the API. +type accessKeyRequest struct { + Project string // project ID or slug (required) + StorageClass string // standard | high_performance (required) + Site string // site slug; required for high_performance + Name string // key name (normalized server-side) + Scope string // config.ScopeFullAccess | config.ScopeLimitedAccess + Buckets map[string]string // bkt_ id → rw|readonly (limited_access only) +} + +// createdAccessKey is the normalized result of a create call, independent of +// the backend's field names (Wasabi: access_key_id/secret_access_key; VAST: +// access_key/secret_key). +type createdAccessKey struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key,omitempty"` + Username string `json:"username,omitempty"` + Status string `json:"status,omitempty"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Scope string `json:"scope"` + // Buckets lists bucket display names → permission for limited keys. Right + // after createAccessKey it holds the bkt_ IDs of the request; the create + // command swaps them for display names before printing. + Buckets map[string]string `json:"buckets,omitempty"` + // Endpoint and SigningRegion are filled by the caller from the buckets + // the key covers (the API does not return them). + Endpoint string `json:"endpoint,omitempty"` + SigningRegion string `json:"signing_region,omitempty"` +} + +// stored converts the created key into the config.json representation. +func (k createdAccessKey) stored(projectID string, bucketPerms map[string]string, source string) config.StoredAccessKey { + return config.StoredAccessKey{ + AccessKeyID: k.AccessKeyID, + SecretAccessKey: k.SecretAccessKey, + StorageClass: k.StorageClass, + Site: k.Site, + ProjectID: projectID, + Scope: k.Scope, + Buckets: bucketPerms, + Username: k.Username, + Source: source, + CreatedAt: time.Now().UTC(), + } +} + +// createAccessKey calls POST /storage/access_keys with the CLI's API client. +// mb and configure call it to offer a key right after bucket creation. +func createAccessKey(ctx context.Context, cmd *cobra.Command, req accessKeyRequest) (*createdAccessKey, error) { + _ = cmd + return newKeysAPI().create(ctx, req) +} + +// provisioningRetryDelays are the waits before re-sending a create that failed +// because the backend had not registered the new key yet. Creating a key is a +// multi-step operation on the provider side (create the user, create the key, +// then attach the bucket policy), and the steps do not become visible to each +// other instantly, so the first attempt can fail with a 404 that succeeds a +// few seconds later. The API rolls its own steps back before returning the +// error, so re-sending the same request is safe. +var provisioningRetryDelays = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second} + +// createAccessKeyRetrying creates the key, retrying two failures the API +// itself considers recoverable: a duplicate name (only when the name was +// auto-generated — a fresh pet name is rolled, the way the dashboard avoids +// collisions) and the propagation lag above (same request, after a wait). +func createAccessKeyRetrying(ctx context.Context, cmd *cobra.Command, req accessKeyRequest, autoName bool) (*createdAccessKey, error) { + return retryCreate(ctx, req, autoName, func(r accessKeyRequest) (*createdAccessKey, error) { + return createAccessKey(ctx, cmd, r) + }) +} + +// retryCreate holds the retry policy of createAccessKeyRetrying, with the +// create call injected so the policy can be tested without the API. +func retryCreate(ctx context.Context, req accessKeyRequest, autoName bool, create func(accessKeyRequest) (*createdAccessKey, error)) (*createdAccessKey, error) { + namesLeft, waited := 5, 0 + for { + created, err := create(req) + if err == nil { + return created, nil + } + switch { + case autoName && isNameConflict(err) && namesLeft > 1: + namesLeft-- + req.Name = generateKeyName(req.StorageClass) + case isProvisioningLag(err) && waited < len(provisioningRetryDelays): + delay := provisioningRetryDelays[waited] + waited++ + if waited == 1 { + objectstorage.Hintf("the storage backend has not registered the key yet; retrying in %s…", delay) + } + if waitErr := sleepCtx(ctx, delay); waitErr != nil { + return nil, waitErr + } + case isProvisioningLag(err): + return nil, exitcode.Errorf(exitcode.Of(err), "%v — the storage backend did not register the key in time; retry in a few seconds", err) + default: + return nil, err + } + } +} + +// isNameConflict reports whether err is the API rejecting a duplicate key name. +func isNameConflict(err error) bool { + m := strings.ToLower(err.Error()) + if !strings.Contains(m, "name") { + return false + } + return strings.Contains(m, "taken") || strings.Contains(m, "already") || + strings.Contains(m, "exist") || strings.Contains(m, "unique") || + strings.Contains(m, "in use") +} + +// isProvisioningLag reports whether err is the API surfacing a backend that is +// still converging: STORAGE_RESOURCE_NOT_FOUND ("Storage resource not found.") +// while the new key propagates, or STORAGE_UNAVAILABLE. Both are transient and +// the same request succeeds once the backend catches up. +func isProvisioningLag(err error) bool { + m := strings.ToLower(err.Error()) + return strings.Contains(m, "storage resource not found") || + strings.Contains(m, "temporarily unavailable") +} + +// sleepCtx waits for d, returning early (with the right exit code) if the +// context is cancelled by Ctrl-C or a deadline. +func sleepCtx(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.Canceled) { + return exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + return exitcode.Errorf(exitcode.Generic, "timed out waiting for the storage backend: %v", ctx.Err()) + } +} + +// keysAPI wraps the Latitude API calls of the access-keys commands so the +// command cores can run against a test server. Create and delete go through +// the SDK; listing reads the raw JSON:API document because the generated +// model drops `buckets[]` and `access` (see list). +type keysAPI struct { + sdk *sdk.Latitudesh + opts []operations.Option + // baseURL, token and apiVersion configure the raw GET (same settings the + // SDK client is built with). + baseURL string + token string + apiVersion string + http *http.Client +} + +// newKeysAPI builds the wrapper around the CLI's authenticated client. +func newKeysAPI() *keysAPI { + host := viper.GetString("hostname") + if host == "" { + host = "api.latitude.sh" + } + scheme := viper.GetString("scheme") + if scheme == "" { + scheme = "https" + } + return &keysAPI{ + sdk: apiClient(), + opts: []operations.Option{operations.WithRetries(lsh.RetryConfig())}, + baseURL: scheme + "://" + host, + token: viper.GetString("Authorization"), + apiVersion: viper.GetString("api-version"), + } +} + +// rawGet performs an authenticated GET and returns the body. Non-2xx +// responses are mapped through objectstorage.HumanizeAPI (as an SDK +// *components.APIError) so exit codes match the SDK calls. +func (a *keysAPI) rawGet(ctx context.Context, path string, query url.Values, what string) ([]byte, error) { + if a.token == "" { + return nil, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login' first") + } + u := a.baseURL + path + if len(query) > 0 { + u += "?" + query.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+strings.TrimPrefix(a.token, "Bearer ")) + req.Header.Set("Accept", "application/vnd.api+json, application/json") + req.Header.Set("User-Agent", "lsh/"+version.Version) + if a.apiVersion != "" { + req.Header.Set("API-Version", a.apiVersion) + } + client := a.http + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + resp, err := client.Do(req) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, what) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, objectstorage.HumanizeAPI(&components.APIError{Message: "API error", StatusCode: resp.StatusCode, Body: string(body), RawResponse: resp}, what) + } + return body, nil +} + +// validate checks the request before it reaches the API. +func (r accessKeyRequest) validate() error { + switch { + case r.Project == "": + return exitcode.Errorf(exitcode.Usage, "a project is required to create an access key; pass --project") + case r.StorageClass != objectstorage.ClassStandard && r.StorageClass != objectstorage.ClassHighPerformance: + return exitcode.Errorf(exitcode.Usage, "invalid storage class %q (use standard or high_performance)", r.StorageClass) + case r.Name == "": + return exitcode.Errorf(exitcode.Usage, "an access key name is required; pass --name") + case r.StorageClass == objectstorage.ClassHighPerformance && r.Site == "": + return exitcode.Errorf(exitcode.Usage, "--region is required for high_performance keys (a Latitude site slug such as TYO4 or DAL)") + case r.Scope == config.ScopeLimitedAccess && len(r.Buckets) == 0: + return exitcode.Errorf(exitcode.Usage, "a limited_access key needs at least one --bucket") + case r.Scope != config.ScopeLimitedAccess && r.Scope != config.ScopeFullAccess: + return exitcode.Errorf(exitcode.Usage, "invalid access scope %q", r.Scope) + } + for id, perm := range r.Buckets { + if perm != config.PermissionRW && perm != config.PermissionReadOnly { + return exitcode.Errorf(exitcode.Usage, "invalid permission %q for bucket %s (use rw or readonly)", perm, id) + } + } + return nil +} + +// buildCreateRequest converts the request into the SDK body. Bucket +// permissions are sorted so the payload (and dry-run output) is stable. +func buildCreateRequest(req accessKeyRequest) operations.PostStorageAccessKeysRequestBody { + attrs := operations.PostStorageAccessKeysAttributes{ + Project: req.Project, + AccessKeyStorageClass: operations.AccessKeyStorageClass(req.StorageClass), + Name: req.Name, + AccessScope: operations.AccessScope(req.Scope), + Region: req.Site, + } + if req.Scope == config.ScopeLimitedAccess { + ids := make([]string, 0, len(req.Buckets)) + for id := range req.Buckets { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + attrs.BucketPermissions = append(attrs.BucketPermissions, operations.BucketPermissions{ + BucketID: id, + Permission: operations.Permission(req.Buckets[id]), + }) + } + } + return operations.PostStorageAccessKeysRequestBody{ + Data: operations.PostStorageAccessKeysData{ + Type: operations.PostStorageAccessKeysTypeAccessKeys, + Attributes: attrs, + }, + } +} + +// create performs the POST and normalizes the response. +func (a *keysAPI) create(ctx context.Context, req accessKeyRequest) (*createdAccessKey, error) { + if err := req.validate(); err != nil { + return nil, err + } + resp, err := a.sdk.ObjectStorage.PostStorageAccessKeys(ctx, buildCreateRequest(req), a.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "") + } + return normalizeCreated(resp, req) +} + +// normalizeCreated maps both backend shapes onto createdAccessKey. +func normalizeCreated(resp *operations.PostStorageAccessKeysResponse, req accessKeyRequest) (*createdAccessKey, error) { + var ak *operations.AccessKey + if resp != nil && resp.Object != nil && resp.Object.Data != nil && resp.Object.Data.Attributes != nil { + ak = resp.Object.Data.Attributes.AccessKey + } + if ak == nil { + return nil, exitcode.Errorf(exitcode.Generic, "the API created the access key but returned no credentials; check 'lsh s3 access-keys list --project %s'", req.Project) + } + k := &createdAccessKey{ + Name: firstNonEmptyStr(ptrStr(ak.Name), req.Name), + AccessKeyID: firstNonEmptyStr(ptrStr(ak.AccessKeyID), ptrStr(ak.AccessKey)), + SecretAccessKey: firstNonEmptyStr(ptrStr(ak.SecretAccessKey), ptrStr(ak.SecretKey)), + Username: ptrStr(ak.Username), + Status: ptrStr(ak.Status), + StorageClass: req.StorageClass, + Site: req.Site, + Project: req.Project, + Scope: req.Scope, + } + if k.Scope == config.ScopeLimitedAccess && len(req.Buckets) > 0 { + k.Buckets = make(map[string]string, len(req.Buckets)) + for id, perm := range req.Buckets { + k.Buckets[id] = perm + } + } + if k.AccessKeyID == "" || k.SecretAccessKey == "" { + return nil, exitcode.Errorf(exitcode.Generic, "the API response is missing the access key id or secret (backend %s); the key may exist: check 'lsh s3 access-keys list --project %s'", req.StorageClass, req.Project) + } + return k, nil +} + +// apiKey is one access key as listed by the API, merged with the raw +// scope information (buckets[] and access) the SDK model drops. +type apiKey struct { + Name string + Username string + AccessKeyID string + Status string + CreatedAt string + StorageClass string + Site string + // Project is the project reference (ID or slug) the key was listed under. + Project string + // Access is fullaccess, rw, readonly or "" when unknown. + Access string + // Buckets are backend bucket names. + Buckets []string +} + +// scope maps the API access value onto the config scope constants. +func (k apiKey) scope() string { + switch k.Access { + case config.ScopeFullAccess: + return config.ScopeFullAccess + case config.PermissionRW, config.PermissionReadOnly: + return config.ScopeLimitedAccess + } + return config.ScopeUnknown +} + +// rawAccessKey is one record of GET /storage/access_keys as the API returns +// it (both storage classes share the shape; `buckets` and `access` are the +// fields the SDK model drops). +type rawAccessKey struct { + Name string `json:"name"` + Username string `json:"username"` + AccessKeyID string `json:"access_key_id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + Region string `json:"region"` + Access string `json:"access"` + Buckets []string `json:"buckets"` +} + +// list returns the project's keys of both storage classes with one GET +// /storage/access_keys call: the raw document carries every field the rows +// need, so the SDK call (which would drop buckets[]/access) is not made. +func (a *keysAPI) list(ctx context.Context, project string) ([]apiKey, error) { + if project == "" { + return nil, exitcode.Errorf(exitcode.Usage, "a project is required to list access keys; pass --project") + } + what := fmt.Sprintf("project %q", project) + body, err := a.rawGet(ctx, "/storage/access_keys", url.Values{"project": {project}}, what) + if err != nil { + return nil, err + } + var env struct { + Data map[string][]rawAccessKey `json:"data"` + } + if err := json.Unmarshal(body, &env); err != nil { + return nil, exitcode.Errorf(exitcode.Generic, "could not parse the access keys of %s: %v", what, err) + } + var out []apiKey + for class, keys := range env.Data { + for _, k := range keys { + out = append(out, apiKey{ + Name: k.Name, Username: k.Username, AccessKeyID: k.AccessKeyID, Status: k.Status, + CreatedAt: k.CreatedAt, Site: k.Region, Access: k.Access, Buckets: k.Buckets, + StorageClass: class, Project: project, + }) + } + } + sortAPIKeys(out) + return out, nil +} + +// listForBucket returns the keys covering one bucket (GET /storage/buckets/{id}/access_keys). +func (a *keysAPI) listForBucket(ctx context.Context, b *objectstorage.Bucket) ([]apiKey, error) { + resp, err := a.sdk.ObjectStorage.GetStorageBucketAccessKeys(ctx, b.ID, a.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("bucket %s", b.Display())) + } + var out []apiKey + if resp.Object != nil { + for _, d := range resp.Object.Data { + out = append(out, apiKey{ + Name: ptrStr(d.Name), Username: ptrStr(d.Username), AccessKeyID: ptrStr(d.AccessKeyID), + Status: ptrStr(d.Status), CreatedAt: ptrStr(d.CreatedAt), Access: ptrStr(d.Access), + StorageClass: b.StorageClass, Site: b.Site, Project: b.ProjectRef(), + Buckets: []string{b.BucketName}, + }) + } + } + sortAPIKeys(out) + return out, nil +} + +// listProjects lists the keys of several projects, concatenated. +func (a *keysAPI) listProjects(ctx context.Context, projects []string) ([]apiKey, error) { + var out []apiKey + for _, p := range projects { + keys, err := a.list(ctx, p) + if err != nil { + return nil, err + } + out = append(out, keys...) + } + return out, nil +} + +// delete removes a key. site is only sent for high_performance keys (the API +// ignores it for standard ones). +func (a *keysAPI) delete(ctx context.Context, username, storageClass, project, site string) error { + if username == "" { + return exitcode.Errorf(exitcode.Generic, "the access key has no username; the API needs it to delete the key — run 'lsh s3 access-keys list --project %s' to refresh it", project) + } + if project == "" { + return exitcode.Errorf(exitcode.Usage, "a project is required to delete an access key; pass --project") + } + var region *string + if storageClass == objectstorage.ClassHighPerformance { + if site == "" { + return exitcode.Errorf(exitcode.Usage, "--region is required to delete a high_performance key (a Latitude site slug such as TYO4)") + } + region = &site + } + _, err := a.sdk.ObjectStorage.DeleteStorageAccessKeysUsername(ctx, username, operations.PathParamStorageClass(storageClass), project, region, a.opts...) + if err != nil { + return objectstorage.HumanizeAPI(err, fmt.Sprintf("access key %q", username)) + } + return nil +} + +// resolveProjectID returns the proj_ ID for a project reference (ID or +// slug) by looking at the project's buckets; it falls back to the reference +// itself when nothing resolves so callers can still store something useful. +func resolveProjectID(ctx context.Context, r *objectstorage.Resolver, project string) string { + if project == "" || strings.HasPrefix(project, "proj_") || r == nil || r.API == nil { + return project + } + scoped := *r + scoped.Project = project + list, err := scoped.ListBuckets(ctx) + if err != nil { + lsh.LogDebugf("[s3] could not resolve project %q to an ID: %v", project, err) + return project + } + for _, d := range list { + if id := objectstorage.BucketFromData(d).ProjectID; id != "" { + return id + } + } + return project +} + +// teamProjects returns the distinct projects (IDs, with a slug lookup for +// display) that own buckets in the team, or the one given as filter. +func teamProjects(ctx context.Context, r *objectstorage.Resolver) ([]string, map[string]string, error) { + list, err := r.ListBuckets(ctx) + if err != nil { + return nil, nil, err + } + seen := map[string]bool{} + slugs := map[string]string{} + var out []string + for _, d := range list { + b := objectstorage.BucketFromData(d) + ref := b.ProjectID + if ref == "" { + ref = b.ProjectSlug + } + if ref == "" || seen[ref] { + continue + } + seen[ref] = true + out = append(out, ref) + if b.ProjectSlug != "" { + slugs[ref] = b.ProjectSlug + } + } + sort.Strings(out) + return out, slugs, nil +} + +func sortAPIKeys(keys []apiKey) { + sort.SliceStable(keys, func(i, j int) bool { + if keys[i].Project != keys[j].Project { + return keys[i].Project < keys[j].Project + } + if keys[i].Name != keys[j].Name { + return keys[i].Name < keys[j].Name + } + return keys[i].AccessKeyID < keys[j].AccessKeyID + }) +} + +// bucketSpec is one parsed --bucket value: [=rw|readonly]. +type bucketSpec struct { + Token string + Permission string +} + +// bucketSpecUsage documents the --bucket grammar once for every command +// that accepts it (create, import). +const bucketSpecUsage = "bucket the key covers, as [=rw|readonly] (':' also separates; 'write' = rw, 'ro'/'read' = readonly; default rw; repeatable or comma-separated; a bucket, never s3://bucket/key)" + +// parseBucketSpec accepts "name", "name=rw", "name:readonly" (and "s3://" +// prefixes). The permission defaults to rw. Anything with a key part +// ("bucket/dir/file") is refused: keys grant access to whole buckets. +func parseBucketSpec(s string) (bucketSpec, error) { + raw := strings.TrimSpace(s) + raw = strings.TrimPrefix(raw, "s3://") + raw = strings.TrimSuffix(raw, "/") + if raw == "" { + return bucketSpec{}, exitcode.Errorf(exitcode.Usage, "empty --bucket value") + } + if strings.Contains(raw, "/") { + return bucketSpec{}, exitcode.Errorf(exitcode.Usage, "invalid --bucket %q: access keys cover whole buckets, not keys; pass the bucket only ([=rw|readonly])", s) + } + token, perm := raw, config.PermissionRW + if i := strings.LastIndexAny(raw, "=:"); i >= 0 { + token, perm = raw[:i], strings.ToLower(strings.TrimSpace(raw[i+1:])) + } + token = strings.TrimSpace(token) + if token == "" { + return bucketSpec{}, exitcode.Errorf(exitcode.Usage, "invalid --bucket %q: missing bucket name", s) + } + switch perm { + case config.PermissionRW, "readwrite", "read-write", "write": + perm = config.PermissionRW + case config.PermissionReadOnly, "ro", "read-only", "read": + perm = config.PermissionReadOnly + default: + return bucketSpec{}, exitcode.Errorf(exitcode.Usage, "invalid permission %q in --bucket %q (use =rw or =readonly)", perm, s) + } + return bucketSpec{Token: token, Permission: perm}, nil +} + +// parseBucketSpecs parses every --bucket flag, rejecting duplicates. +func parseBucketSpecs(values []string) ([]bucketSpec, error) { + out := make([]bucketSpec, 0, len(values)) + seen := map[string]bool{} + for _, v := range values { + // Allow comma-separated lists in one flag too. + for _, part := range strings.Split(v, ",") { + if strings.TrimSpace(part) == "" { + continue + } + spec, err := parseBucketSpec(part) + if err != nil { + return nil, err + } + if seen[spec.Token] { + return nil, exitcode.Errorf(exitcode.Usage, "bucket %q given more than once in --bucket", spec.Token) + } + seen[spec.Token] = true + out = append(out, spec) + } + } + return out, nil +} + +// scopedBucket is a resolved bucket plus the permission requested on it. +type scopedBucket struct { + Bucket *objectstorage.Bucket + Permission string +} + +// validateSameGroup makes sure every bucket shares storage class and +// project, and (for high_performance) site. It returns exit 2 listing the +// groups otherwise, since one key cannot span them. +func validateSameGroup(buckets []scopedBucket) error { + if len(buckets) < 2 { + return nil + } + groups := map[string][]string{} + var order []string + for _, sb := range buckets { + b := sb.Bucket + key := fmt.Sprintf("class=%s project=%s", b.StorageClass, firstNonEmptyStr(b.ProjectID, b.ProjectSlug)) + if b.StorageClass == objectstorage.ClassHighPerformance { + key += " site=" + strings.ToUpper(b.Site) + } + if _, ok := groups[key]; !ok { + order = append(order, key) + } + groups[key] = append(groups[key], b.Name) + } + if len(groups) < 2 { + return nil + } + lines := make([]string, 0, len(order)) + for _, k := range order { + lines = append(lines, fmt.Sprintf(" %s: %s", k, strings.Join(groups[k], ", "))) + } + return exitcode.Errorf(exitcode.Usage, "one access key cannot cover buckets of different storage classes, sites or projects:\n%s\ncreate one key per group", strings.Join(lines, "\n")) +} + +var keyNameUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) +var keyNameDashes = regexp.MustCompile(`-{2,}`) + +// sanitizeKeyPart lower-cases s and replaces anything outside [a-z0-9-] +// with '-', collapsing runs. +func sanitizeKeyPart(s string) string { + s = keyNameUnsafe.ReplaceAllString(strings.ToLower(strings.TrimSpace(s)), "-") + s = keyNameDashes.ReplaceAllString(s, "-") + return strings.Trim(s, "-") +} + +// maxAccessKeyNameLen is the API limit on an access key name. +const maxAccessKeyNameLen = 25 + +// capKeyName truncates a name to the API limit and drops a trailing hyphen. +func capKeyName(s string) string { + if len(s) > maxAccessKeyNameLen { + s = s[:maxAccessKeyNameLen] + } + return strings.Trim(s, "-") +} + +// validateAccessKeyName rejects an explicit --name that the API would refuse, +// before the request is sent, with an actionable usage error. +func validateAccessKeyName(name string) error { + if len(name) > maxAccessKeyNameLen { + return exitcode.Errorf(exitcode.Usage, "access key name %q is %d characters; the API allows at most %d", name, len(name), maxAccessKeyNameLen) + } + return nil +} + +// discardUnsavedKey deletes a key that was created but could not be stored in +// the active profile. The API returns the secret exactly once, so a key whose +// secret nobody holds is unusable and would linger with live permissions: +// removing it is the recovery path that neither discloses the secret nor +// leaves a credential behind. Callers fall back to printing the secret (on +// stderr) only when this fails, or when the user asked for it explicitly. +// discardKey is the indirection tests replace; production always deletes +// through the API. +var discardKey = discardUnsavedKey + +func discardUnsavedKey(ctx context.Context, req accessKeyRequest, created *createdAccessKey) error { + if created == nil || created.Username == "" { + return exitcode.Errorf(exitcode.Generic, "the API did not return the key's username, which is required to delete it") + } + return newKeysAPI().delete(ctx, created.Username, req.StorageClass, req.Project, req.Site) +} + +// reportUnsavedKey is the shared tail of a failed save: it removes the key when +// it can, and otherwise prints the only copy of the secret (always on stderr, +// never on stdout) together with the command that stores it. showSecret skips +// the removal because the user asked to keep and print the key. +func reportUnsavedKey(ctx context.Context, w io.Writer, req accessKeyRequest, created *createdAccessKey, plan *createPlan, name string, showSecret bool, saveErr error) { + if !showSecret { + if delErr := discardKey(ctx, req, created); delErr == nil { + fmt.Fprintf(w, "warning: access key %q was created but could not be saved to the profile: %v\n", name, saveErr) + fmt.Fprintf(w, "It was deleted again, so nothing was left behind and no secret was printed. Retry once the profile is writable.\n") + return + } else { + fmt.Fprintf(w, "warning: access key %q could not be saved (%v) nor deleted again (%v), so its secret is printed below — it cannot be retrieved another way.\n", name, saveErr, delErr) + } + } else { + fmt.Fprintf(w, "warning: access key %q was created but could not be saved to the profile: %v\n", name, saveErr) + } + printCreatedHuman(w, newCreatedKeyView(created, plan, true), true) + fmt.Fprintf(w, "Save it once the profile is writable with: %s\n", importHint(name, created, req, created.Buckets)) +} + +// importHint renders the `access-keys import` command that stores a key whose +// secret could not be saved automatically. buckets maps bucket display names +// to permissions and is only used for a limited_access key. +func importHint(name string, created *createdAccessKey, req accessKeyRequest, buckets map[string]string) string { + hint := fmt.Sprintf("lsh s3 access-keys import --name %s --access-key-id %s --project %s --storage-class %s", + name, created.AccessKeyID, req.Project, req.StorageClass) + if req.StorageClass == objectstorage.ClassHighPerformance && req.Site != "" { + hint += " --region " + req.Site + } + if req.Scope == config.ScopeLimitedAccess && len(buckets) > 0 { + names := make([]string, 0, len(buckets)) + for b := range buckets { + names = append(names, b) + } + sort.Strings(names) + for _, b := range names { + hint += fmt.Sprintf(" --bucket %s=%s", b, buckets[b]) + } + return hint + } + return hint + " --all-buckets" +} + +// secretDecision says whether the freshly created secret may be printed and +// which warning (if any) explains an omission. +type secretDecision struct { + Show bool + Warning string +} + +// decideSecretDisplay implements the display rules of the plan (J3): +// - human output, not saving: the secret is printed once in clear; +// - saving (--save): only with --show-secret, in every format; +// - structured output: only when -o was passed explicitly or --show-secret +// (never because of LSH_OUTPUT/config), otherwise omitted with a warning. +func decideSecretDisplay(human, save, showSecret, outputExplicit bool) secretDecision { + switch { + case showSecret: + return secretDecision{Show: true} + case save: + return secretDecision{Warning: "secret omitted: it was saved in the profile; pass --show-secret to print it"} + case human: + return secretDecision{Show: true} + case outputExplicit: + return secretDecision{Show: true} + default: + return secretDecision{Warning: "secret omitted: pass -o json explicitly or --show-secret"} + } +} + +// keyRow renders one API access key (list/get). +type keyRow struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + Scope string `json:"scope"` + Access string `json:"access,omitempty"` + Buckets []string `json:"buckets,omitempty"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project,omitempty"` + Status string `json:"status,omitempty"` + Username string `json:"username,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + Saved bool `json:"saved"` + SavedAs string `json:"saved_as,omitempty"` +} + +// newKeyRow builds the row, marking it saved when a profile key has the +// same access key ID. +func newKeyRow(k apiKey, saved map[string]string, slugs map[string]string) keyRow { + project := k.Project + if s, ok := slugs[project]; ok && s != "" { + project = s + } + row := keyRow{ + Name: k.Name, AccessKeyID: k.AccessKeyID, Scope: k.scope(), Access: k.Access, Buckets: k.Buckets, + StorageClass: k.StorageClass, Site: strings.ToUpper(k.Site), Project: project, Status: k.Status, + Username: k.Username, CreatedAt: k.CreatedAt, + } + if name, ok := saved[k.AccessKeyID]; ok { + row.Saved, row.SavedAs = true, name + } + return row +} + +func (r keyRow) TableRow() table.Row { + return table.Row{ + "name": {Label: "Name", Value: r.Name}, + "access_key_id": {Label: "Access Key ID", Value: r.AccessKeyID}, + "scope": {Label: "Scope", Value: r.scopeCell()}, + "buckets": {Label: "Buckets", Value: r.bucketsCell(), MaxLength: 40}, + "storage_class": {Label: "Class", Value: r.StorageClass}, + "region": {Label: "Site", Value: dash(r.Site)}, + "project": {Label: "Project", Value: dash(r.Project)}, + "status": {Label: "Status", Value: dash(r.Status)}, + "saved": {Label: "Saved", Value: yesNo(r.Saved)}, + } +} + +// scopeCell is the SCOPE column: the raw access (fullaccess|rw|readonly) or '-'. +func (r keyRow) scopeCell() string { return dash(r.Access) } + +// bucketsCell is the BUCKETS column: "all" for fullaccess, the backend names +// otherwise (truncated). +func (r keyRow) bucketsCell() string { + if r.Access == config.ScopeFullAccess { + return "all" + } + if len(r.Buckets) == 0 { + return emptyCell + } + return truncateText(strings.Join(r.Buckets, ", "), 40) +} + +// savedKeyRow renders one saved (profile) key; never the secret. +type savedKeyRow struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + ProjectID string `json:"project_id,omitempty"` + Scope string `json:"scope"` + Buckets map[string]string `json:"buckets,omitempty"` + Username string `json:"username,omitempty"` + Source string `json:"source,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + Profile string `json:"profile,omitempty"` + // API is "ok", "missing on API" or "" when no listing was available. + API string `json:"api_status,omitempty"` +} + +func newSavedKeyRow(name string, k config.StoredAccessKey, profile string) savedKeyRow { + created := "" + if !k.CreatedAt.IsZero() { + created = k.CreatedAt.UTC().Format(time.RFC3339) + } + return savedKeyRow{ + Name: name, AccessKeyID: k.AccessKeyID, StorageClass: k.StorageClass, Site: strings.ToUpper(k.Site), + ProjectID: k.ProjectID, Scope: k.Scope, Buckets: k.Buckets, Username: k.Username, Source: k.Source, + CreatedAt: created, Profile: profile, + } +} + +func (r savedKeyRow) TableRow() table.Row { + return table.Row{ + "name": {Label: "Name", Value: r.Name}, + "access_key_id": {Label: "Access Key ID", Value: r.AccessKeyID}, + "storage_class": {Label: "Class", Value: dash(r.StorageClass)}, + "region": {Label: "Site", Value: dash(r.Site)}, + "project": {Label: "Project", Value: dash(r.ProjectID)}, + "scope": {Label: "Scope", Value: dash(r.Scope)}, + "buckets": {Label: "Buckets", Value: r.bucketsCell(), MaxLength: 40}, + "source": {Label: "Source", Value: dash(r.Source)}, + "created_at": {Label: "Created", Value: dash(r.CreatedAt)}, + "status": {Label: "API", Value: dash(r.API)}, + } +} + +// bucketsCell renders "bkt_a=rw bkt_b=readonly" (sorted, truncated). +func (r savedKeyRow) bucketsCell() string { + if r.Scope == config.ScopeFullAccess { + return "all" + } + if len(r.Buckets) == 0 { + return emptyCell + } + return truncateText(formatPerms(r.Buckets), 40) +} + +// formatPerms renders a name→permission map as "a=rw b=readonly", sorted. +func formatPerms(perms map[string]string) string { + ids := make([]string, 0, len(perms)) + for id := range perms { + ids = append(ids, id) + } + sort.Strings(ids) + parts := make([]string, 0, len(ids)) + for _, id := range ids { + parts = append(parts, id+"="+perms[id]) + } + return strings.Join(parts, " ") +} + +// savedKeysByID indexes the active profile's keys by access key ID. Errors +// (no profile) yield an empty map so listings still work under LATITUDESH_TOKEN. +func savedKeysByID(cmd *cobra.Command) (map[string]string, string, map[string]config.StoredAccessKey) { + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return map[string]string{}, "", map[string]config.StoredAccessKey{} + } + keys := p.ObjectStorageKeys() + byID := make(map[string]string, len(keys)) + for name, k := range keys { + if k.AccessKeyID != "" { + byID[k.AccessKeyID] = name + } + } + return byID, profileName, keys +} + +// truncateText shortens s to max runes with an ellipsis. +func truncateText(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + if max <= 1 { + return "…" + } + return string(r[:max-1]) + "…" +} + +// dash is the access-keys spelling of the shared table placeholder. +func dash(s string) string { return orEmptyCell(s) } + +func ptrStr(s *string) string { + if s == nil { + return "" + } + return *s +} + +func firstNonEmptyStr(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +// storageClassFlag validates --storage-class. +func storageClassFlag(cmd *cobra.Command) (string, error) { + v, _ := cmd.Flags().GetString("storage-class") + v = strings.ToLower(strings.TrimSpace(v)) + switch v { + case "", objectstorage.ClassStandard, objectstorage.ClassHighPerformance: + return v, nil + case "hp", "high-performance", "highperformance": + return objectstorage.ClassHighPerformance, nil + } + return "", exitcode.Errorf(exitcode.Usage, "invalid --storage-class %q (use standard or high_performance)", v) +} + +// regionFlag returns --region upper-cased, rejecting AWS-looking values. +func regionFlag(cmd *cobra.Command) (string, error) { + v, _ := cmd.Flags().GetString("region") + v = strings.TrimSpace(v) + if v == "" { + return "", nil + } + if looksLikeAWSRegion(v) { + return "", exitcode.Errorf(exitcode.Usage, "--region %q is not a Latitude site; use a site slug such as DAL, NYC or TYO4 (see 'lsh regions list')", v) + } + return strings.ToUpper(v), nil +} diff --git a/cmd/s3/accesskeys_shared_test.go b/cmd/s3/accesskeys_shared_test.go new file mode 100644 index 0000000..707b215 --- /dev/null +++ b/cmd/s3/accesskeys_shared_test.go @@ -0,0 +1,457 @@ +package s3 + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func TestParseBucketSpec(t *testing.T) { + cases := []struct { + in string + token string + perm string + fail bool + }{ + {"backups", "backups", config.PermissionRW, false}, + {"backups=rw", "backups", config.PermissionRW, false}, + {"logs=readonly", "logs", config.PermissionReadOnly, false}, + {"logs:readonly", "logs", config.PermissionReadOnly, false}, + {"logs:ro", "logs", config.PermissionReadOnly, false}, + {"s3://backups/", "backups", config.PermissionRW, false}, + {"bkt_1Gjbang9n0L2w=readonly", "bkt_1Gjbang9n0L2w", config.PermissionReadOnly, false}, + {"backups=write", "backups", config.PermissionRW, false}, + {"backups=admin", "", "", true}, + {"=rw", "", "", true}, + {"", "", "", true}, + // A key part is a usage error: access keys cover whole buckets. + {"backups/2026/", "", "", true}, + {"s3://backups/dump.sql=rw", "", "", true}, + {"backups/dir:readonly", "", "", true}, + } + for _, c := range cases { + spec, err := parseBucketSpec(c.in) + if c.fail { + if err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("%q: want usage error, got spec=%+v err=%v", c.in, spec, err) + } + continue + } + if err != nil { + t.Errorf("%q: unexpected error %v", c.in, err) + continue + } + if spec.Token != c.token || spec.Permission != c.perm { + t.Errorf("%q: got %+v, want token=%q perm=%q", c.in, spec, c.token, c.perm) + } + } +} + +func TestParseBucketSpecsRejectsDuplicatesAndSplitsCommas(t *testing.T) { + specs, err := parseBucketSpecs([]string{"a=rw,b=readonly", "c"}) + if err != nil || len(specs) != 3 { + t.Fatalf("got %+v err=%v", specs, err) + } + if specs[1].Token != "b" || specs[1].Permission != config.PermissionReadOnly { + t.Fatalf("comma split failed: %+v", specs) + } + if _, err := parseBucketSpecs([]string{"a", "a=readonly"}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("duplicate must fail with exit 2, got %v", err) + } +} + +func TestValidateSameGroup(t *testing.T) { + std := func(name, project string) scopedBucket { + return scopedBucket{Bucket: &objectstorage.Bucket{Name: name, StorageClass: objectstorage.ClassStandard, ProjectID: project, Site: "DAL"}, Permission: "rw"} + } + hp := func(name, project, site string) scopedBucket { + return scopedBucket{Bucket: &objectstorage.Bucket{Name: name, StorageClass: objectstorage.ClassHighPerformance, ProjectID: project, Site: site}, Permission: "rw"} + } + if err := validateSameGroup([]scopedBucket{std("a", "p1"), std("b", "p1")}); err != nil { + t.Fatalf("same class/project must pass: %v", err) + } + // Standard keys span sites: differing sites are fine. + other := std("c", "p1") + other.Bucket.Site = "NYC" + if err := validateSameGroup([]scopedBucket{std("a", "p1"), other}); err != nil { + t.Fatalf("standard buckets in different sites must pass: %v", err) + } + err := validateSameGroup([]scopedBucket{std("a", "p1"), hp("b", "p1", "TYO4")}) + if err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("mixed classes must fail with exit 2, got %v", err) + } + if !strings.Contains(err.Error(), "class=standard") || !strings.Contains(err.Error(), "class=high_performance") || !strings.Contains(err.Error(), "a") { + t.Fatalf("error must list the groups: %v", err) + } + if err := validateSameGroup([]scopedBucket{hp("a", "p1", "TYO4"), hp("b", "p1", "DAL")}); err == nil { + t.Fatal("high_performance buckets in different sites must fail") + } + if err := validateSameGroup([]scopedBucket{std("a", "p1"), std("b", "p2")}); err == nil { + t.Fatal("different projects must fail") + } +} + +func TestNormalizeCreatedBothShapes(t *testing.T) { + req := accessKeyRequest{Project: "proj_1", StorageClass: objectstorage.ClassStandard, Site: "DAL", Name: "ci", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw"}} + s := func(v string) *string { return &v } + wasabi := &operations.PostStorageAccessKeysResponse{Object: &operations.PostStorageAccessKeysResponseBody{Data: &operations.PostStorageAccessKeysObjectStorageData{ + Attributes: &operations.PostStorageAccessKeysObjectStorageAttributes{AccessKey: &operations.AccessKey{ + AccessKeyID: s("WASABIID"), SecretAccessKey: s("wasabi-secret"), Name: s("ci"), Status: s("Active"), Username: s("u+ci@x"), + }}, + }}} + k, err := normalizeCreated(wasabi, req) + if err != nil { + t.Fatal(err) + } + if k.AccessKeyID != "WASABIID" || k.SecretAccessKey != "wasabi-secret" || k.Username != "u+ci@x" || k.Scope != config.ScopeLimitedAccess || k.Buckets["bkt_1"] != "rw" { + t.Fatalf("wasabi shape not normalized: %+v", *k) + } + + req.StorageClass = objectstorage.ClassHighPerformance + req.Site = "TYO4" + vast := &operations.PostStorageAccessKeysResponse{Object: &operations.PostStorageAccessKeysResponseBody{Data: &operations.PostStorageAccessKeysObjectStorageData{ + Attributes: &operations.PostStorageAccessKeysObjectStorageAttributes{AccessKey: &operations.AccessKey{ + AccessKey: s("VASTID"), SecretKey: s("vast-secret"), Status: s("Active"), + }}, + }}} + k, err = normalizeCreated(vast, req) + if err != nil { + t.Fatal(err) + } + if k.AccessKeyID != "VASTID" || k.SecretAccessKey != "vast-secret" || k.Name != "ci" || k.Site != "TYO4" { + t.Fatalf("vast shape not normalized: %+v", *k) + } + // Both shapes marshal to identical field names. + b, _ := json.Marshal(k) + for _, field := range []string{`"access_key_id"`, `"secret_access_key"`, `"storage_class"`, `"scope"`} { + if !strings.Contains(string(b), field) { + t.Fatalf("JSON missing %s: %s", field, b) + } + } + if strings.Contains(string(b), `"secret_key"`) || strings.Contains(string(b), `"access_key":`) { + t.Fatalf("backend field names leaked into JSON: %s", b) + } + + if _, err := normalizeCreated(&operations.PostStorageAccessKeysResponse{}, req); err == nil { + t.Fatal("empty response must fail") + } + noSecret := &operations.PostStorageAccessKeysResponse{Object: &operations.PostStorageAccessKeysResponseBody{Data: &operations.PostStorageAccessKeysObjectStorageData{ + Attributes: &operations.PostStorageAccessKeysObjectStorageAttributes{AccessKey: &operations.AccessKey{AccessKeyID: s("X")}}, + }}} + if _, err := normalizeCreated(noSecret, req); err == nil { + t.Fatal("response without secret must fail") + } +} + +func TestBuildCreateRequestSortsBuckets(t *testing.T) { + req := accessKeyRequest{Project: "p", StorageClass: objectstorage.ClassStandard, Site: "DAL", Name: "n", Scope: config.ScopeLimitedAccess, + Buckets: map[string]string{"bkt_b": "readonly", "bkt_a": "rw"}} + body := buildCreateRequest(req) + a := body.Data.Attributes + if a.AccessScope != operations.AccessScopeLimitedAccess || a.Region != "DAL" || len(a.BucketPermissions) != 2 { + t.Fatalf("unexpected attributes: %+v", a) + } + if a.BucketPermissions[0].BucketID != "bkt_a" || a.BucketPermissions[0].Permission != operations.PermissionRw || a.BucketPermissions[1].Permission != operations.PermissionReadonly { + t.Fatalf("permissions not sorted/mapped: %+v", a.BucketPermissions) + } + req.Scope = config.ScopeFullAccess + if body := buildCreateRequest(req); len(body.Data.Attributes.BucketPermissions) != 0 { + t.Fatal("fullaccess must not send bucket_permissions") + } +} + +func TestAccessKeyRequestValidate(t *testing.T) { + base := accessKeyRequest{Project: "p", StorageClass: objectstorage.ClassStandard, Site: "DAL", Name: "n", Scope: config.ScopeFullAccess} + if err := base.validate(); err != nil { + t.Fatalf("valid request rejected: %v", err) + } + hp := base + hp.StorageClass = objectstorage.ClassHighPerformance + hp.Site = "" + err := hp.validate() + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "TYO4") { + t.Fatalf("high_performance without site must name the expected value, got %v", err) + } + limited := base + limited.Scope = config.ScopeLimitedAccess + if err := limited.validate(); err == nil { + t.Fatal("limited_access without buckets must fail") + } + limited.Buckets = map[string]string{"bkt_1": "admin"} + if err := limited.validate(); err == nil { + t.Fatal("invalid permission must fail") + } +} + +func TestDecideSecretDisplay(t *testing.T) { + cases := []struct { + human, save, show, explicit bool + wantShow bool + wantWarn bool + }{ + {human: true, wantShow: true}, // TTY, not saving: shown once in clear + {human: true, save: true, wantShow: false, wantWarn: true}, // saved: not printed + {human: true, save: true, show: true, wantShow: true}, // saved + --show-secret + {human: false, explicit: true, wantShow: true}, // -o json explicit + {human: false, explicit: false, wantShow: false, wantWarn: true}, // LSH_OUTPUT=json: omitted + warning + {human: false, explicit: false, show: true, wantShow: true}, // --show-secret overrides + {human: false, save: true, explicit: true, wantShow: false, wantWarn: true}, // saving wins over explicit -o + {human: false, save: true, show: true, wantShow: true}, + } + for i, c := range cases { + d := decideSecretDisplay(c.human, c.save, c.show, c.explicit) + if d.Show != c.wantShow || (d.Warning != "") != c.wantWarn { + t.Errorf("case %d (%+v): got show=%v warn=%q", i, c, d.Show, d.Warning) + } + } +} + +func TestTruncateAndFormatHelpers(t *testing.T) { + if got := truncateText("abcdef", 4); got != "abc…" { + t.Fatalf("truncateText = %q", got) + } + if got := truncateText("abc", 4); got != "abc" { + t.Fatalf("truncateText short = %q", got) + } + if got := formatPerms(map[string]string{"bkt_b": "rw", "bkt_a": "readonly"}); got != "bkt_a=readonly bkt_b=rw" { + t.Fatalf("formatPerms = %q", got) + } + if !looksLikeAWSRegion("us-east-1") || looksLikeAWSRegion("TYO4") || looksLikeAWSRegion("DAL") { + t.Fatal("AWS region detection is off") + } +} + +func TestRotatedName(t *testing.T) { + now := mustTime(t, "2026-09-07T12:00:00Z") + if got := rotatedName("ci-deploy", now, 1); got != "ci-deploy-20260907" { + t.Fatalf("rotatedName = %q", got) + } + req, err := rotateRequest(config.StoredAccessKey{ + AccessKeyID: "OLD", StorageClass: objectstorage.ClassHighPerformance, Site: "tyo4", ProjectID: "proj_1", + Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw"}, + }, "", "", "ci-deploy-20260907") + if err != nil { + t.Fatal(err) + } + if req.Project != "proj_1" || req.Site != "TYO4" || req.Scope != config.ScopeLimitedAccess || req.Buckets["bkt_1"] != "rw" || req.Name != "ci-deploy-20260907" { + t.Fatalf("rotateRequest = %+v", req) + } + if _, err := rotateRequest(config.StoredAccessKey{Scope: config.ScopeUnknown}, "p", "", "n"); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("unknown scope must fail with exit 2, got %v", err) + } + if _, err := rotateRequest(config.StoredAccessKey{Scope: config.ScopeFullAccess, StorageClass: "standard"}, "", "", "n"); err == nil { + t.Fatal("missing project must fail") + } +} + +func TestValidateAccessKeyName(t *testing.T) { + if err := validateAccessKeyName("ci-deploy"); err != nil { + t.Errorf("valid name rejected: %v", err) + } + if err := validateAccessKeyName("this-name-is-way-too-long-for-the-api"); err == nil { + t.Error("over-long name must be rejected") + } +} + +// lagError reproduces what the API returns while the backend converges: +// 404 STORAGE_RESOURCE_NOT_FOUND, humanized by objectstorage.HumanizeAPI. +func lagError() error { + return exitcode.Errorf(exitcode.NotFound, "not found: Storage resource not found.") +} + +func TestIsProvisioningLag(t *testing.T) { + cases := []struct { + err error + want bool + }{ + {lagError(), true}, + {exitcode.Errorf(exitcode.Generic, "the API returned 502: Storage service is temporarily unavailable. Please try again."), true}, + {exitcode.Errorf(exitcode.Usage, "the API rejected the request: name size cannot be greater than 25"), false}, + {exitcode.Errorf(exitcode.NotFound, "project %q not found", "nope"), false}, + } + for _, c := range cases { + if got := isProvisioningLag(c.err); got != c.want { + t.Errorf("isProvisioningLag(%q) = %v, want %v", c.err, got, c.want) + } + } +} + +// TestRetryCreateProvisioningLag covers the fix for a key created right after +// its bucket: the first attempts fail with the propagation 404 and the same +// request is re-sent (same name) until it succeeds. +func TestRetryCreateProvisioningLag(t *testing.T) { + restore := provisioningRetryDelays + provisioningRetryDelays = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond} + defer func() { provisioningRetryDelays = restore }() + + var names []string + calls := 0 + req := accessKeyRequest{Name: "key-brave-otter-std", StorageClass: objectstorage.ClassStandard} + created, err := retryCreate(context.Background(), req, true, func(r accessKeyRequest) (*createdAccessKey, error) { + calls++ + names = append(names, r.Name) + if calls < 3 { + return nil, lagError() + } + return &createdAccessKey{Name: r.Name}, nil + }) + if err != nil { + t.Fatalf("retryCreate: %v", err) + } + if created.Name != "key-brave-otter-std" { + t.Errorf("created name = %q, want the requested one", created.Name) + } + if calls != 3 { + t.Errorf("calls = %d, want 3", calls) + } + for _, n := range names { + if n != "key-brave-otter-std" { + t.Errorf("name changed to %q; a propagation retry must re-send the same request", n) + } + } +} + +func TestRetryCreateProvisioningLagExhausted(t *testing.T) { + restore := provisioningRetryDelays + provisioningRetryDelays = []time.Duration{time.Millisecond, time.Millisecond} + defer func() { provisioningRetryDelays = restore }() + + calls := 0 + _, err := retryCreate(context.Background(), accessKeyRequest{Name: "key-x"}, true, func(accessKeyRequest) (*createdAccessKey, error) { + calls++ + return nil, lagError() + }) + if err == nil { + t.Fatal("expected an error once the retries are exhausted") + } + if calls != 3 { + t.Errorf("calls = %d, want 3 (initial + 2 retries)", calls) + } + if code := exitcode.Of(err); code != exitcode.NotFound { + t.Errorf("exit code = %d, want %d", code, exitcode.NotFound) + } + if !strings.Contains(err.Error(), "retry in a few seconds") { + t.Errorf("error %q should tell the user the failure is transient", err) + } +} + +func TestRetryCreateNameConflict(t *testing.T) { + calls := 0 + var names []string + req := accessKeyRequest{Name: "key-taken", StorageClass: objectstorage.ClassStandard} + if _, err := retryCreate(context.Background(), req, true, func(r accessKeyRequest) (*createdAccessKey, error) { + calls++ + names = append(names, r.Name) + if calls < 2 { + return nil, exitcode.Errorf(exitcode.Usage, "the API rejected the request: name has already been taken") + } + return &createdAccessKey{Name: r.Name}, nil + }); err != nil { + t.Fatalf("retryCreate: %v", err) + } + if len(names) != 2 || names[0] == names[1] { + t.Errorf("names = %v; a conflict must re-roll the generated name", names) + } + + // An explicit --name is never rewritten: the conflict surfaces instead. + calls = 0 + if _, err := retryCreate(context.Background(), req, false, func(accessKeyRequest) (*createdAccessKey, error) { + calls++ + return nil, exitcode.Errorf(exitcode.Usage, "the API rejected the request: name has already been taken") + }); err == nil { + t.Fatal("expected the conflict to surface for an explicit name") + } + if calls != 1 { + t.Errorf("calls = %d, want 1", calls) + } +} + +func TestSleepCtxCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := sleepCtx(ctx, time.Hour) + if err == nil { + t.Fatal("expected an error for a cancelled context") + } + if code := exitcode.Of(err); code != exitcode.Interrupted { + t.Errorf("exit code = %d, want %d", code, exitcode.Interrupted) + } +} + +func TestImportHint(t *testing.T) { + created := &createdAccessKey{AccessKeyID: "AKIAEXAMPLE"} + limited := accessKeyRequest{Project: "my-project", StorageClass: objectstorage.ClassStandard, Scope: config.ScopeLimitedAccess} + got := importHint("k1", created, limited, map[string]string{"logs": config.PermissionReadOnly, "backups": config.PermissionRW}) + want := "lsh s3 access-keys import --name k1 --access-key-id AKIAEXAMPLE --project my-project --storage-class standard --bucket backups=rw --bucket logs=readonly" + if got != want { + t.Errorf("limited hint =\n%s\nwant\n%s", got, want) + } + + full := accessKeyRequest{Project: "p", StorageClass: objectstorage.ClassHighPerformance, Site: "TYO4", Scope: config.ScopeFullAccess} + got = importHint("k2", created, full, nil) + want = "lsh s3 access-keys import --name k2 --access-key-id AKIAEXAMPLE --project p --storage-class high_performance --region TYO4 --all-buckets" + if got != want { + t.Errorf("fullaccess hint =\n%s\nwant\n%s", got, want) + } +} + +// TestRotatedNameSequenceAndLimit covers same-day re-rotation (a deterministic +// name would collide on the API) and the 25-character name cap. +func TestRotatedNameSequenceAndLimit(t *testing.T) { + now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) + cases := []struct { + base string + seq int + want string + }{ + {"ci-deploy", 1, "ci-deploy-20260907"}, + {"ci-deploy", 2, "ci-deploy-20260907-2"}, + {"", 1, "lsh-key-20260907"}, + // 20 chars + "-20260907" would be 29: the base is trimmed instead. + {"backups-nightly-full", 1, "backups-nightly-20260907"}, + {"backups-nightly-full", 3, "backups-nightl-20260907-3"}, + } + for _, c := range cases { + got := rotatedName(c.base, now, c.seq) + if got != c.want { + t.Errorf("rotatedName(%q, seq=%d) = %q, want %q", c.base, c.seq, got, c.want) + } + if len(got) > maxAccessKeyNameLen { + t.Errorf("rotatedName(%q, seq=%d) = %q is %d chars, over the API limit", c.base, c.seq, got, len(got)) + } + } +} + +// TestMatchSavedFiltersLikeMatchAPI locks the two rules matchSaved was missing: +// an empty site is a wildcard (as in matchAPI) and a project reference only +// filters when it is comparable with what the key stored. +func TestMatchSavedFiltersLikeMatchAPI(t *testing.T) { + standard := config.StoredAccessKey{StorageClass: objectstorage.ClassStandard, ProjectID: "proj_1"} + cases := []struct { + name string + filter listFilter + key config.StoredAccessKey + want bool + }{ + {"no filter", listFilter{}, standard, true}, + {"site filter, key without a site", listFilter{Site: "DAL"}, standard, true}, + {"site filter, key in another site", listFilter{Site: "DAL"}, config.StoredAccessKey{Site: "NYC"}, false}, + {"site filter, same site", listFilter{Site: "DAL"}, config.StoredAccessKey{Site: "dal"}, true}, + {"project slug vs stored ID", listFilter{Project: "my-project"}, standard, true}, + {"project ID mismatch", listFilter{Project: "proj_2"}, standard, false}, + {"project ID match", listFilter{Project: "proj_1"}, standard, true}, + {"class mismatch", listFilter{StorageClass: objectstorage.ClassHighPerformance}, standard, false}, + } + for _, c := range cases { + if got := c.filter.matchSaved(c.key); got != c.want { + t.Errorf("%s: matchSaved = %v, want %v", c.name, got, c.want) + } + } +} diff --git a/cmd/s3/accesskeys_test.go b/cmd/s3/accesskeys_test.go new file mode 100644 index 0000000..b3e1e7f --- /dev/null +++ b/cmd/s3/accesskeys_test.go @@ -0,0 +1,158 @@ +package s3 + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/lsh/internal/config" + homedir "github.com/mitchellh/go-homedir" +) + +// fakeAPI is a minimal Latitude API for the access-keys endpoints. It records +// the requests it receives and answers with canned JSON:API documents. +type fakeAPI struct { + mu sync.Mutex + srv *httptest.Server + requests []fakeRequest + // createShape is "wasabi" (access_key_id/secret_access_key) or "vast" + // (access_key/secret_key). + createShape string + // keys is the GET /storage/access_keys document keyed by project. + keys map[string]string + // buckets is the GET /storage/buckets document. + buckets string + // deleteStatus is returned by DELETE (204 by default). + deleteStatus int +} + +type fakeRequest struct { + Method string + Path string + Query url.Values + Body string +} + +func newFakeAPI(t *testing.T) *fakeAPI { + t.Helper() + f := &fakeAPI{createShape: "wasabi", keys: map[string]string{}, deleteStatus: http.StatusNoContent} + f.srv = httptest.NewServer(http.HandlerFunc(f.handle)) + t.Cleanup(f.srv.Close) + return f +} + +// client builds a keysAPI whose SDK client and raw GET both point at the fake +// server. +func (f *fakeAPI) client() *keysAPI { + return &keysAPI{ + sdk: sdk.New(sdk.WithServerURL(f.srv.URL), sdk.WithSecurity("test-token")), + baseURL: f.srv.URL, + token: "test-token", + } +} + +func (f *fakeAPI) Requests() []fakeRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]fakeRequest(nil), f.requests...) +} + +func (f *fakeAPI) handle(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + f.mu.Lock() + f.requests = append(f.requests, fakeRequest{Method: r.Method, Path: r.URL.Path, Query: r.URL.Query(), Body: string(body)}) + f.mu.Unlock() + w.Header().Set("Content-Type", "application/vnd.api+json") + + switch { + case r.Method == http.MethodPost && r.URL.Path == "/storage/access_keys": + var req struct { + Data struct { + Attributes struct { + Name string `json:"name"` + } `json:"attributes"` + } `json:"data"` + } + _ = json.Unmarshal(body, &req) + name := strings.ToLower(req.Data.Attributes.Name) + w.WriteHeader(http.StatusCreated) + if f.createShape == "vast" { + io.WriteString(w, `{"data":{"type":"access_keys","attributes":{"access_key":{"access_key":"VASTKEYID","secret_key":"vast-secret-value","name":"`+name+`","status":"Active","username":"user_1-`+name+`"}}}}`) + return + } + io.WriteString(w, `{"data":{"type":"access_keys","attributes":{"access_key":{"access_key_id":"WASABIKEYID","secret_access_key":"wasabi-secret-value","name":"`+name+`","status":"Active","username":"someone+`+name+`@example.com"}}}}`) + case r.Method == http.MethodGet && r.URL.Path == "/storage/access_keys": + doc, ok := f.keys[r.URL.Query().Get("project")] + if !ok { + w.WriteHeader(http.StatusNotFound) + io.WriteString(w, `{"errors":[{"status":"404","title":"Not Found","detail":"project not found"}]}`) + return + } + io.WriteString(w, doc) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/storage/access_keys/"): + w.WriteHeader(f.deleteStatus) + if f.deleteStatus == http.StatusNotFound { + io.WriteString(w, `{"errors":[{"status":"404","title":"Not Found","detail":"access key not found"}]}`) + } + case r.Method == http.MethodGet && r.URL.Path == "/storage/buckets": + if f.buckets == "" { + io.WriteString(w, `{"data":[]}`) + return + } + io.WriteString(w, f.buckets) + default: + w.WriteHeader(http.StatusNotFound) + io.WriteString(w, `{"errors":[{"status":"404","title":"Not Found"}]}`) + } +} + +// keysDocument is the canned GET /storage/access_keys payload: one standard +// fullaccess key, one standard limited key and one high_performance key, +// including the buckets[]/access fields the SDK drops. +const keysDocument = `{"data":{ + "standard":[ + {"name":"ops","username":"ops@example.com","access_key_id":"OPSKEYID000000000001","status":"Active","created_at":"2026-09-01T10:00:00Z","region":"DAL","access":"fullaccess","buckets":["backups-7f3a","logs-91aa"]}, + {"name":"ci-deploy","username":"ci@example.com","access_key_id":"CIKEYID0000000000002","status":"Active","created_at":"2026-09-02T10:00:00Z","region":"DAL","access":"rw","buckets":["backups-7f3a"]} + ], + "high_performance":[ + {"name":"fast","username":"user_1-fast","access_key_id":"HPKEYID0000000000003","status":"Active","created_at":"2026-09-03T10:00:00Z","region":"TYO4","access":"readonly","buckets":["fast-bucket-01"]} + ]}}` + +// withTempProfile points the config file at a temporary HOME holding one +// profile named "test" with the given saved keys, and returns the directory. +func withTempProfile(t *testing.T, keys map[string]config.StoredAccessKey) string { + t.Helper() + dir := t.TempDir() + homedir.DisableCache = true + t.Setenv("HOME", dir) + t.Setenv("LSH_PROFILE", "") + t.Setenv("LATITUDESH_TOKEN", "") + f := &config.File{ + DefaultProfile: "test", + Profiles: map[string]config.Profile{ + "test": {Authorization: "tok", Email: "Lanusse.Morais@latitude.sh", ObjectStorage: &config.ObjectStorageConfig{Keys: keys}}, + }, + } + if err := config.Save(f); err != nil { + t.Fatalf("save config: %v", err) + } + return dir +} + +// readConfigRaw returns the on-disk config so tests can assert on it. +func readConfigRaw(t *testing.T, home string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(home, ".config", "lsh", "config.json")) + if err != nil { + t.Fatalf("read config: %v", err) + } + return string(b) +} diff --git a/cmd/s3/bucket_render.go b/cmd/s3/bucket_render.go new file mode 100644 index 0000000..4292344 --- /dev/null +++ b/cmd/s3/bucket_render.go @@ -0,0 +1,118 @@ +package s3 + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" +) + +// BucketRow wraps the SDK bucket payload so the shared renderer can print it +// (-o table|json|yaml|csv, --query). JSON output is the API document as-is. +type BucketRow struct { + components.ObjectStorageData + // Site is the Latitude site slug when known (the SDK model drops it). + Site string `json:"-"` + // resolved is set when the row came from a resolved bucket. It carries the + // fields that exist without an API document (endpoint-override mode). + resolved *objectstorage.Bucket +} + +// MarshalJSON emits the API document plus the site slug (which the SDK model +// drops) so `-o json` carries everything `stat` shows. +func (m *BucketRow) MarshalJSON() ([]byte, error) { + raw, err := json.Marshal(m.ObjectStorageData) + if err != nil { + return nil, err + } + var doc map[string]json.RawMessage + if unmarshalErr := json.Unmarshal(raw, &doc); unmarshalErr != nil { + doc = nil + } + if len(doc) == 0 && m.resolved != nil { + // No API payload (endpoint-override mode): describe the bucket from + // what was resolved locally instead of emitting an empty object. + return json.Marshal(map[string]string{ + "name": m.resolved.Name, + "bucket_name": m.resolved.BucketName, + "endpoint": m.resolved.Endpoint, + "signing_region": m.resolved.SigningRegion, + }) + } + if m.Site == "" || doc == nil { + return raw, nil + } + site, _ := json.Marshal(m.Site) + doc["site"] = site + return json.Marshal(doc) +} + +// NewBucketRow builds a row from a resolved bucket. +func NewBucketRow(b *objectstorage.Bucket) *BucketRow { + return &BucketRow{ObjectStorageData: b.Data, Site: b.Site, resolved: b} +} + +// BucketRows converts SDK payloads into renderable rows. +func BucketRows(data []components.ObjectStorageData, sites map[string]string) []renderer.ResponseData { + out := make([]renderer.ResponseData, 0, len(data)) + for i := range data { + row := &BucketRow{ObjectStorageData: data[i]} + if data[i].ID != nil && sites != nil { + row.Site = sites[*data[i].ID] + } + out = append(out, row) + } + return out +} + +func (m *BucketRow) TableRow() table.Row { + b := objectstorage.BucketFromData(m.ObjectStorageData) + site := m.Site + if site == "" { + site = b.Site + } + if site == "" { + site = b.City + } + created := "" + if b.CreatedAt != nil { + created = objectstorage.FormatTime(*b.CreatedAt) + } + return table.Row{ + "id": {Label: "ID", Value: b.ID}, + "name": {Label: "Name", Value: b.Name}, + "bucket_name": {Label: "Bucket Name", Value: b.BucketName}, + "project": {Label: "Project", Value: b.ProjectRef()}, + "storage_class": {Label: "Class", Value: b.StorageClass}, + "region": {Label: "Site", Value: site}, + "endpoint": {Label: "Endpoint", Value: b.Endpoint, MaxLength: 45}, + "versioning": {Label: "Versioning", Value: yesNo(b.Versioning)}, + "locking": {Label: "Locking", Value: lockingLabel(b)}, + "created_at": {Label: "Created At", Value: created}, + } +} + +func yesNo(v bool) string { + if v { + return "yes" + } + return "no" +} + +func lockingLabel(b *objectstorage.Bucket) string { + if !b.Locking { + return "no" + } + mode := strings.ToUpper(b.RetentionMode) + if mode == "" || mode == "NONE" { + return "yes" + } + if b.RetentionDays > 0 { + return fmt.Sprintf("%s (%dd)", mode, b.RetentionDays) + } + return mode +} diff --git a/cmd/s3/configure.go b/cmd/s3/configure.go new file mode 100644 index 0000000..d01dc8d --- /dev/null +++ b/cmd/s3/configure.go @@ -0,0 +1,959 @@ +package s3 + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +// Flag names of `lsh s3 configure`. +const ( + flagCfgStorageClass = "storage-class" + flagCfgRegion = "region" + flagCfgAllBuckets = "all-buckets" + flagCfgBucket = "bucket" + flagCfgName = "name" + flagCfgSave = "save" + flagCfgShowSecret = "show-secret" +) + +// configureNonInteractiveHint lists the alternatives to the wizard for +// scripts and CI (exit 2 when a prompt would be needed without a terminal). +const configureNonInteractiveHint = "pass every input as a flag (--project, --storage-class, --region, --all-buckets or --bucket [=rw|readonly]), " + + "or run 'lsh s3 access-keys create --bucket --save', or set " + objectstorage.EnvAccessKeyID + "/" + objectstorage.EnvSecretAccessKey + +// NewConfigureCmd builds `lsh s3 configure` (alias `credentials`): the +// interactive equivalent of `aws configure` for S3 access keys. +func NewConfigureCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "configure", + Aliases: []string{"credentials"}, + Short: "Set up S3 access keys on this machine", + Long: `Create an S3 access key and save it in the active lsh profile so the object +commands (ls, cp, rm, stat, presign) can use it automatically. + +The wizard asks for the project, the storage class (standard = general +purpose; high_performance = low latency on selected sites), the site for +high_performance, the scope (all buckets of that class in the project, or +specific buckets with rw/readonly permission) and a name. Every question can +be answered with a flag; when all of them are given the wizard runs without +prompts, which is how scripts should call it. The site of a standard key is +inferred from the project's standard buckets when --region is omitted. + +The access key is separate from your API token. It is stored in +~/.config/lsh/config.json (mode 0600) and never printed unless --save=false +is passed, in which case it is shown once in clear and not stored.`, + Example: ` lsh s3 configure + lsh s3 configure --project my-project --storage-class standard --all-buckets + lsh s3 configure --project my-project --storage-class high_performance --region TYO4 --all-buckets + lsh s3 configure --project my-project --bucket backups=rw --bucket logs=readonly --name laptop + lsh s3 configure --project my-project --storage-class standard --all-buckets --save=false -o json`, + Args: cobra.NoArgs, + RunE: runConfigure, + }) + addProjectFlag(cmd, true, "project ID or slug the key belongs to (default: LSH_PROJECT, or a menu)") + cmd.Flags().String(flagCfgStorageClass, "", "storage class of the key: standard | high_performance") + cmd.Flags().String(flagCfgRegion, "", "site slug (DAL, TYO4…) of the key; inferred from the project's buckets when omitted") + cmd.Flags().Bool(flagCfgAllBuckets, false, "key for all buckets of the class (and site) in the project (fullaccess)") + cmd.Flags().StringArray(flagCfgBucket, nil, "restrict the key to a bucket: [=rw|readonly] (repeatable)") + cmd.Flags().String(flagCfgName, "", "key name (default: lsh--)") + cmd.Flags().Bool(flagCfgSave, true, "save the key in the active profile; --save=false prints the secret once instead") + cmd.Flags().Bool(flagCfgShowSecret, false, "include the secret in structured output even when -o comes from the environment") + cmd.AddCommand(newConfigureExportCmd()) + return cmd +} + +// configureOptions collects the flag values of `lsh s3 configure`. +type configureOptions struct { + Project string + StorageClass string + Site string + AllBuckets bool + Buckets []bucketSpec + Name string + Save bool +} + +// parseConfigureOptions reads and validates the flags. +func parseConfigureOptions(cmd *cobra.Command) (configureOptions, error) { + o := configureOptions{Project: projectFlag(cmd)} + rawClass, _ := cmd.Flags().GetString(flagCfgStorageClass) + class, err := objectstorage.ParseStorageClass(rawClass) + if err != nil { + return o, err + } + o.StorageClass = class + if o.Site, err = regionFlag(cmd); err != nil { + return o, err + } + o.AllBuckets, _ = cmd.Flags().GetBool(flagCfgAllBuckets) + values, _ := cmd.Flags().GetStringArray(flagCfgBucket) + if o.Buckets, err = parseBucketSpecs(values); err != nil { + return o, err + } + if o.AllBuckets && len(o.Buckets) > 0 { + return o, objectstorage.ErrUsagef("--all-buckets and --bucket are mutually exclusive") + } + o.Name, _ = cmd.Flags().GetString(flagCfgName) + o.Name = strings.TrimSpace(o.Name) + o.Save, _ = cmd.Flags().GetBool(flagCfgSave) + return o, nil +} + +// cfgSelectedBucket is a bucket chosen for a limited_access key. +type cfgSelectedBucket struct { + Bucket *objectstorage.Bucket + Permission string +} + +// cfgValidateSelection checks that every selected bucket shares the storage +// class, site and project, which is what the API requires of one key. It +// returns the common values. +func cfgValidateSelection(sel []cfgSelectedBucket) (class, site, projectID string, err error) { + if len(sel) == 0 { + return "", "", "", objectstorage.ErrUsagef("no bucket selected") + } + first := sel[0].Bucket + class, site, projectID = first.StorageClass, first.Site, first.ProjectID + for _, s := range sel[1:] { + b := s.Bucket + if b.StorageClass != class { + return "", "", "", objectstorage.ErrUsagef("buckets %s (%s) and %s (%s) have different storage classes; one key covers a single class", + first.Display(), class, b.Display(), b.StorageClass) + } + if class == objectstorage.ClassHighPerformance && site != "" && b.Site != "" && !strings.EqualFold(site, b.Site) { + return "", "", "", objectstorage.ErrUsagef("buckets %s (%s) and %s (%s) live on different sites; a high_performance key covers a single site", + first.Display(), site, b.Display(), b.Site) + } + if projectID != "" && b.ProjectID != "" && projectID != b.ProjectID { + return "", "", "", objectstorage.ErrUsagef("buckets %s (%s) and %s (%s) belong to different projects; one key covers a single project", + first.Display(), first.ProjectRef(), b.Display(), b.ProjectRef()) + } + if site == "" { + site = b.Site + } + if projectID == "" { + projectID = b.ProjectID + } + } + return class, site, projectID, nil +} + +// configurePlan is the structured description of the key the wizard creates +// (rendered for --dry-run and as the result when the key is saved). +type configurePlan struct { + Name string `json:"name"` + Project string `json:"project"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Scope string `json:"scope"` + Buckets map[string]string `json:"buckets,omitempty"` + Save bool `json:"save"` + DryRun bool `json:"dry_run,omitempty"` + AccessKeyID string `json:"access_key_id,omitempty"` + // SavedAs is the name the key was stored under, which differs from Name + // when that name already held another key (saveNewKey appends -2, -3…). + SavedAs string `json:"saved_as,omitempty"` + Profile string `json:"profile,omitempty"` +} + +func (p configurePlan) TableRow() table.Row { + status := "created" + if p.DryRun { + status = "dryrun" + } + row := table.Row{ + "name": {Label: "Name", Value: p.Name}, + "project": {Label: "Project", Value: p.Project}, + "storage_class": {Label: "Class", Value: p.StorageClass}, + "site": {Label: "Site", Value: orEmptyCell(p.Site)}, + "scope": {Label: "Scope", Value: p.Scope}, + "buckets": {Label: "Buckets", Value: cfgFormatBucketPerms(p.Buckets), MaxLength: 60}, + "save": {Label: "Saved", Value: yesNo(p.Save && !p.DryRun)}, + "status": {Label: "Status", Value: status}, + } + if p.AccessKeyID != "" { + row["access_key_id"] = table.Cell{Label: "Access Key ID", Value: p.AccessKeyID} + } + if p.SavedAs != "" { + row["saved_as"] = table.Cell{Label: "Saved As", Value: p.SavedAs} + } + return row +} + +// cfgFormatBucketPerms renders "a=rw b=readonly" sorted by name (the shared +// empty-cell placeholder when empty). +func cfgFormatBucketPerms(perms map[string]string) string { + if len(perms) == 0 { + return emptyCell + } + parts := make([]string, 0, len(perms)) + for name, perm := range perms { + parts = append(parts, name+"="+perm) + } + sort.Strings(parts) + return strings.Join(parts, " ") +} + +// configureSecretResult is the --save=false output: the key shown once. +type configureSecretResult struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey *string `json:"secret_access_key"` + Scope string `json:"scope"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Endpoint string `json:"endpoint,omitempty"` + SigningRegion string `json:"signing_region,omitempty"` + Buckets map[string]string `json:"buckets,omitempty"` +} + +func (r configureSecretResult) TableRow() table.Row { + return table.Row{ + "name": {Label: "Name", Value: r.Name}, + "access_key_id": {Label: "Access Key ID", Value: r.AccessKeyID}, + "scope": {Label: "Scope", Value: r.Scope}, + "storage_class": {Label: "Class", Value: r.StorageClass}, + "site": {Label: "Site", Value: orEmptyCell(r.Site)}, + "project": {Label: "Project", Value: r.Project}, + "buckets": {Label: "Buckets", Value: cfgFormatBucketPerms(r.Buckets), MaxLength: 60}, + } +} + +// configureNeedsPrompt returns the exit-2 error for a question the wizard +// cannot ask without a terminal. +func configureNeedsPrompt(what string) error { + return exitcode.Errorf(exitcode.Usage, "lsh s3 configure needs a terminal to ask for %s; %s", what, configureNonInteractiveHint) +} + +// cfgProjectChoice is one project offered by the wizard. +type cfgProjectChoice struct { + ID string + Slug string + Name string +} + +// configureListProjects fetches the team's projects (all pages, capped). +func configureListProjects(ctx context.Context, api *sdk.Latitudesh) ([]cfgProjectChoice, error) { + size := int64(100) + resp, err := api.Projects.List(ctx, operations.GetProjectsRequest{PageSize: &size}, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "projects") + } + var out []cfgProjectChoice + for page := 0; resp != nil && resp.Projects != nil && page < 20; page++ { + for _, p := range resp.Projects.Data { + c := cfgProjectChoice{ID: ptrStr(p.ID)} + if p.Attributes != nil { + c.Slug = ptrStr(p.Attributes.Slug) + c.Name = ptrStr(p.Attributes.Name) + } + if c.ID != "" { + out = append(out, c) + } + } + if resp.Next == nil { + break + } + next, err := resp.Next() + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "projects") + } + if next == nil { + break + } + resp = next + } + sort.Slice(out, func(i, j int) bool { return out[i].label() < out[j].label() }) + return out, nil +} + +func (p cfgProjectChoice) label() string { + switch { + case p.Slug != "" && p.Name != "" && p.Slug != p.Name: + return fmt.Sprintf("%s (%s)", p.Slug, p.Name) + case p.Slug != "": + return p.Slug + case p.Name != "": + return p.Name + } + return p.ID +} + +// cfgResolveProjectID turns a project token (ID or slug) into the proj_ ID the +// saved key is bound to. Unknown tokens are returned unchanged. +func cfgResolveProjectID(ctx context.Context, api *sdk.Latitudesh, token string) string { + if token == "" || strings.HasPrefix(token, "proj_") || api == nil { + return token + } + projects, err := configureListProjects(ctx, api) + if err != nil { + return token + } + for _, p := range projects { + if p.Slug == token { + return p.ID + } + } + for _, p := range projects { + if p.Name == token { + return p.ID + } + } + return token +} + +// cfgSiteChoice is one region offered by the wizard. +type cfgSiteChoice struct { + Slug string + Name string + Country string +} + +func (s cfgSiteChoice) label() string { + return joinNonEmpty(" — ", s.Slug, joinNonEmpty(", ", s.Name, s.Country)) +} + +// configureListSites fetches every region (including storage-only ones). +func configureListSites(ctx context.Context, api *sdk.Latitudesh) ([]cfgSiteChoice, error) { + size := int64(100) + includeCustom := true + resp, err := api.Regions.Get(ctx, operations.GetRegionsRequest{IncludeCustom: &includeCustom, PageSize: &size}, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "regions") + } + var out []cfgSiteChoice + for page := 0; resp != nil && resp.Regions != nil && page < 20; page++ { + for _, r := range resp.Regions.Data { + if r.Attributes == nil { + continue + } + c := cfgSiteChoice{Slug: ptrStr(r.Attributes.Slug), Name: ptrStr(r.Attributes.Name)} + if r.Attributes.Country != nil { + c.Country = ptrStr(r.Attributes.Country.Name) + } + if c.Slug != "" { + out = append(out, c) + } + } + if resp.Next == nil { + break + } + next, err := resp.Next() + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "regions") + } + if next == nil { + break + } + resp = next + } + sort.Slice(out, func(i, j int) bool { return out[i].Slug < out[j].Slug }) + return out, nil +} + +// configureState is the wizard's accumulated answers. +type configureState struct { + ProjectToken string // as given (ID or slug) + ProjectID string + StorageClass string + Site string + Scope string + Selected []cfgSelectedBucket + Name string + // Endpoint/SigningRegion are known when specific buckets were selected. + Endpoint string + SigningRegion string +} + +// project returns the best project reference known so far. +func (s *configureState) project() string { + return firstNonEmptyStr(s.ProjectToken, s.ProjectID) +} + +// bucketPerms returns bkt_ ID → permission for a limited selection. +func (s *configureState) bucketPerms() map[string]string { + if s.Scope != config.ScopeLimitedAccess { + return nil + } + out := map[string]string{} + for _, sel := range s.Selected { + out[sel.Bucket.ID] = sel.Permission + } + return out +} + +// bucketNamePerms returns display name → permission for messages. +func (s *configureState) bucketNamePerms() map[string]string { + if s.Scope != config.ScopeLimitedAccess { + return nil + } + out := map[string]string{} + for _, sel := range s.Selected { + out[sel.Bucket.Name] = sel.Permission + } + return out +} + +func runConfigure(cmd *cobra.Command, _ []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + opts, err := parseConfigureOptions(cmd) + if err != nil { + return printErr(err) + } + + canPrompt := objectstorage.CanPrompt(cmd) + api := apiClient() + + // Saving needs an active profile; check before creating anything so a + // LATITUDESH_TOKEN-only session does not end up with an unsaved key. + if opts.Save { + if _, _, _, err := objectstorage.ActiveProfile(profileFlag(cmd)); err != nil { + return printErr(exitcode.Errorf(exitcode.Of(err), "%v\n configure saves the key in a profile; run 'lsh login' first, or pass --save=false to print the key instead", err)) + } + } + + st := &configureState{StorageClass: opts.StorageClass, Site: opts.Site} + + // Step 4 first when --bucket was given: the buckets pin the project, + // class and site, so nothing else has to be asked. + if len(opts.Buckets) > 0 { + if err := configureSelectFromSpecs(ctx, cmd, st, opts); err != nil { + return printErr(err) + } + } + + // Step 1: project. + if st.ProjectID == "" { + st.ProjectToken = opts.Project + if st.ProjectToken == "" { + if !canPrompt { + return printErr(configureNeedsPrompt("the project (--project or LSH_PROJECT)")) + } + projects, err := configureListProjects(ctx, api) + if err != nil { + return printErr(err) + } + if len(projects) == 0 { + return printErr(exitcode.Errorf(exitcode.NotFound, "no projects found; create one with 'lsh projects create'")) + } + labels := make([]string, len(projects)) + for i, p := range projects { + labels[i] = p.label() + } + idx, err := objectstorage.Choose("Which project should the key belong to?", labels, 0) + if err != nil { + return printErr(err) + } + if idx < 0 { + return printErr(exitcode.Errorf(exitcode.Refused, "cancelled")) + } + st.ProjectToken, st.ProjectID = projects[idx].ID, projects[idx].ID + } else { + st.ProjectID = cfgResolveProjectID(ctx, api, st.ProjectToken) + } + } + + // Step 2: storage class. + if st.StorageClass == "" { + if !canPrompt { + return printErr(configureNeedsPrompt("the storage class (--storage-class standard|high_performance)")) + } + idx, err := objectstorage.Choose("Which storage class?", []string{ + "standard — general purpose object storage, one key covers every site of the project", + "high_performance — low latency storage on selected sites, one key per site", + }, 0) + if err != nil { + return printErr(err) + } + if idx < 0 { + return printErr(exitcode.Errorf(exitcode.Refused, "cancelled")) + } + st.StorageClass = []string{objectstorage.ClassStandard, objectstorage.ClassHighPerformance}[idx] + } + + // Step 3: site. A high_performance key is bound to one cluster, so the + // site is asked up front (it also filters the bucket menu below). + if st.StorageClass == objectstorage.ClassHighPerformance && st.Site == "" { + if err := configureAskSite(ctx, api, st, canPrompt, "the site of a high_performance key"); err != nil { + return printErr(err) + } + } + + // Step 4: scope. + if st.Scope == "" { + switch { + case opts.AllBuckets: + st.Scope = config.ScopeFullAccess + case !canPrompt: + return printErr(configureNeedsPrompt("the scope (--all-buckets or --bucket [=rw|readonly])")) + default: + where := st.StorageClass + " buckets" + if st.Site != "" { + where += " in " + st.Site + } + idx, err := objectstorage.Choose("Which buckets should the key cover?", []string{ + "all " + where + " of the project, including future ones (fullaccess, recommended)", + "specific buckets (limited_access, rw or readonly)", + }, 0) + if err != nil { + return printErr(err) + } + switch idx { + case 0: + st.Scope = config.ScopeFullAccess + case 1: + if err := configureSelectInteractive(ctx, cmd, st); err != nil { + return printErr(err) + } + default: + return printErr(exitcode.Errorf(exitcode.Refused, "cancelled")) + } + } + } + + // A fullaccess key still needs a site (the API requires one). Without + // --region it is inferred from the project's buckets of that class, the way + // 'access-keys create --all-buckets' does; a standard key covers every site + // so any site of a standard bucket will do. + if st.Scope == config.ScopeFullAccess && st.Site == "" { + site, err := configureInferSite(ctx, cmd, st) + if err != nil { + return printErr(err) + } + if site == "" { + what := fmt.Sprintf("the site of the key (project %s has no %s buckets to infer it from)", st.project(), st.StorageClass) + if err := configureAskSite(ctx, api, st, canPrompt, what); err != nil { + return printErr(err) + } + } else { + st.Site = site + } + } + + // Step 5: name. + st.Name = opts.Name + if st.Name == "" { + st.Name = generateKeyName(st.StorageClass) + if canPrompt && !dryRun() { + answer, err := objectstorage.ReadLine(fmt.Sprintf("Key name [%s]: ", st.Name)) + if err == nil && answer != "" { + st.Name = answer + } + } + } + + plan := configurePlan{ + Name: st.Name, + Project: st.project(), + StorageClass: st.StorageClass, + Site: st.Site, + Scope: st.Scope, + Buckets: st.bucketNamePerms(), + Save: opts.Save, + DryRun: dryRun(), + } + + // Dry-run: describe, never create. + if dryRun() { + if isHuman() { + fmt.Println(configurePlanLine(plan)) + } else { + render([]renderer.ResponseData{plan}) + } + return nil + } + + // Step 6: create (and save). When the name was auto-generated (the user did + // not pass --name), let a collision re-roll a fresh pet name. + created, err := createAccessKeyRetrying(ctx, cmd, accessKeyRequest{ + Project: firstNonEmptyStr(st.ProjectID, st.ProjectToken), + StorageClass: st.StorageClass, + Site: st.Site, + Name: st.Name, + Scope: st.Scope, + Buckets: st.bucketPerms(), + }, opts.Name == "") + if err != nil { + return printErr(objectstorage.HumanizeAPI(err, "access key")) + } + if created.Name != "" { + // The API normalizes names; keep the server's spelling as the index. + st.Name = created.Name + plan.Name = created.Name + } + plan.AccessKeyID = created.AccessKeyID + + if !opts.Save { + return configurePrintSecret(cmd, st, created) + } + + savedAs, profileName, err := saveNewKey(cmd, st.Name, created.stored(st.ProjectID, st.bucketPerms(), config.KeySourceConfigure)) + if err != nil { + // configure creates the key on the user's behalf, so a key it cannot + // store is removed again instead of having its one-time secret + // printed. --show-secret opts into keeping and printing it. + showSecret, _ := cmd.Flags().GetBool(flagCfgShowSecret) + if !showSecret { + req := accessKeyRequest{Project: firstNonEmptyStr(st.ProjectID, st.ProjectToken), StorageClass: st.StorageClass, Site: st.Site, Scope: st.Scope} + if delErr := discardKey(ctx, req, created); delErr == nil { + return printErr(exitcode.Errorf(exitcode.Generic, + "access key %q was created but could not be saved: %v — it was deleted again, so nothing was left behind; retry once the profile is writable", st.Name, err)) + } + objectstorage.Warnf("access key %q could not be saved (%v) nor deleted again, so its secret is shown below", st.Name, err) + return configurePrintSecret(cmd, st, created) + } + objectstorage.Warnf("access key %q was created but could not be saved: %v", st.Name, err) + return configurePrintSecret(cmd, st, created) + } + plan.Profile, plan.SavedAs = profileName, savedAs + path, _ := config.Path() + if path == "" { + path = "~/.config/lsh/config.json" + } + objectstorage.Hintf("Saved to %s (0600) as %q in profile %s. The CLI uses it automatically.", path, savedAs, profileName) + if savedAs != st.Name { + objectstorage.Hintf("(%q already held another key, so this one was saved as %q)", st.Name, savedAs) + } + objectstorage.Hintf("Next: lsh s3 ls%s", configureExampleBucket(st, " lsh s3 cp ./file s3://%s/")) + objectstorage.Hintf("For apps or CI create a separate key: lsh s3 access-keys create --bucket --name ") + objectstorage.Hintf("Export it for other tools: lsh s3 configure export --format env%s", configureExampleBucket(st, " s3://%s")) + if !isHuman() { + render([]renderer.ResponseData{plan}) + } + return nil +} + +// configureAskSite fills st.Site from a menu of Latitude sites, or fails with +// exit 2 naming --region when no terminal is available. what describes the +// value being asked for in the error message. +func configureAskSite(ctx context.Context, api *sdk.Latitudesh, st *configureState, canPrompt bool, what string) error { + sites, listErr := configureListSites(ctx, api) + if !canPrompt { + slugs := make([]string, 0, len(sites)) + for _, s := range sites { + slugs = append(slugs, s.Slug) + } + hint := "" + if len(slugs) > 0 { + hint = "; available: " + strings.Join(slugs, ", ") + } + return configureNeedsPrompt(what + " (--region " + hint + ")") + } + if listErr != nil { + return listErr + } + if len(sites) == 0 { + return exitcode.Errorf(exitcode.NotFound, "no sites returned by the API; pass --region ") + } + labels := make([]string, len(sites)) + for i, s := range sites { + labels[i] = s.label() + } + question := "Which site should the key be created in?" + if st.StorageClass == objectstorage.ClassHighPerformance { + question = "Which site is the high_performance cluster on?" + } + idx, err := objectstorage.Choose(question, labels, 0) + if err != nil { + return err + } + if idx < 0 { + return exitcode.Errorf(exitcode.Refused, "cancelled") + } + st.Site = strings.ToUpper(sites[idx].Slug) + return nil +} + +// configureInferSite lists the project's buckets and returns the site for a +// fullaccess key of st.StorageClass, or "" when the project has no bucket of +// that class. Sites come from the raw API because the SDK model drops them. +func configureInferSite(ctx context.Context, cmd *cobra.Command, st *configureState) (string, error) { + r := newResolver(cmd) + r.Project = st.project() + if r.EndpointURL != "" { + return "", objectstorage.ErrUsagef("configure needs the Latitude API; unset --endpoint-url/%s", objectstorage.EnvEndpointURL) + } + list, err := r.ListBuckets(ctx) + if err != nil { + return "", err + } + buckets := make([]*objectstorage.Bucket, 0, len(list)) + for _, d := range list { + buckets = append(buckets, objectstorage.BucketFromData(d)) + } + var sites map[string]string + if len(buckets) > 0 { + if sites, err = objectstorage.RawBucketSitesForProject(ctx, "", r.Project); err != nil { + lsh.LogDebugf("[s3] site lookup failed for project %s: %v", r.Project, err) + } + } + return configureSiteFromBuckets(st.StorageClass, r.Project, buckets, sites) +} + +// configureSiteFromBuckets picks the site for a fullaccess key from the +// project's buckets of the given class (sites supplies bkt_ ID → site for +// buckets whose Site is empty). A standard key covers every site, so the +// first site in sorted order is used; several high_performance sites are +// ambiguous and rejected with exit 2. "" means nothing to infer from. +func configureSiteFromBuckets(class, project string, buckets []*objectstorage.Bucket, sites map[string]string) (string, error) { + distinct := map[string]bool{} + for _, b := range buckets { + if b.StorageClass != class { + continue + } + if s := strings.ToUpper(firstNonEmptyStr(b.Site, sites[b.ID])); s != "" { + distinct[s] = true + } + } + if len(distinct) == 0 { + return "", nil + } + names := make([]string, 0, len(distinct)) + for s := range distinct { + names = append(names, s) + } + sort.Strings(names) + if class == objectstorage.ClassHighPerformance && len(names) > 1 { + return "", objectstorage.ErrUsagef("project %s has high_performance buckets in several sites (%s); pass --region ", project, strings.Join(names, ", ")) + } + return names[0], nil +} + +// configureExampleBucket formats an example with the first selected bucket, +// or "" when the key is fullaccess (no bucket to point at). +func configureExampleBucket(st *configureState, format string) string { + if len(st.Selected) == 0 || st.Selected[0].Bucket.Name == "" { + return "" + } + return fmt.Sprintf(format, st.Selected[0].Bucket.Name) +} + +// configurePlanLine renders the dry-run plan as one human line. +func configurePlanLine(p configurePlan) string { + scope := "all " + p.StorageClass + " buckets" + if p.Site != "" { + scope += " in " + p.Site + } + scope += " (fullaccess)" + if p.Scope == config.ScopeLimitedAccess { + scope = "buckets " + cfgFormatBucketPerms(p.Buckets) + " (limited_access)" + } + action := "save it in the active profile" + if !p.Save { + action = "print the secret once (not saved)" + } + return fmt.Sprintf("(dryrun) create access key %q in project %s for %s and %s", p.Name, p.Project, scope, action) +} + +// configurePrintSecret shows the created key once, in clear (--save=false). +// Structured output embeds the secret only when -o/--json was given on the +// command line (never because of LSH_OUTPUT or the config file). +func configurePrintSecret(cmd *cobra.Command, st *configureState, created *createdAccessKey) error { + res := configureSecretResult{ + Name: st.Name, + AccessKeyID: created.AccessKeyID, + Scope: st.Scope, + StorageClass: st.StorageClass, + Site: st.Site, + Project: st.project(), + Endpoint: st.Endpoint, + SigningRegion: st.SigningRegion, + Buckets: st.bucketNamePerms(), + } + showSecret, _ := cmd.Flags().GetBool(flagCfgShowSecret) + decision := decideSecretDisplay(isHuman(), false, showSecret, outputExplicit(cmd)) + if isHuman() { + fmt.Printf("Access key %q created. The secret is shown once and cannot be retrieved again:\n", res.Name) + fmt.Printf(" Access Key ID: %s\n", created.AccessKeyID) + fmt.Printf(" Secret Access Key: %s\n", created.SecretAccessKey) + if res.Endpoint != "" { + fmt.Printf(" Endpoint: %s Signing region: %s\n", res.Endpoint, res.SigningRegion) + } + if len(res.Buckets) > 0 { + fmt.Printf(" Buckets: %s\n", cfgFormatBucketPerms(res.Buckets)) + } else { + fmt.Printf(" Scope: all %s buckets of the project%s\n", res.StorageClass, cfgSiteSuffix(res.Site)) + } + objectstorage.Hintf("Save it later with: lsh s3 access-keys import --name %s --access-key-id %s", res.Name, created.AccessKeyID) + return nil + } + if decision.Show { + secret := created.SecretAccessKey + res.SecretAccessKey = &secret + } else { + // The key exists on the API and the secret is returned exactly once, so + // it cannot simply be dropped: stdout stays clean (it was not asked for + // explicitly and CI captures it) and the secret goes to stderr. + objectstorage.Warnf("secret_access_key omitted from stdout: structured output came from LSH_OUTPUT/config, not -o; pass -o json or --show-secret to include it. The secret is printed on stderr below — it cannot be retrieved again.") + fmt.Fprintf(os.Stderr, " Access Key ID: %s\n", created.AccessKeyID) + fmt.Fprintf(os.Stderr, " Secret Access Key: %s\n", created.SecretAccessKey) + fmt.Fprintf(os.Stderr, " Save it with: lsh s3 access-keys import --name %s --access-key-id %s\n", res.Name, created.AccessKeyID) + } + render([]renderer.ResponseData{res}) + return nil +} + +func cfgSiteSuffix(site string) string { + if site == "" { + return "" + } + return " in " + site +} + +// configureSelectFromSpecs resolves the --bucket values, validates them and +// fills the state (class, site, project, scope) from the buckets. +func configureSelectFromSpecs(ctx context.Context, cmd *cobra.Command, st *configureState, opts configureOptions) error { + r := newResolver(cmd) + if r.EndpointURL != "" { + return objectstorage.ErrUsagef("configure needs the Latitude API; unset --endpoint-url/%s", objectstorage.EnvEndpointURL) + } + sel := make([]cfgSelectedBucket, 0, len(opts.Buckets)) + for _, spec := range opts.Buckets { + b, err := r.Resolve(ctx, spec.Token) + if err != nil { + return err + } + if err := r.FillSite(ctx, b); err != nil { + lsh.LogDebugf("[s3] could not fetch site of %s: %v", b.Display(), err) + } + sel = append(sel, cfgSelectedBucket{Bucket: b, Permission: spec.Permission}) + } + class, site, projectID, err := cfgValidateSelection(sel) + if err != nil { + return err + } + if opts.StorageClass != "" && opts.StorageClass != class { + return objectstorage.ErrUsagef("--storage-class %s does not match the selected buckets (%s)", opts.StorageClass, class) + } + if st.Site != "" && site != "" && !strings.EqualFold(st.Site, site) { + return objectstorage.ErrUsagef("--region %s does not match the selected buckets (%s)", st.Site, site) + } + if opts.Project != "" && projectID != "" && opts.Project != projectID && !anyProjectSlugMatches(sel, opts.Project) { + return objectstorage.ErrUsagef("--project %s does not match the selected buckets (%s)", opts.Project, projectID) + } + st.StorageClass = class + if site != "" { + st.Site = strings.ToUpper(site) + } + st.ProjectID = projectID + st.ProjectToken = firstNonEmptyStr(opts.Project, sel[0].Bucket.ProjectRef(), projectID) + st.Scope = config.ScopeLimitedAccess + st.Selected = sel + st.Endpoint = sel[0].Bucket.Endpoint + st.SigningRegion = sel[0].Bucket.SigningRegion + if st.StorageClass == objectstorage.ClassHighPerformance && st.Site == "" { + return objectstorage.ErrUsagef("could not determine the site of the selected high_performance buckets; pass --region ") + } + return nil +} + +func anyProjectSlugMatches(sel []cfgSelectedBucket, token string) bool { + for _, s := range sel { + if s.Bucket.ProjectSlug == token || s.Bucket.ProjectName == token { + return true + } + } + return false +} + +// configureSelectInteractive lists the buckets of the chosen class (and site) +// in the project and lets the user pick some by number, then a permission. +func configureSelectInteractive(ctx context.Context, cmd *cobra.Command, st *configureState) error { + r := newResolver(cmd) + r.Project = st.project() + if r.EndpointURL != "" { + return objectstorage.ErrUsagef("configure needs the Latitude API; unset --endpoint-url/%s", objectstorage.EnvEndpointURL) + } + data, err := r.ListBuckets(ctx) + if err != nil { + return err + } + var sites map[string]string + if st.StorageClass == objectstorage.ClassHighPerformance { + sites, err = objectstorage.RawBucketSitesForProject(ctx, "", r.Project) + if err != nil { + lsh.LogDebugf("[s3] could not fetch bucket sites: %v", err) + sites = nil + } + } + var candidates []*objectstorage.Bucket + for _, d := range data { + b := objectstorage.BucketFromData(d) + if b.StorageClass != st.StorageClass { + continue + } + if b.Site == "" { + b.Site = sites[b.ID] + } + if st.StorageClass == objectstorage.ClassHighPerformance && st.Site != "" && b.Site != "" && !strings.EqualFold(b.Site, st.Site) { + continue + } + candidates = append(candidates, b) + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].Name < candidates[j].Name }) + if len(candidates) == 0 { + where := st.StorageClass + if st.Site != "" { + where += " in " + st.Site + } + return exitcode.Errorf(exitcode.NotFound, "no %s buckets found in project %s; create one with 'lsh s3 mb s3:// --region --project %s'", where, r.Project, r.Project) + } + labels := make([]string, len(candidates)) + for i, b := range candidates { + labels[i] = fmt.Sprintf("%s (%s, %s)", b.Name, b.ID, joinNonEmpty(" ", b.StorageClass, firstNonEmptyStr(b.Site, b.City))) + } + idx, err := objectstorage.ChooseMany("Which buckets should the key cover?", labels) + if err != nil { + return err + } + if len(idx) == 0 { + return exitcode.Errorf(exitcode.Refused, "no bucket selected; pick at least one with space, or pass --bucket ") + } + permIdx, err := objectstorage.Choose("Which permission on these buckets?", []string{ + "rw — read and write", + "readonly — list and download only", + }, 0) + if err != nil { + return err + } + if permIdx < 0 { + return exitcode.Errorf(exitcode.Refused, "cancelled") + } + perm := []string{config.PermissionRW, config.PermissionReadOnly}[permIdx] + sel := make([]cfgSelectedBucket, 0, len(idx)) + for _, i := range idx { + sel = append(sel, cfgSelectedBucket{Bucket: candidates[i], Permission: perm}) + } + class, site, projectID, err := cfgValidateSelection(sel) + if err != nil { + return err + } + st.StorageClass = class + if site != "" { + st.Site = strings.ToUpper(site) + } + if projectID != "" { + st.ProjectID = projectID + } + st.Scope = config.ScopeLimitedAccess + st.Selected = sel + st.Endpoint = sel[0].Bucket.Endpoint + st.SigningRegion = sel[0].Bucket.SigningRegion + return nil +} diff --git a/cmd/s3/configure_export.go b/cmd/s3/configure_export.go new file mode 100644 index 0000000..e0d701f --- /dev/null +++ b/cmd/s3/configure_export.go @@ -0,0 +1,375 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + cobra "github.com/spf13/cobra" +) + +// Flag names of `lsh s3 configure export`. +const ( + flagExportFormat = "format" + flagExportProfileName = "profile-name" + flagExportNoSecret = "no-secret" +) + +// exportFormats lists the supported --format values in help order. +var exportFormats = []string{"env", "aws", "rclone", "mc", "s3cmd", "process"} + +// exportSecretPlaceholder replaces the secret with --no-secret. +const exportSecretPlaceholder = "" + +// newConfigureExportCmd builds `lsh s3 configure export`. +func newConfigureExportCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "export [s3://bucket]", + Short: "Print a saved access key in the format of another S3 tool", + Long: `Print the credential and connection settings for a bucket in the format of +another tool: shell variables (env), an aws CLI profile (aws), an rclone +remote (rclone), a MinIO client alias (mc), an s3cmd config (s3cmd) or the +JSON of an aws credential_process (process). + +With a bucket argument the endpoint, signing region and backend bucket name +come from the Latitude API and the credential is selected like every other +command (LSH_S3_* environment, --access-key, or the least-privileged saved key +covering the bucket). Without a bucket, --access-key names the saved key and +--endpoint-url supplies the endpoint. + +The output goes to stdout and contains the secret unless --no-secret is +given; -o/--output is ignored because the template is the output. The aws +format prints two blocks that belong to two different files (~/.aws/config and +~/.aws/credentials), each labelled with a comment, so it cannot be appended to +a single file in one redirect.`, + Example: ` eval "$(lsh s3 configure export s3://backups --format env)" + lsh s3 configure export s3://backups --format aws # two blocks: one per file, see the comments + lsh s3 configure export s3://backups --format rclone --profile-name latitude >> ~/.config/rclone/rclone.conf + lsh s3 configure export s3://backups --format mc | sh + lsh s3 configure export --access-key ci-deploy --endpoint-url https://s3.us-central-1.storage.sh --format process + lsh s3 configure export s3://backups --format s3cmd --no-secret`, + Args: cobra.MaximumNArgs(1), + RunE: runConfigureExport, + }) + addProjectFlag(cmd, true, "project ID or slug to disambiguate the bucket name") + cmd.Flags().String(flagExportFormat, "", "output format: "+strings.Join(exportFormats, " | ")) + cmd.Flags().String(flagExportProfileName, "", "profile/remote/alias name in the output (default: lsh- or lsh-)") + cmd.Flags().Bool(flagExportNoSecret, false, "replace the secret with a "+exportSecretPlaceholder+" placeholder") + return cmd +} + +// exportTarget is everything a template needs. +type exportTarget struct { + ProfileName string + Endpoint string // scheme://host[:port] + Host string // host[:port] without scheme + Secure bool + SigningRegion string + BucketName string // backend bucket name; empty when unknown + StorageClass string // standard | high_performance | "" + AccessKeyID string + Secret string +} + +// newExportTarget derives Host/Secure from the endpoint. +func newExportTarget(profileName, endpoint, region, bucketName, class, accessKeyID, secret string) (exportTarget, error) { + host, secure, err := objectstorage.EndpointHost(endpoint) + if err != nil { + return exportTarget{}, exitcode.New(exitcode.Usage, err) + } + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + if region == "" { + region = objectstorage.SigningRegion(endpoint) + } + return exportTarget{ + ProfileName: profileName, + Endpoint: strings.TrimRight(endpoint, "/"), + Host: host, + Secure: secure, + SigningRegion: region, + BucketName: bucketName, + StorageClass: class, + AccessKeyID: accessKeyID, + Secret: secret, + }, nil +} + +// bucketOrPlaceholder returns the backend bucket name or "" for the +// example commands when no bucket is known. +func (t exportTarget) bucketOrPlaceholder() string { + if t.BucketName == "" { + return "" + } + return t.BucketName +} + +// header is the comment every text template starts with. +func (t exportTarget) header() string { + if t.BucketName == "" { + return "" + } + return "# bucket name on the endpoint: " + t.BucketName + "\n" +} + +// rcloneProvider maps the storage class to rclone's provider setting: the +// standard tier is Wasabi-backed, high_performance (VAST) is generic S3. +func (t exportTarget) rcloneProvider() string { + switch t.StorageClass { + case objectstorage.ClassStandard: + return "Wasabi" + case objectstorage.ClassHighPerformance: + return "Other" + } + // Infer from the endpoint shape when the class is unknown. + if strings.HasPrefix(strings.ToLower(t.Host), "s3.") { + return "Wasabi" + } + return "Other" +} + +// renderExport dispatches to the template for format. +func renderExport(format string, t exportTarget) (string, error) { + switch strings.ToLower(strings.TrimSpace(format)) { + case "env": + return exportEnv(t), nil + case "aws": + return exportAWS(t), nil + case "rclone": + return exportRclone(t), nil + case "mc": + return exportMc(t), nil + case "s3cmd": + return exportS3cmd(t), nil + case "process": + return exportProcess(t), nil + } + return "", objectstorage.ErrUsagef("--format must be one of %s (got %q)", strings.Join(exportFormats, ", "), format) +} + +// cfgShellValue quotes v for a POSIX shell when it contains characters outside +// the safe set (aws export-credentials prints plain values; secrets are +// base64-like so they normally pass through untouched). +func cfgShellValue(v string) string { + safe := true + for _, r := range v { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case strings.ContainsRune("_./:+=@-", r): + default: + safe = false + } + } + if safe && v != "" { + return v + } + return "'" + strings.ReplaceAll(v, "'", `'\''`) + "'" +} + +// exportEnv renders shell exports for the aws CLI/SDKs and for lsh itself. +func exportEnv(t exportTarget) string { + var b strings.Builder + b.WriteString(t.header()) + fmt.Fprintf(&b, "export AWS_ACCESS_KEY_ID=%s\n", cfgShellValue(t.AccessKeyID)) + fmt.Fprintf(&b, "export AWS_SECRET_ACCESS_KEY=%s\n", cfgShellValue(t.Secret)) + fmt.Fprintf(&b, "export AWS_ENDPOINT_URL_S3=%s\n", cfgShellValue(t.Endpoint)) + fmt.Fprintf(&b, "export AWS_REGION=%s\n", cfgShellValue(t.SigningRegion)) + b.WriteString("export AWS_REQUEST_CHECKSUM_CALCULATION=when_required\n") + b.WriteString("export AWS_RESPONSE_CHECKSUM_VALIDATION=when_required\n") + fmt.Fprintf(&b, "export %s=%s\n", objectstorage.EnvAccessKeyID, cfgShellValue(t.AccessKeyID)) + fmt.Fprintf(&b, "export %s=%s\n", objectstorage.EnvSecretAccessKey, cfgShellValue(t.Secret)) + fmt.Fprintf(&b, "export %s=%s\n", objectstorage.EnvEndpointURL, cfgShellValue(t.Endpoint)) + fmt.Fprintf(&b, "export %s=%s\n", objectstorage.EnvSigningRegion, cfgShellValue(t.SigningRegion)) + fmt.Fprintf(&b, "# aws s3 ls s3://%s/\n", t.bucketOrPlaceholder()) + return b.String() +} + +// exportAWS renders an aws CLI profile: a config block and a credentials +// block, each commented with the file it belongs to. +func exportAWS(t exportTarget) string { + var b strings.Builder + b.WriteString(t.header()) + b.WriteString("# ~/.aws/config\n") + fmt.Fprintf(&b, "[profile %s]\n", t.ProfileName) + fmt.Fprintf(&b, "region = %s\n", t.SigningRegion) + fmt.Fprintf(&b, "endpoint_url = %s\n", t.Endpoint) + b.WriteString("s3 =\n") + b.WriteString(" addressing_style = path\n") + b.WriteString("request_checksum_calculation = when_required\n") + b.WriteString("response_checksum_validation = when_required\n") + b.WriteString("\n") + b.WriteString("# ~/.aws/credentials\n") + fmt.Fprintf(&b, "[%s]\n", t.ProfileName) + fmt.Fprintf(&b, "aws_access_key_id = %s\n", t.AccessKeyID) + fmt.Fprintf(&b, "aws_secret_access_key = %s\n", t.Secret) + b.WriteString("\n") + fmt.Fprintf(&b, "# aws --profile %s s3 ls s3://%s/\n", t.ProfileName, t.bucketOrPlaceholder()) + return b.String() +} + +// exportRclone renders an rclone remote section. +func exportRclone(t exportTarget) string { + var b strings.Builder + b.WriteString(t.header()) + fmt.Fprintf(&b, "[%s]\n", t.ProfileName) + b.WriteString("type = s3\n") + fmt.Fprintf(&b, "provider = %s\n", t.rcloneProvider()) + b.WriteString("env_auth = false\n") + fmt.Fprintf(&b, "access_key_id = %s\n", t.AccessKeyID) + fmt.Fprintf(&b, "secret_access_key = %s\n", t.Secret) + fmt.Fprintf(&b, "endpoint = %s\n", t.Host) + fmt.Fprintf(&b, "region = %s\n", t.SigningRegion) + b.WriteString("force_path_style = true\n") + b.WriteString("\n") + fmt.Fprintf(&b, "# rclone ls %s:%s\n", t.ProfileName, t.bucketOrPlaceholder()) + return b.String() +} + +// exportMc renders the MinIO client alias command. +func exportMc(t exportTarget) string { + var b strings.Builder + b.WriteString(t.header()) + fmt.Fprintf(&b, "mc alias set %s %s %s %s --api S3v4 --path on\n", cfgShellValue(t.ProfileName), cfgShellValue(t.Endpoint), cfgShellValue(t.AccessKeyID), cfgShellValue(t.Secret)) + fmt.Fprintf(&b, "# mc ls %s/%s\n", t.ProfileName, t.bucketOrPlaceholder()) + return b.String() +} + +// exportS3cmd renders an s3cmd configuration. +func exportS3cmd(t exportTarget) string { + useHTTPS := "True" + if !t.Secure { + useHTTPS = "False" + } + var b strings.Builder + b.WriteString(t.header()) + b.WriteString("[default]\n") + fmt.Fprintf(&b, "access_key = %s\n", t.AccessKeyID) + fmt.Fprintf(&b, "secret_key = %s\n", t.Secret) + fmt.Fprintf(&b, "host_base = %s\n", t.Host) + fmt.Fprintf(&b, "host_bucket = %s\n", t.Host) + fmt.Fprintf(&b, "bucket_location = %s\n", t.SigningRegion) + fmt.Fprintf(&b, "use_https = %s\n", useHTTPS) + b.WriteString("signature_v2 = False\n") + b.WriteString("\n") + fmt.Fprintf(&b, "# s3cmd ls s3://%s\n", t.bucketOrPlaceholder()) + return b.String() +} + +// exportProcess renders the JSON an aws credential_process must print. +func exportProcess(t exportTarget) string { + payload := struct { + Version int `json:"Version"` + AccessKeyID string `json:"AccessKeyId"` + SecretAccessKey string `json:"SecretAccessKey"` + }{Version: 1, AccessKeyID: t.AccessKeyID, SecretAccessKey: t.Secret} + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) // keep the placeholder readable + if err := enc.Encode(payload); err != nil { + return "" + } + return b.String() // Encode appends the trailing newline +} + +// exportProfileName picks the default profile/remote name. +func exportProfileName(explicit string, b *objectstorage.Bucket, cred objectstorage.Credential) string { + if explicit = strings.TrimSpace(explicit); explicit != "" { + return explicit + } + if b != nil && b.Name != "" { + return "lsh-" + b.Name + } + if cred.Name != "" { + return "lsh-" + cred.Name + } + if cred.AccessKeyID != "" { + return "lsh-" + strings.ToLower(cred.AccessKeyID) + } + return "lsh" +} + +func runConfigureExport(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + format, _ := cmd.Flags().GetString(flagExportFormat) + if strings.TrimSpace(format) == "" { + return printErr(objectstorage.ErrUsagef("--format is required: one of %s", strings.Join(exportFormats, ", "))) + } + if _, err := renderExport(format, exportTarget{}); err != nil { + return printErr(err) + } + profileName, _ := cmd.Flags().GetString(flagExportProfileName) + noSecret, _ := cmd.Flags().GetBool(flagExportNoSecret) + accessKey, _ := cmd.Flags().GetString(flagAccessKey) + + var ( + bucket *objectstorage.Bucket + cred objectstorage.Credential + ) + if len(args) == 1 { + ref, err := objectstorage.ParseBucketOnly(args[0]) + if err != nil { + return printErr(err) + } + bucket, err = resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + if err := bucket.Validate(); err != nil { + return printErr(err) + } + cred, err = objectstorage.ResolveCredential(bucket, objectstorage.CredentialOptions{ + AccessKeyName: accessKey, + ProfileOverride: profileFlag(cmd), + }) + if err != nil { + return printErr(err) + } + } else { + endpoint := endpointOverride(cmd) + if endpoint == "" { + return printErr(objectstorage.ErrUsagef("pass s3://, or --access-key together with --endpoint-url (or %s)", objectstorage.EnvEndpointURL)) + } + var err error + cred, err = objectstorage.ResolveCredential(nil, objectstorage.CredentialOptions{ + AccessKeyName: accessKey, + ProfileOverride: profileFlag(cmd), + }) + if err != nil { + return printErr(err) + } + bucket = &objectstorage.Bucket{ + Endpoint: endpoint, + SigningRegion: signingRegionOverride(cmd), + StorageClass: cred.Key.StorageClass, + EndpointOverride: true, + } + } + + secret := cred.Secret() + if noSecret { + secret = exportSecretPlaceholder + } + target, err := newExportTarget( + exportProfileName(profileName, bucket, cred), + bucket.Endpoint, bucket.SigningRegion, bucket.BucketName, bucket.StorageClass, + cred.AccessKeyID, secret, + ) + if err != nil { + return printErr(err) + } + out, err := renderExport(format, target) + if err != nil { + return printErr(err) + } + fmt.Print(out) + if !noSecret { + objectstorage.Warnf("output contains a secret; do not commit it") + } + return nil +} diff --git a/cmd/s3/configure_export_test.go b/cmd/s3/configure_export_test.go new file mode 100644 index 0000000..10cf2a2 --- /dev/null +++ b/cmd/s3/configure_export_test.go @@ -0,0 +1,255 @@ +package s3 + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +// fixedExportTarget is the bucket/credential every golden test renders. +func fixedExportTarget(t *testing.T) exportTarget { + t.Helper() + target, err := newExportTarget("lsh-backups", "https://s3.us-central-1.storage.sh", "", "backups-7f3a", + objectstorage.ClassStandard, "XL68DDURVGUUOULWPCAE", "sEcReT/KeY+123") + if err != nil { + t.Fatalf("newExportTarget: %v", err) + } + return target +} + +func TestNewExportTargetDerivesHostAndRegion(t *testing.T) { + tg := fixedExportTarget(t) + if tg.Host != "s3.us-central-1.storage.sh" || !tg.Secure { + t.Fatalf("host/secure: %q %v", tg.Host, tg.Secure) + } + if tg.SigningRegion != "us-central-1" { + t.Fatalf("signing region should come from the endpoint, got %q", tg.SigningRegion) + } + plain, err := newExportTarget("p", "http://127.0.0.1:9000/", "us-east-1", "b", "", "AK", "SK") + if err != nil { + t.Fatal(err) + } + if plain.Secure || plain.Host != "127.0.0.1:9000" || plain.Endpoint != "http://127.0.0.1:9000" { + t.Fatalf("plain http target: %+v", plain) + } + if _, err := newExportTarget("p", "ftp://x", "", "", "", "", ""); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("bad scheme must be a usage error, got %v", err) + } +} + +func TestExportEnvGolden(t *testing.T) { + want := strings.Join([]string{ + "# bucket name on the endpoint: backups-7f3a", + "export AWS_ACCESS_KEY_ID=XL68DDURVGUUOULWPCAE", + "export AWS_SECRET_ACCESS_KEY=sEcReT/KeY+123", + "export AWS_ENDPOINT_URL_S3=https://s3.us-central-1.storage.sh", + "export AWS_REGION=us-central-1", + "export AWS_REQUEST_CHECKSUM_CALCULATION=when_required", + "export AWS_RESPONSE_CHECKSUM_VALIDATION=when_required", + "export LSH_S3_ACCESS_KEY_ID=XL68DDURVGUUOULWPCAE", + "export LSH_S3_SECRET_ACCESS_KEY=sEcReT/KeY+123", + "export LSH_S3_ENDPOINT_URL=https://s3.us-central-1.storage.sh", + "export LSH_S3_SIGNING_REGION=us-central-1", + "# aws s3 ls s3://backups-7f3a/", + "", + }, "\n") + if got := exportEnv(fixedExportTarget(t)); got != want { + t.Fatalf("env template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestExportAWSGolden(t *testing.T) { + want := strings.Join([]string{ + "# bucket name on the endpoint: backups-7f3a", + "# ~/.aws/config", + "[profile lsh-backups]", + "region = us-central-1", + "endpoint_url = https://s3.us-central-1.storage.sh", + "s3 =", + " addressing_style = path", + "request_checksum_calculation = when_required", + "response_checksum_validation = when_required", + "", + "# ~/.aws/credentials", + "[lsh-backups]", + "aws_access_key_id = XL68DDURVGUUOULWPCAE", + "aws_secret_access_key = sEcReT/KeY+123", + "", + "# aws --profile lsh-backups s3 ls s3://backups-7f3a/", + "", + }, "\n") + if got := exportAWS(fixedExportTarget(t)); got != want { + t.Fatalf("aws template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestExportRcloneGolden(t *testing.T) { + want := strings.Join([]string{ + "# bucket name on the endpoint: backups-7f3a", + "[lsh-backups]", + "type = s3", + "provider = Wasabi", + "env_auth = false", + "access_key_id = XL68DDURVGUUOULWPCAE", + "secret_access_key = sEcReT/KeY+123", + "endpoint = s3.us-central-1.storage.sh", + "region = us-central-1", + "force_path_style = true", + "", + "# rclone ls lsh-backups:backups-7f3a", + "", + }, "\n") + if got := exportRclone(fixedExportTarget(t)); got != want { + t.Fatalf("rclone template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + + hp := fixedExportTarget(t) + hp.StorageClass = objectstorage.ClassHighPerformance + if !strings.Contains(exportRclone(hp), "provider = Other\n") { + t.Fatal("high_performance must map to provider = Other") + } + unknown := fixedExportTarget(t) + unknown.StorageClass = "" + unknown.Host = "objects.tyo4.storage.sh" + if !strings.Contains(exportRclone(unknown), "provider = Other\n") { + t.Fatal("objects.* hosts without a class must map to provider = Other") + } +} + +func TestExportMcGolden(t *testing.T) { + want := "# bucket name on the endpoint: backups-7f3a\n" + + "mc alias set lsh-backups https://s3.us-central-1.storage.sh XL68DDURVGUUOULWPCAE sEcReT/KeY+123 --api S3v4 --path on\n" + + "# mc ls lsh-backups/backups-7f3a\n" + if got := exportMc(fixedExportTarget(t)); got != want { + t.Fatalf("mc template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestExportS3cmdGolden(t *testing.T) { + want := strings.Join([]string{ + "# bucket name on the endpoint: backups-7f3a", + "[default]", + "access_key = XL68DDURVGUUOULWPCAE", + "secret_key = sEcReT/KeY+123", + "host_base = s3.us-central-1.storage.sh", + "host_bucket = s3.us-central-1.storage.sh", + "bucket_location = us-central-1", + "use_https = True", + "signature_v2 = False", + "", + "# s3cmd ls s3://backups-7f3a", + "", + }, "\n") + if got := exportS3cmd(fixedExportTarget(t)); got != want { + t.Fatalf("s3cmd template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + plain := fixedExportTarget(t) + plain.Secure = false + if !strings.Contains(exportS3cmd(plain), "use_https = False\n") { + t.Fatal("http endpoints must render use_https = False") + } +} + +func TestExportProcessGolden(t *testing.T) { + want := `{"Version":1,"AccessKeyId":"XL68DDURVGUUOULWPCAE","SecretAccessKey":"sEcReT/KeY+123"}` + "\n" + got := exportProcess(fixedExportTarget(t)) + if got != want { + t.Fatalf("process template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + var decoded map[string]interface{} + if err := json.Unmarshal([]byte(got), &decoded); err != nil { + t.Fatalf("process output is not valid JSON: %v", err) + } + if decoded["Version"].(float64) != 1 { + t.Fatalf("Version must be 1: %v", decoded) + } +} + +func TestRenderExportDispatchAndNoSecret(t *testing.T) { + base := fixedExportTarget(t) + base.Secret = exportSecretPlaceholder + for _, format := range exportFormats { + out, err := renderExport(format, base) + if err != nil { + t.Fatalf("%s: %v", format, err) + } + if out == "" || !strings.HasSuffix(out, "\n") { + t.Fatalf("%s: output must be non-empty and newline-terminated: %q", format, out) + } + if strings.Contains(out, "sEcReT") { + t.Fatalf("%s: real secret leaked with --no-secret: %s", format, out) + } + if !strings.Contains(out, exportSecretPlaceholder) { + t.Fatalf("%s: placeholder missing: %s", format, out) + } + // Nothing time-bound: the templates never mention expiry. + if strings.Contains(strings.ToLower(out), "expire") { + t.Fatalf("%s: templates must not carry an expiry: %s", format, out) + } + } + if _, err := renderExport("toml", base); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("unknown format must be a usage error, got %v", err) + } + // Case-insensitive format names. + if _, err := renderExport("ENV", base); err != nil { + t.Fatalf("format names must be case-insensitive: %v", err) + } +} + +func TestExportWithoutBucketUsesPlaceholders(t *testing.T) { + tg, err := newExportTarget("lsh-ci", "https://objects.tyo4.storage.sh", "", "", "", "AK", "SK") + if err != nil { + t.Fatal(err) + } + env := exportEnv(tg) + if strings.Contains(env, "# bucket name on the endpoint") { + t.Fatalf("no bucket header expected when the bucket is unknown:\n%s", env) + } + if !strings.HasSuffix(env, "# aws s3 ls s3:///\n") { + t.Fatalf("example must use the placeholder:\n%s", env) + } + if tg.SigningRegion != "tyo4" { + t.Fatalf("signing region derived from objects. host: %q", tg.SigningRegion) + } +} + +func TestShellValueQuotesUnsafeCharacters(t *testing.T) { + cases := map[string]string{ + "abc123": "abc123", + "a/b+c=d": "a/b+c=d", + "has space": "'has space'", + "it's": `'it'\''s'`, + "": "''", + "dollar$sign": "'dollar$sign'", + } + for in, want := range cases { + if got := cfgShellValue(in); got != want { + t.Errorf("cfgShellValue(%q) = %q, want %q", in, got, want) + } + } +} + +func TestExportProfileNameDefaults(t *testing.T) { + b := &objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a"} + saved := objectstorage.NewCredential("AK", "SK", "saved") + saved.Name = "ci-deploy" + if got := exportProfileName(" custom ", b, saved); got != "custom" { + t.Fatalf("explicit name wins: %q", got) + } + if got := exportProfileName("", b, saved); got != "lsh-backups" { + t.Fatalf("bucket display name: %q", got) + } + if got := exportProfileName("", nil, saved); got != "lsh-ci-deploy" { + t.Fatalf("saved key name: %q", got) + } + env := objectstorage.NewCredential("AKIAENV", "SK", "env") + if got := exportProfileName("", nil, env); got != "lsh-akiaenv" { + t.Fatalf("env credential falls back to the key id: %q", got) + } + if got := exportProfileName("", nil, objectstorage.Credential{}); got != "lsh" { + t.Fatalf("last resort: %q", got) + } +} diff --git a/cmd/s3/configure_test.go b/cmd/s3/configure_test.go new file mode 100644 index 0000000..990491c --- /dev/null +++ b/cmd/s3/configure_test.go @@ -0,0 +1,254 @@ +package s3 + +import ( + "regexp" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +// TestConfigureDefaultNameIsPetName covers that configure uses the same +// pet-name generator as the dashboard for the default key name. +func TestConfigureDefaultNameIsPetName(t *testing.T) { + re := regexp.MustCompile(`^key-[a-z]+-[a-z]+-(std|hp)$`) + if got := generateKeyName(objectstorage.ClassHighPerformance); !re.MatchString(got) { + t.Errorf("generateKeyName(hp) = %q", got) + } +} + +// TestConfigureBucketSpecsUseSharedParser checks the --bucket spellings the +// configure help advertises through the parser shared with access-keys +// create (configure used to have its own copy). +func TestConfigureBucketSpecsUseSharedParser(t *testing.T) { + cases := map[string]bucketSpec{ + "backups": {Token: "backups", Permission: config.PermissionRW}, + "backups=rw": {Token: "backups", Permission: config.PermissionRW}, + "logs=readonly": {Token: "logs", Permission: config.PermissionReadOnly}, + " logs = RO ": {Token: "logs", Permission: config.PermissionReadOnly}, + "s3://bkt_1Gjb=rw": {Token: "bkt_1Gjb", Permission: config.PermissionRW}, + "s3://backups/": {Token: "backups", Permission: config.PermissionRW}, + "backups-7f3a=read": {Token: "backups-7f3a", Permission: config.PermissionReadOnly}, + "backups=read-write": {Token: "backups", Permission: config.PermissionRW}, + "bkt_9aQ2xL0mPq7Rt=ro": {Token: "bkt_9aQ2xL0mPq7Rt", Permission: config.PermissionReadOnly}, + } + for in, want := range cases { + specs, err := parseBucketSpecs([]string{in}) + if err != nil { + t.Errorf("parseBucketSpecs(%q): %v", in, err) + continue + } + if len(specs) != 1 || specs[0] != want { + t.Errorf("parseBucketSpecs(%q) = %+v, want %+v", in, specs, want) + } + } + for _, bad := range []string{"", "=rw", "backups=admin", "s3://"} { + if _, err := parseBucketSpec(bad); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("parseBucketSpec(%q) must fail with exit %d, got %v", bad, exitcode.Usage, err) + } + } + if _, err := parseBucketSpecs([]string{"backups", "backups=readonly"}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("duplicate bucket must be a usage error, got %v", err) + } + if specs, err := parseBucketSpecs(nil); err != nil || len(specs) != 0 { + t.Fatalf("empty input: %v %v", specs, err) + } +} + +func TestCfgValidateSelection(t *testing.T) { + std := func(id, name, project string) *objectstorage.Bucket { + return &objectstorage.Bucket{ID: id, Name: name, BucketName: name + "-1", StorageClass: objectstorage.ClassStandard, ProjectID: project, Site: "DAL"} + } + hp := func(id, name, site, project string) *objectstorage.Bucket { + return &objectstorage.Bucket{ID: id, Name: name, BucketName: name + "-1", StorageClass: objectstorage.ClassHighPerformance, ProjectID: project, Site: site} + } + + class, site, project, err := cfgValidateSelection([]cfgSelectedBucket{ + {Bucket: std("bkt_1", "backups", "proj_1"), Permission: config.PermissionRW}, + {Bucket: std("bkt_2", "logs", "proj_1"), Permission: config.PermissionReadOnly}, + }) + if err != nil || class != objectstorage.ClassStandard || site != "DAL" || project != "proj_1" { + t.Fatalf("same group must pass: %q %q %q %v", class, site, project, err) + } + + // Standard buckets on different sites are fine (one key covers every site). + b2 := std("bkt_2", "logs", "proj_1") + b2.Site = "NYC" + if _, _, _, err := cfgValidateSelection([]cfgSelectedBucket{{Bucket: std("bkt_1", "backups", "proj_1")}, {Bucket: b2}}); err != nil { + t.Fatalf("standard buckets may span sites: %v", err) + } + + // Empty site on one side is treated as unknown, not as a conflict. + unknownSite := hp("bkt_3", "fast", "", "proj_1") + _, site, _, err = cfgValidateSelection([]cfgSelectedBucket{{Bucket: hp("bkt_4", "faster", "TYO4", "proj_1")}, {Bucket: unknownSite}}) + if err != nil || site != "TYO4" { + t.Fatalf("unknown site must not conflict and the known one wins: %q %v", site, err) + } + + failures := []struct { + name string + sel []cfgSelectedBucket + want string + }{ + {"class", []cfgSelectedBucket{{Bucket: std("bkt_1", "backups", "proj_1")}, {Bucket: hp("bkt_3", "fast", "DAL", "proj_1")}}, "storage classes"}, + {"site", []cfgSelectedBucket{{Bucket: hp("bkt_3", "fast", "TYO4", "proj_1")}, {Bucket: hp("bkt_4", "faster", "DAL", "proj_1")}}, "different sites"}, + {"project", []cfgSelectedBucket{{Bucket: std("bkt_1", "backups", "proj_1")}, {Bucket: std("bkt_2", "logs", "proj_2")}}, "different projects"}, + {"empty", nil, "no bucket"}, + } + for _, f := range failures { + _, _, _, err := cfgValidateSelection(f.sel) + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), f.want) { + t.Errorf("%s: want usage error containing %q, got %v", f.name, f.want, err) + } + } +} + +// TestConfigureSiteFromBuckets covers the site inference of a fullaccess key +// (F16): a standard key is never created without a site; the site comes from +// the project's buckets of the key's class, the raw site map filling gaps. +func TestConfigureSiteFromBuckets(t *testing.T) { + std := func(id, site string) *objectstorage.Bucket { + return &objectstorage.Bucket{ID: id, Name: id, StorageClass: objectstorage.ClassStandard, Site: site} + } + hp := func(id, site string) *objectstorage.Bucket { + return &objectstorage.Bucket{ID: id, Name: id, StorageClass: objectstorage.ClassHighPerformance, Site: site} + } + + // Standard: any site of a standard bucket is valid; the choice is + // deterministic (sorted) and other classes are ignored. + site, err := configureSiteFromBuckets(objectstorage.ClassStandard, "proj_1", + []*objectstorage.Bucket{std("bkt_1", "nyc"), std("bkt_2", "DAL"), hp("bkt_3", "TYO4")}, nil) + if err != nil || site != "DAL" { + t.Fatalf("standard site: %q %v", site, err) + } + // The raw site map fills buckets whose SDK model lost the site. + site, err = configureSiteFromBuckets(objectstorage.ClassStandard, "proj_1", + []*objectstorage.Bucket{std("bkt_1", "")}, map[string]string{"bkt_1": "sao"}) + if err != nil || site != "SAO" { + t.Fatalf("site from raw map: %q %v", site, err) + } + // Nothing of the class → "" (the caller asks or fails with exit 2). + site, err = configureSiteFromBuckets(objectstorage.ClassStandard, "proj_1", []*objectstorage.Bucket{hp("bkt_3", "TYO4")}, nil) + if err != nil || site != "" { + t.Fatalf("no standard buckets must yield an empty site: %q %v", site, err) + } + site, err = configureSiteFromBuckets(objectstorage.ClassStandard, "proj_1", nil, nil) + if err != nil || site != "" { + t.Fatalf("no buckets at all: %q %v", site, err) + } + // high_performance: one site is fine, several are ambiguous. + site, err = configureSiteFromBuckets(objectstorage.ClassHighPerformance, "proj_1", []*objectstorage.Bucket{hp("bkt_3", "TYO4"), hp("bkt_4", "tyo4")}, nil) + if err != nil || site != "TYO4" { + t.Fatalf("single hp site: %q %v", site, err) + } + _, err = configureSiteFromBuckets(objectstorage.ClassHighPerformance, "proj_1", []*objectstorage.Bucket{hp("bkt_3", "TYO4"), hp("bkt_4", "DAL")}, nil) + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "DAL, TYO4") || !strings.Contains(err.Error(), "--region") { + t.Fatalf("several hp sites must be a usage error naming them and --region, got %v", err) + } +} + +func TestConfigurePlanLineAndBuckets(t *testing.T) { + full := configurePlan{Name: "lsh-lanusse-tyo4", Project: "my-project", StorageClass: objectstorage.ClassHighPerformance, Site: "TYO4", Scope: config.ScopeFullAccess, Save: true} + want := `(dryrun) create access key "lsh-lanusse-tyo4" in project my-project for all high_performance buckets in TYO4 (fullaccess) and save it in the active profile` + if got := configurePlanLine(full); got != want { + t.Fatalf("fullaccess plan line:\n got %s\nwant %s", got, want) + } + // A standard fullaccess key keeps its site too (it is no longer cleared). + std := configurePlan{Name: "lsh-lanusse-standard", Project: "my-project", StorageClass: objectstorage.ClassStandard, Site: "DAL", Scope: config.ScopeFullAccess, Save: true} + if got := configurePlanLine(std); !strings.Contains(got, "all standard buckets in DAL (fullaccess)") { + t.Fatalf("standard plan line must name the site: %s", got) + } + limited := configurePlan{Name: "ci", Project: "proj_1", StorageClass: objectstorage.ClassStandard, Scope: config.ScopeLimitedAccess, + Buckets: map[string]string{"logs": config.PermissionReadOnly, "backups": config.PermissionRW}, Save: false} + want = `(dryrun) create access key "ci" in project proj_1 for buckets backups=rw logs=readonly (limited_access) and print the secret once (not saved)` + if got := configurePlanLine(limited); got != want { + t.Fatalf("limited plan line:\n got %s\nwant %s", got, want) + } + // Empty cells use the placeholder shared by the whole group. + if got := cfgFormatBucketPerms(nil); got != emptyCell { + t.Fatalf("empty perms placeholder: %q", got) + } + row := limited.TableRow() + if row["buckets"].Value != "backups=rw logs=readonly" || row["status"].Value != "created" || row["save"].Value != "no" || row["site"].Value != emptyCell { + t.Fatalf("table row: %+v", row) + } + if _, ok := row["saved_as"]; ok { + t.Fatal("saved_as cell only when the key was stored") + } + saved := full + saved.SavedAs, saved.Profile = "lsh-lanusse-tyo4-2", "default" + if got := saved.TableRow()["saved_as"].Value; got != "lsh-lanusse-tyo4-2" { + t.Fatalf("saved_as cell: %q", got) + } +} + +func TestConfigureStatePerms(t *testing.T) { + st := &configureState{Scope: config.ScopeLimitedAccess, Selected: []cfgSelectedBucket{ + {Bucket: &objectstorage.Bucket{ID: "bkt_1", Name: "backups"}, Permission: config.PermissionRW}, + {Bucket: &objectstorage.Bucket{ID: "bkt_2", Name: "logs"}, Permission: config.PermissionReadOnly}, + }} + if p := st.bucketPerms(); p["bkt_1"] != config.PermissionRW || p["bkt_2"] != config.PermissionReadOnly || len(p) != 2 { + t.Fatalf("bucketPerms by id: %v", p) + } + if p := st.bucketNamePerms(); p["backups"] != config.PermissionRW || p["logs"] != config.PermissionReadOnly { + t.Fatalf("bucketNamePerms by name: %v", p) + } + st.Scope = config.ScopeFullAccess + if st.bucketPerms() != nil || st.bucketNamePerms() != nil { + t.Fatal("fullaccess keys carry no bucket map") + } + if (&configureState{ProjectToken: "my-project", ProjectID: "proj_1"}).project() != "my-project" || (&configureState{ProjectID: "proj_1"}).project() != "proj_1" { + t.Fatal("project() prefers the token as given, then the resolved ID") + } +} + +func TestConfigureNeedsPromptIsUsageError(t *testing.T) { + err := configureNeedsPrompt("the project") + if exitcode.Of(err) != exitcode.Usage { + t.Fatalf("exit code %d, want %d", exitcode.Of(err), exitcode.Usage) + } + msg := err.Error() + for _, want := range []string{"the project", "lsh s3 access-keys create --bucket --save", objectstorage.EnvAccessKeyID, objectstorage.EnvSecretAccessKey} { + if !strings.Contains(msg, want) { + t.Errorf("message must mention %q: %s", want, msg) + } + } +} + +// TestConfigureFlagsParse exercises the flag parsing that now goes through +// the shared helpers (ParseStorageClass, regionFlag, parseBucketSpecs). +func TestConfigureFlagsParse(t *testing.T) { + parse := func(args ...string) (configureOptions, error) { + cmd := NewConfigureCmd() + cmd.SetArgs(args) + if err := cmd.ParseFlags(args); err != nil { + t.Fatalf("ParseFlags(%v): %v", args, err) + } + return parseConfigureOptions(cmd) + } + o, err := parse("--storage-class", "high-performance", "--region", "tyo4", "--bucket", "backups=rw", "--bucket", "logs=ro", "--name", " laptop ") + if err != nil { + t.Fatal(err) + } + if o.StorageClass != objectstorage.ClassHighPerformance || o.Site != "TYO4" || o.Name != "laptop" || !o.Save { + t.Fatalf("options: %+v", o) + } + if len(o.Buckets) != 2 || o.Buckets[1].Token != "logs" || o.Buckets[1].Permission != config.PermissionReadOnly { + t.Fatalf("bucket specs: %+v", o.Buckets) + } + if o, err := parse("--save=false"); err != nil || o.Save { + t.Fatalf("--save=false: %+v %v", o, err) + } + for _, bad := range [][]string{ + {"--storage-class", "glacier"}, + {"--region", "us-east-1"}, + {"--bucket", "backups=admin"}, + {"--all-buckets", "--bucket", "backups"}, + } { + if _, err := parse(bad...); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("%v must be a usage error, got %v", bad, err) + } + } +} diff --git a/cmd/s3/cp.go b/cmd/s3/cp.go new file mode 100644 index 0000000..4af24a5 --- /dev/null +++ b/cmd/s3/cp.go @@ -0,0 +1,53 @@ +package s3 + +import ( + "github.com/spf13/cobra" +) + +// NewCpCmd builds `lsh s3 cp `. +func NewCpCmd() *cobra.Command { + var f transferFlags + cmd := newCmd(&cobra.Command{ + Use: "cp ", + Aliases: []string{"copy", "upload", "download"}, + Short: "Copy files and objects", + Long: `Copy a local file to a bucket, an object to the local disk, or an object to +another object (server-side, same endpoint; when the two buckets resolve +different access keys the object is streamed through this machine instead, +since one signed copy request cannot read with one key and write with +another). The destination is overwritten +by default (--no-overwrite to skip existing ones), +--recursive copies whole directories or prefixes, and --exclude/--include +filter relative paths (rules apply in order; the last match wins). + +` + destinationRulesHelp + ` + +Uploads larger than 16 MiB use multipart; an interrupted upload is aborted. +To clean up leftovers from killed processes create a lifecycle rule: +'lsh s3 lifecycle create s3:// --abort-incomplete-multipart-days 1'. + +Streams from stdin ('-') are always multipart with 16 MiB parts, enough for +160 GiB. For larger streams pass --expected-size so the parts grow to +fit S3's 10000-part limit; the value is only a hint that also drives the +progress meter, and the upload succeeds whatever the real length turns out to be. + +The Content-Type is taken from --content-type, then the file extension, then +the first bytes of the file (--no-guess-mime-type stores binary/octet-stream). +Output lines (upload:/download:/copy:) go to stdout; progress, hints and +errors to stderr. --dry-run (or --dryrun) prints the plan without writing.`, + Example: ` lsh s3 cp ./dump.sql s3://backups/2026/09/ + lsh s3 cp s3://backups/2026/09/dump.sql ./restore/ + lsh s3 cp ./site s3://www --recursive --exclude "*" --include "*.html" --content-type text/html + lsh s3 cp s3://backups/2026/ ./backups/ --recursive --dryrun + pg_dump mydb | lsh s3 cp - s3://backups/mydb.sql + tar cz /data | lsh s3 cp - s3://backups/data.tgz --expected-size 500000000000 # approximate; sizes parts for streams over 160 GiB + lsh s3 cp s3://backups/report.pdf - > report.pdf + lsh s3 cp s3://backups/a.txt s3://archive/2026/a.txt --metadata owner=ops,team=infra`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return runCopyCommand(cmd, args, &f, false) + }, + }) + addTransferFlags(cmd, &f, true) + return cmd +} diff --git a/cmd/s3/e2e_test.go b/cmd/s3/e2e_test.go new file mode 100644 index 0000000..d4723ee --- /dev/null +++ b/cmd/s3/e2e_test.go @@ -0,0 +1,791 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/objectstorage/s3test" +) + +// The binary-level tests below compile lsh once per test process and drive +// the real cobra wiring through exec. They are skipped with -short. +var ( + lshBinOnce sync.Once + lshBinDir string + lshBinPath string + lshBinErr error +) + +// TestMain removes the shared binary built by buildLshBinary once every test +// in the package has run (t.TempDir would delete it after the first test). +func TestMain(m *testing.M) { + code := m.Run() + if lshBinDir != "" { + _ = os.RemoveAll(lshBinDir) + } + os.Exit(code) +} + +// buildLshBinary builds the lsh binary once and returns its path. Every +// binary-level test shares the same build. +func buildLshBinary(t *testing.T) string { + t.Helper() + if testing.Short() { + t.Skip("builds the binary; skipped with -short") + } + lshBinOnce.Do(func() { + root, err := filepath.Abs("../..") + if err != nil { + lshBinErr = err + return + } + dir, err := os.MkdirTemp("", "lsh-e2e-") + if err != nil { + lshBinErr = err + return + } + lshBinDir = dir + lshBinPath = filepath.Join(dir, "lsh-e2e") + build := exec.Command("go", "build", "-o", lshBinPath, ".") + build.Dir = root + if out, err := build.CombinedOutput(); err != nil { + lshBinErr = fmt.Errorf("go build: %v\n%s", err, out) + } + }) + if lshBinErr != nil { + t.Fatal(lshBinErr) + } + return lshBinPath +} + +// runResult is the outcome of one binary invocation. +type runResult struct { + stdout, stderr string + code int +} + +// runLsh executes the binary with env and args in dir, returning stdout, +// stderr and the exit code. Any failure that is not a non-zero exit is fatal. +func runLsh(t *testing.T, bin, dir string, env []string, args ...string) runResult { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, bin, args...) + cmd.Dir = dir + cmd.Env = env + var o, e bytes.Buffer + cmd.Stdout, cmd.Stderr = &o, &e + err := cmd.Run() + res := runResult{stdout: o.String(), stderr: e.String()} + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + res.code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("%v: %v\nstdout=%q\nstderr=%q", args, err, res.stdout, res.stderr) + } + return res +} + +// TestBinaryAgainstFakeS3 builds the real lsh binary and drives the cobra +// wiring end to end in the API-less mode (LSH_S3_ENDPOINT_URL + env +// credentials), covering flag parsing, output formats, dry-run, confirmation +// gating and exit codes that the unit tests bypass. +func TestBinaryAgainstFakeS3(t *testing.T) { + bin := buildLshBinary(t) + + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + srv.AddObject("backups-7f3a", "2026/09/a.sql", []byte("aaa"), "application/sql") + srv.AddObject("backups-7f3a", "2026/09/b.sql", []byte("bbbb"), "application/sql") + srv.AddObject("backups-7f3a", "readme.txt", []byte("hi"), "text/plain") + + home := t.TempDir() + work := t.TempDir() + env := append(os.Environ(), + "HOME="+home, + "LSH_S3_ENDPOINT_URL="+srv.URL(), + "LSH_S3_ACCESS_KEY_ID=AKIAEXAMPLE", + "LSH_S3_SECRET_ACCESS_KEY=topsecret", + "LATITUDESH_TOKEN=", + "LSH_S3_USE_AWS_ENV=", + "LSH_OUTPUT=", + "LSH_CLASSIC_OUTPUT=true", + "NO_COLOR=1", + ) + + run := func(args ...string) (stdout, stderr string, code int) { + r := runLsh(t, bin, work, env, args...) + return r.stdout, r.stderr, r.code + } + + // ls default: the shared lsh table (headers + borders), like every other + // list command; the prefix shows as a PRE row and the object by key. + out, errOut, code := run("s3", "ls", "s3://backups-7f3a/") + if code != 0 { + t.Fatalf("ls exit %d: %s%s", code, out, errOut) + } + if !strings.Contains(out, "KEY") || !strings.Contains(out, "2026/") || !strings.Contains(out, "readme.txt") { + t.Errorf("ls table output unexpected:\n%s", out) + } + if strings.Contains(out, "topsecret") || strings.Contains(errOut, "topsecret") { + t.Fatal("secret leaked") + } + + // ls -o json --query works (aws s3 ls ignores --output). + out, errOut, code = run("s3", "ls", "s3://backups-7f3a/2026/09/", "-o", "json", "--query", "[].key") + if code != 0 { + t.Fatalf("ls json exit %d: %s%s", code, out, errOut) + } + var keys []string + if err := json.Unmarshal([]byte(out), &keys); err != nil || len(keys) != 2 { + t.Errorf("ls -o json --query: %v %q", err, out) + } + + // cp upload with trailing slash keeps the file name; cp download to a + // directory; cp to stdout prints only the payload. + src := filepath.Join(work, "dump.sql") + if err := os.WriteFile(src, []byte("select 1;"), 0o600); err != nil { + t.Fatal(err) + } + out, errOut, code = run("s3", "cp", src, "s3://backups-7f3a/2026/10/") + if code != 0 || !strings.HasPrefix(out, "upload: ") || !strings.Contains(out, "to s3://backups-7f3a/2026/10/dump.sql") { + t.Fatalf("cp upload: exit %d out=%q err=%q", code, out, errOut) + } + if o := srv.Object("backups-7f3a", "2026/10/dump.sql"); o == nil || string(o.Data) != "select 1;" || o.ContentType == "" { + t.Fatalf("uploaded object missing or wrong: %+v", o) + } + if srv.HasChecksumHeaders() { + t.Error("upload sent checksum headers") + } + dest := filepath.Join(work, "restore") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + out, errOut, code = run("s3", "cp", "s3://backups-7f3a/2026/10/dump.sql", dest+string(os.PathSeparator)) + if code != 0 || !strings.HasPrefix(out, "download: ") { + t.Fatalf("cp download: exit %d out=%q err=%q", code, out, errOut) + } + if data, err := os.ReadFile(filepath.Join(dest, "dump.sql")); err != nil || string(data) != "select 1;" { + t.Fatalf("downloaded file: %v %q", err, data) + } + out, _, code = run("s3", "cp", "s3://backups-7f3a/readme.txt", "-") + if code != 0 || out != "hi" { + t.Errorf("cp to stdout: exit %d out=%q", code, out) + } + + // stat object. + out, errOut, code = run("s3", "stat", "s3://backups-7f3a/readme.txt", "-o", "json") + if code != 0 || !strings.Contains(out, `"key": "readme.txt"`) { + t.Errorf("stat: exit %d out=%q err=%q", code, out, errOut) + } + + // presign prints only a URL. + out, _, code = run("s3", "presign", "s3://backups-7f3a/readme.txt", "--expires-in", "15m") + if code != 0 || !strings.HasPrefix(strings.TrimSpace(out), srv.URL()+"/backups-7f3a/readme.txt?") || strings.Count(strings.TrimSpace(out), "\n") != 0 { + t.Errorf("presign: exit %d out=%q", code, out) + } + if _, _, code = run("s3", "presign", "s3://backups-7f3a/readme.txt", "--expires-in", "8d"); code != 2 { + t.Errorf("presign above 7d must be a usage error (2), got %d", code) + } + + // rm --recursive --dryrun: plan lines, zero writes. + srv.ResetRequests() + out, errOut, code = run("s3", "rm", "s3://backups-7f3a/2026/09/", "--recursive", "--dryrun") + if code != 0 || !strings.Contains(out, "(dryrun) delete: s3://backups-7f3a/2026/09/a.sql") { + t.Errorf("rm dryrun: exit %d out=%q err=%q", code, out, errOut) + } + if len(srv.WriteRequests()) != 0 { + t.Error("dry-run issued write requests") + } + + // rm --recursive without --yes in a non-interactive session is refused (7). + if _, errOut, code = run("s3", "rm", "s3://backups-7f3a/2026/09/", "--recursive"); code != 7 || !strings.Contains(errOut, "--yes") { + t.Errorf("rm recursive non-interactive: exit %d err=%q", code, errOut) + } + // Whole bucket requires --all. + if _, _, code = run("s3", "rm", "s3://backups-7f3a/", "--recursive", "--yes"); code != 2 { + t.Errorf("rm whole bucket without --all must exit 2, got %d", code) + } + // With --yes it deletes. + out, _, code = run("s3", "rm", "s3://backups-7f3a/2026/09/", "--recursive", "--yes") + if code != 0 || strings.Count(out, "delete: ") != 2 || srv.Object("backups-7f3a", "2026/09/a.sql") != nil { + t.Errorf("rm recursive: exit %d out=%q", code, out) + } + // Single delete of a missing key is idempotent. + if _, _, code = run("s3", "rm", "s3://backups-7f3a/does-not-exist"); code != 0 { + t.Errorf("rm missing key must exit 0, got %d", code) + } + + // Unknown bucket → 3 (not found). + if _, errOut, code = run("s3", "ls", "s3://nope-1234/"); code != 3 { + t.Errorf("ls unknown bucket: exit %d err=%q", code, errOut) + } + + // Unsupported aws flag gets an explanation, exit 2. + if _, errOut, code = run("s3", "cp", src, "s3://backups-7f3a/x", "--acl", "public-read"); code != 2 || !strings.Contains(errOut, "--acl") { + t.Errorf("--acl: exit %d err=%q", code, errOut) + } + + // rb needs the API: refused in endpoint mode with a usage error. + if _, _, code = run("s3", "rb", "s3://backups-7f3a"); code == 0 { + t.Error("rb in endpoint-override mode must fail") + } + + // mb is an API command: in endpoint-override mode it refuses with a usage + // error; without the override, --dryrun prints the plan even when not + // logged in. + if _, _, code = run("s3", "mb", "s3://newbucket", "--region", "DAL", "--project", "proj_1", "--dryrun"); code != 2 { + t.Errorf("mb with LSH_S3_ENDPOINT_URL set must exit 2, got %d", code) + } + { + noOverride := append(append([]string{}, env...), "LSH_S3_ENDPOINT_URL=") + r := runLsh(t, bin, work, noOverride, "s3", "mb", "s3://newbucket", "--region", "DAL", "--project", "proj_1", "--dryrun") + if r.code != 0 || !strings.Contains(r.stdout, "(dryrun) make_bucket: s3://newbucket") { + t.Errorf("mb dryrun: exit %d out=%q stderr=%q", r.code, r.stdout, r.stderr) + } + } + + // Missing credentials → 4 with the variable names in the message. + noCreds := append(append([]string{}, env...), "LSH_S3_ACCESS_KEY_ID=", "LSH_S3_SECRET_ACCESS_KEY=") + r := runLsh(t, bin, work, noCreds, "s3", "ls", "s3://backups-7f3a/") + if r.code != 4 || !strings.Contains(r.stderr, "LSH_S3_ACCESS_KEY_ID") { + t.Errorf("no credentials: exit %d stderr=%q", r.code, r.stderr) + } +} + +// Fixed identifiers served by the fake Latitude API. +const ( + e2eSecret = "S3CR3T-e2e-xyz" + e2eAccessKeyID = "AKIAE2E" + e2eBucketName = "bkt-e2e-7f3a" + e2eBucketID = "bkt_e2e" +) + +// fakeLatitudeAPI is a minimal JSON:API server for the /storage endpoints the +// access-keys and bucket resolution paths call. It accepts any Authorization +// and API-Version header without validation and records what it served. +type fakeLatitudeAPI struct { + srv *httptest.Server + s3URL string + + mu sync.Mutex + requests []string // "METHOD /path" + created []string // names of the keys created through POST +} + +func newFakeLatitudeAPI(s3URL string) *fakeLatitudeAPI { + f := &fakeLatitudeAPI{s3URL: s3URL} + f.srv = httptest.NewServer(http.HandlerFunc(f.handle)) + return f +} + +func (f *fakeLatitudeAPI) Close() { f.srv.Close() } + +// HostPort returns "127.0.0.1:" for --hostname. +func (f *fakeLatitudeAPI) HostPort() string { return strings.TrimPrefix(f.srv.URL, "http://") } + +func (f *fakeLatitudeAPI) Requests() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.requests...) +} + +func (f *fakeLatitudeAPI) Created() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.created...) +} + +func (f *fakeLatitudeAPI) bucket() map[string]any { + return map[string]any{ + "id": e2eBucketID, + "type": "object_storages", + "attributes": map[string]any{ + "name": "e2e", + "bucket_name": e2eBucketName, + "storage_class": "standard", + "endpoint": f.s3URL, + "versioning": false, + "locking": false, + // The real API serializes this as "" for buckets without object + // lock; the SDK model wants a number, so the CLI must normalize it. + "retention_period": "", + "source": "default", + "region": map[string]any{ + "city": "Dallas", + "site": map[string]any{"slug": "DAL"}, + }, + "project": map[string]any{"id": "proj_1", "slug": "e2e-project", "name": "E2E"}, + }, + } +} + +func (f *fakeLatitudeAPI) handle(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + f.mu.Lock() + f.requests = append(f.requests, r.Method+" "+r.URL.Path) + f.mu.Unlock() + + writeJSON := func(status int, v any) { + w.Header().Set("Content-Type", "application/vnd.api+json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) + } + path := strings.TrimSuffix(r.URL.Path, "/") + switch { + case r.Method == http.MethodGet && path == "/storage/buckets": + writeJSON(200, map[string]any{"data": []any{f.bucket()}}) + case r.Method == http.MethodGet && path == "/storage/buckets/"+e2eBucketID: + writeJSON(200, map[string]any{"data": f.bucket()}) + case r.Method == http.MethodPost && path == "/storage/access_keys": + var req struct { + Data struct { + Attributes struct { + Name string `json:"name"` + } `json:"attributes"` + } `json:"data"` + } + _ = json.Unmarshal(body, &req) + name := req.Data.Attributes.Name + f.mu.Lock() + f.created = append(f.created, name) + f.mu.Unlock() + writeJSON(201, map[string]any{"data": map[string]any{ + "type": "access_keys", + "attributes": map[string]any{ + "access_key": map[string]any{ + "access_key_id": e2eAccessKeyID, + "secret_access_key": e2eSecret, + "name": name, + "username": "e2e+x@latitude.sh", + "status": "Active", + }, + }, + }}) + case r.Method == http.MethodGet && path == "/storage/access_keys": + writeJSON(200, map[string]any{"data": map[string]any{ + "standard": []any{map[string]any{ + "name": "k1", + "username": "e2e+x@latitude.sh", + "access_key_id": e2eAccessKeyID, + "status": "Active", + "created_at": "2026-09-07T10:00:00Z", + "region": "DAL", + "access": "rw", + "buckets": []string{e2eBucketName}, + }}, + "high_performance": []any{}, + }}) + case r.Method == http.MethodDelete && strings.HasPrefix(path, "/storage/access_keys/"): + w.WriteHeader(http.StatusNoContent) + default: + writeJSON(404, map[string]any{"errors": []any{map[string]any{ + "status": "404", "title": "Not Found", "detail": r.Method + " " + r.URL.Path, + }}}) + } +} + +// writeE2EConfig writes ~/.config/lsh/config.json under home with a profile +// named e2e as the default. jsonDefault adds the top-level "json": true that +// the config file uses to make json the default output format. +func writeE2EConfig(t *testing.T, home string, jsonDefault bool) string { + t.Helper() + cfg := map[string]any{ + "default_profile": "e2e", + "profiles": map[string]any{ + "e2e": map[string]any{"authorization": "test-token"}, + }, + } + if jsonDefault { + cfg["json"] = true + } + dir := filepath.Join(home, ".config", "lsh") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + data, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + t.Fatal(err) + } + p := filepath.Join(dir, "config.json") + if err := os.WriteFile(p, data, 0o600); err != nil { + t.Fatal(err) + } + return p +} + +// e2eEnv is the environment for the API-backed runs: token from the +// environment, no S3 endpoint override, no env credentials, and no inherited +// output preference unless the case sets one. +func e2eEnv(home string, extra ...string) []string { + env := append(os.Environ(), + "HOME="+home, + "LATITUDESH_TOKEN=test-token", + "LSH_S3_ENDPOINT_URL=", + "LSH_S3_ACCESS_KEY_ID=", + "LSH_S3_SECRET_ACCESS_KEY=", + "LSH_S3_USE_AWS_ENV=", + "LSH_OUTPUT=", + "LSH_PROFILE=", + "LSH_PROJECT=", + "LSH_CLASSIC_OUTPUT=true", + "NO_COLOR=1", + ) + return append(env, extra...) +} + +// jsonField decodes structured output (one object, or a one-element array) +// and returns the string value of field. +func jsonField(t *testing.T, out, field string) (string, bool) { + t.Helper() + var v any + if err := json.Unmarshal([]byte(out), &v); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + obj, ok := v.(map[string]any) + if arr, isArr := v.([]any); isArr { + if len(arr) != 1 { + t.Fatalf("expected one JSON object, got %d: %s", len(arr), out) + } + obj, ok = arr[0].(map[string]any) + } + if !ok { + t.Fatalf("expected a JSON object: %s", out) + } + val, present := obj[field] + if !present { + return "", false + } + s, _ := val.(string) + return s, true +} + +// TestBinaryNoSecretContract drives the binary against a fake Latitude API +// and the fake S3 endpoint to pin the secret-display contract of +// `access-keys create` (spec section 10) and the cobra error handling of +// newCmd (one message, exit 2). Unless a case says otherwise, neither stdout +// nor stderr may contain the secret. +func TestBinaryNoSecretContract(t *testing.T) { + bin := buildLshBinary(t) + + s3srv := s3test.New() + defer s3srv.Close() + s3srv.CreateBucket(e2eBucketName) + s3srv.AddObject(e2eBucketName, "readme.txt", []byte("hi"), "text/plain") + + api := newFakeLatitudeAPI(s3srv.URL()) + defer api.Close() + + work := t.TempDir() + if err := os.WriteFile(filepath.Join(work, "f"), []byte("payload"), 0o600); err != nil { + t.Fatal(err) + } + apiArgs := []string{"--hostname", api.HostPort(), "--scheme", "http"} + run := func(env []string, args ...string) runResult { + t.Helper() + return runLsh(t, bin, work, env, append(append([]string{}, args...), apiArgs...)...) + } + noSecret := func(name string, r runResult) { + t.Helper() + if strings.Contains(r.stdout, e2eSecret) { + t.Errorf("%s: secret leaked on stdout:\n%s", name, r.stdout) + } + if strings.Contains(r.stderr, e2eSecret) { + t.Errorf("%s: secret leaked on stderr:\n%s", name, r.stderr) + } + } + createArgs := []string{"s3", "access-keys", "create", "--bucket", "e2e", "--name", "k1"} + + home := t.TempDir() + writeE2EConfig(t, home, false) + env := e2eEnv(home) + + // 8. Cobra errors: printed once, exit 2, no usage dump. These never reach + // the API, so they run first and report even when the API cases cannot. + r := run(env, "s3", "ls", "--bogus-flag") + if r.code != 2 || strings.Count(r.stderr, "unknown flag") != 1 { + t.Errorf("ls --bogus-flag: exit %d (want 2), stderr must mention 'unknown flag' once:\n%s", r.code, r.stderr) + } + if strings.Contains(r.stderr, "Usage:") { + t.Errorf("ls --bogus-flag: usage must not be echoed:\n%s", r.stderr) + } + r = run(env, "s3", "stat") + if r.code != 2 || strings.TrimSpace(r.stderr) == "" || strings.Count(r.stderr, "accepts 1 arg") != 1 { + t.Errorf("stat without args: exit %d (want 2), stderr=%q", r.code, r.stderr) + } + r = run(env, "s3", "cp", "onlyone") + if r.code != 2 || strings.TrimSpace(r.stderr) == "" { + t.Errorf("cp with one arg: exit %d (want 2), stderr=%q", r.code, r.stderr) + } + + // 9. aws-style --region on an object command is redirected to + // --signing-region with exit 2 (checked in PreRunE, before any API call). + r = run(env, "s3", "cp", "./f", "s3://e2e/", "--region", "us-east-1") + if r.code != 2 || !strings.Contains(r.stderr, "--signing-region") { + t.Errorf("cp --region: exit %d (want 2), stderr must explain --signing-region:\n%s", r.code, r.stderr) + } + if strings.Contains(r.stderr, "unknown flag") { + t.Errorf("cp --region must not be reported as an unknown flag:\n%s", r.stderr) + } + + // 1. Human output, not saved: the secret is shown in clear, exactly once, + // on stdout (it cannot be retrieved again), never on stderr. + r = run(env, createArgs...) + if r.code != 0 { + reqs := api.Requests() + hint := "" + if len(reqs) == 0 { + hint = "\nthe fake API received no request: the SDK client ignores --hostname/--scheme (objectstorage.NewAPIClient builds sdk.New without WithServerURL)" + } + t.Fatalf("create (human): exit %d\nstdout=%s\nstderr=%s\napi requests=%v%s", r.code, r.stdout, r.stderr, reqs, hint) + } + if n := strings.Count(r.stdout, e2eSecret); n != 1 { + t.Errorf("create (human): secret must be printed exactly once on stdout, found %d:\n%s", n, r.stdout) + } + if strings.Contains(r.stderr, e2eSecret) { + t.Errorf("create (human): secret on stderr:\n%s", r.stderr) + } + if !strings.Contains(r.stdout, e2eAccessKeyID) { + t.Errorf("create (human): access key id missing:\n%s", r.stdout) + } + if got := api.Created(); len(got) != 1 || got[0] != "k1" { + t.Errorf("fake API saw created keys %v, want [k1]", got) + } + + // 2. LSH_OUTPUT=json without -o: structured output does not embed the + // secret and stderr tells how to get it. + r = run(e2eEnv(home, "LSH_OUTPUT=json"), createArgs...) + if r.code != 0 { + t.Fatalf("create (LSH_OUTPUT=json): exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("create (LSH_OUTPUT=json)", r) + if _, present := jsonField(t, r.stdout, "secret_access_key"); present { + t.Errorf("create (LSH_OUTPUT=json): secret_access_key present in JSON:\n%s", r.stdout) + } + if !strings.Contains(r.stderr, "--show-secret") || !strings.Contains(r.stderr, "-o json") { + t.Errorf("create (LSH_OUTPUT=json): stderr must point at --show-secret / -o json:\n%s", r.stderr) + } + + // 3. Explicit -o json embeds the secret. + r = run(env, append(createArgs, "-o", "json")...) + if r.code != 0 { + t.Fatalf("create (-o json): exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + if got, _ := jsonField(t, r.stdout, "secret_access_key"); got != e2eSecret { + t.Errorf("create (-o json): secret_access_key=%q, want %q:\n%s", got, e2eSecret, r.stdout) + } + if strings.Contains(r.stderr, e2eSecret) { + t.Errorf("create (-o json): secret on stderr:\n%s", r.stderr) + } + + // 4. --json on the command line counts as explicit too. + r = run(env, append(createArgs, "--json")...) + if r.code != 0 { + t.Fatalf("create (--json): exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + if got, _ := jsonField(t, r.stdout, "secret_access_key"); got != e2eSecret { + t.Errorf("create (--json): secret_access_key=%q, want %q:\n%s", got, e2eSecret, r.stdout) + } + + // 5. "json": true in the config file is not an explicit request. + { + jsonHome := t.TempDir() + writeE2EConfig(t, jsonHome, true) + r = run(e2eEnv(jsonHome), createArgs...) + if r.code != 0 { + t.Fatalf("create (config json=true): exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("create (config json=true)", r) + if _, present := jsonField(t, r.stdout, "secret_access_key"); present { + t.Errorf("create (config json=true): secret_access_key present in JSON:\n%s", r.stdout) + } + if !strings.Contains(r.stderr, "--show-secret") { + t.Errorf("create (config json=true): stderr must point at --show-secret:\n%s", r.stderr) + } + } + + // 6. --save stores the key in profile e2e and prints no secret anywhere. + r = run(env, append(createArgs, "--save")...) + if r.code != 0 { + t.Fatalf("create --save: exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("create --save", r) + if !strings.Contains(r.stdout+r.stderr, "Saved as") { + t.Errorf("create --save: expected a 'Saved as' confirmation:\nstdout=%s\nstderr=%s", r.stdout, r.stderr) + } + { + data, err := os.ReadFile(filepath.Join(home, ".config", "lsh", "config.json")) + if err != nil { + t.Fatal(err) + } + var cfg struct { + Profiles map[string]struct { + ObjectStorage struct { + Keys map[string]struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + } `json:"keys"` + } `json:"object_storage"` + } `json:"profiles"` + } + if err := json.Unmarshal(data, &cfg); err != nil { + t.Fatalf("config.json: %v\n%s", err, data) + } + found := false + for name, k := range cfg.Profiles["e2e"].ObjectStorage.Keys { + if k.AccessKeyID == e2eAccessKeyID { + found = true + if k.SecretAccessKey != e2eSecret { + t.Errorf("saved key %q has secret %q, want the API secret", name, k.SecretAccessKey) + } + } + } + if !found { + t.Errorf("config.json has no key with access_key_id %s under profile e2e:\n%s", e2eAccessKeyID, data) + } + } + + // 6b. --save with an unwritable profile: the key cannot be stored, so it is + // deleted again. Nothing is left live on the API and the one-time secret is + // never printed on either stream. + if os.Geteuid() != 0 { + roHome := t.TempDir() + cfgDir := filepath.Join(roHome, ".config", "lsh") + writeE2EConfig(t, roHome, false) + if err := os.Chmod(cfgDir, 0o500); err != nil { + t.Fatal(err) + } + // Restore write access so the temp dir can be cleaned up. + t.Cleanup(func() { _ = os.Chmod(cfgDir, 0o700) }) + + r = run(e2eEnv(roHome), append(createArgs, "--save", "-o", "json")...) + // --save means create *and* persist: a caller must not read exit 0 as + // "the credential is in the profile". + if r.code != 1 { + t.Fatalf("create --save (unwritable profile): exit %d (want 1)\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + if !strings.Contains(r.stderr, "not saved in the profile") { + t.Errorf("create --save (unwritable profile): stderr must state the outcome:\n%s", r.stderr) + } + // The key does not exist any more, so there is no document to emit: + // stdout stays empty rather than describing a credential that is gone. + if strings.TrimSpace(r.stdout) != "" { + t.Errorf("create --save (unwritable profile): stdout must stay empty, got:\n%s", r.stdout) + } + if strings.Contains(r.stderr, e2eSecret) { + t.Errorf("create --save (unwritable profile): the key was removed, so no secret may be printed:\n%s", r.stderr) + } + for _, want := range []string{"could not be saved", "It was deleted again", "Retry once the profile is writable"} { + if !strings.Contains(r.stderr, want) { + t.Errorf("create --save (unwritable profile): stderr lacks %q:\n%s", want, r.stderr) + } + } + deletes := 0 + for _, req := range api.Requests() { + if strings.HasPrefix(req, "DELETE ") && strings.Contains(req, "/storage/access_keys") { + deletes++ + } + } + if deletes != 1 { + t.Errorf("create --save (unwritable profile): expected one access-key delete, got %d (%v)", deletes, api.Requests()) + } + if strings.Contains(r.stderr, "it was saved in the profile") { + t.Errorf("create --save (unwritable profile): stderr must not claim the key was saved:\n%s", r.stderr) + } + } + + // 7. The saved key is selected automatically for the bucket; --debug + // tracing never shows the secret and the listing reaches the S3 server. + r = run(env, "s3", "ls", "s3://e2e/", "--debug") + if r.code != 0 { + t.Fatalf("ls --debug with saved key: exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("ls --debug", r) + if !strings.Contains(r.stdout, " readme.txt") { + t.Errorf("ls --debug: listing missing the seeded object:\n%s", r.stdout) + } + if !strings.Contains(r.stderr, "saved key") { + t.Errorf("ls --debug: expected the credential source in the debug trace:\n%s", r.stderr) + } + + // The list command exposes the id and scope but never a secret. + r = run(env, "s3", "access-keys", "list", "--project", "e2e-project", "-o", "json") + if r.code != 0 { + t.Fatalf("access-keys list: exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("access-keys list", r) + if !strings.Contains(r.stdout, e2eAccessKeyID) { + t.Errorf("access-keys list: access key id missing:\n%s", r.stdout) + } +} + +// TestBinaryFlagValidationExitCodes covers the exit-code contract for +// command-line errors detected by the root pre-run: an invalid --output or +// --query is a usage error (2), not a generic failure (1). +func TestBinaryFlagValidationExitCodes(t *testing.T) { + bin := buildLshBinary(t) + home := t.TempDir() + writeE2EConfig(t, home, false) + env := e2eEnv(home) + work := t.TempDir() + + cases := []struct { + name string + args []string + }{ + {"invalid output format", []string{"s3", "ls", "-o", "bogus"}}, + {"invalid query", []string{"s3", "ls", "--query", "[[["}}, + {"invalid page size", []string{"s3", "ls", "--page-size", "-1"}}, + } + for _, c := range cases { + r := runLsh(t, bin, work, env, c.args...) + if r.code != 2 { + t.Errorf("%s: exit %d, want 2\nstderr=%s", c.name, r.code, r.stderr) + } + if strings.TrimSpace(r.stderr) == "" { + t.Errorf("%s: expected an actionable message on stderr", c.name) + } + } +} + +// TestBinaryLegacyExitCodesUnchanged pins the boundary of the documented S3 +// exit codes: they apply to the s3 subtree (including its legacy aliases) and +// must not change what the older command groups return. +func TestBinaryLegacyExitCodesUnchanged(t *testing.T) { + bin := buildLshBinary(t) + home := t.TempDir() + writeE2EConfig(t, home, false) + env := e2eEnv(home) + work := t.TempDir() + + cases := []struct { + name string + args []string + want int + }{ + {"s3 opted in", []string{"s3", "ls", "-o", "bogus"}, 2}, + {"s3 through the legacy alias", []string{"storage-objects", "list", "-o", "bogus"}, 2}, + {"older group keeps exiting 1", []string{"servers", "list", "-o", "bogus"}, 1}, + {"older group, bad page size", []string{"servers", "list", "--page-size", "-1"}, 1}, + } + for _, c := range cases { + r := runLsh(t, bin, work, env, c.args...) + if r.code != c.want { + t.Errorf("%s: exit %d, want %d\nstderr=%s", c.name, r.code, c.want, r.stderr) + } + } +} diff --git a/cmd/s3/lifecycle.go b/cmd/s3/lifecycle.go new file mode 100644 index 0000000..fa27bc0 --- /dev/null +++ b/cmd/s3/lifecycle.go @@ -0,0 +1,385 @@ +package s3 + +import ( + "context" + "fmt" + "sort" + "strings" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +// Flag names shared by lifecycle create/update. +const ( + flagRuleName = "name" + flagRulePrefix = "prefix" + flagExpirationDays = "expiration-days" + flagExpireDays = "expire-days" // hidden alias of --expiration-days + flagNoncurrentDays = "noncurrent-days" + flagAbortMpuDays = "abort-incomplete-multipart-days" +) + +// lifecycleIDPrefix is the prefix of every lifecycle rule ID returned by the API. +const lifecycleIDPrefix = "lifecycle_" + +// NewLifecycleCmd builds `lsh s3 lifecycle`, the bucket lifecycle rule group +// (list, get, create, update, delete). Rules are managed through the Latitude +// API; the bucket may be given as s3:// or . +func NewLifecycleCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "lifecycle", + Aliases: []string{"lifecycle-rules"}, + Short: "Manage bucket lifecycle rules", + Long: `Manage the lifecycle rules of a bucket (automatic expiration of objects, +noncurrent versions and incomplete multipart uploads). + +Rules are stored by the Latitude API, so no S3 credential is needed. Every +rule needs --expiration-days; rules can be addressed by id (lifecycle_...) or +by name in get/update/delete.`, + Example: ` lsh s3 lifecycle create s3://logs --prefix tmp/ --expiration-days 7 + lsh s3 lifecycle create s3://logs --name abort-mpu --expiration-days 3650 --abort-incomplete-multipart-days 2 + lsh s3 lifecycle list s3://logs + lsh s3 lifecycle update s3://logs expire-7d-tmp --expiration-days 14 + lsh s3 lifecycle delete s3://logs expire-7d-tmp`, + } + cmd.AddCommand( + newLifecycleListCmd(), + newLifecycleGetCmd(), + newLifecycleCreateCmd(), + newLifecycleUpdateCmd(), + newLifecycleDeleteCmd(), + ) + return cmd +} + +// LifecycleRule wraps the SDK payload so the shared renderer can print it +// (-o table|json|yaml|csv, --query). JSON output is the API document as-is. +type LifecycleRule struct { + components.LifecycleRuleData +} + +// RuleID returns the rule id or "". +func (r LifecycleRule) RuleID() string { return ruleID(r.LifecycleRuleData) } + +// RuleName returns the rule name or "". +func (r LifecycleRule) RuleName() string { return ruleName(r.LifecycleRuleData) } + +// TableRow renders the rule as a table row: id, name, prefix, expiration, +// noncurrent, abort-mpu and enabled. +func (r LifecycleRule) TableRow() table.Row { + a := r.Attributes + return table.Row{ + "id": {Label: "ID", Value: r.RuleID()}, + "name": {Label: "Name", Value: r.RuleName()}, + "prefix": {Label: "Prefix", Value: strOrDash(a.GetPrefix())}, + "expiration_days": {Label: "Expiration", Value: daysLabel(a.GetExpirationDays())}, + "noncurrent_days": {Label: "Noncurrent", Value: daysLabel(a.GetNoncurrentDays())}, + "abort_mpu_days": {Label: "Abort MPU", Value: daysLabel(a.GetAbortMpuDaysAfterInitiation())}, + "enabled": {Label: "Enabled", Value: boolLabel(a.GetEnabled())}, + } +} + +// lifecycleRuleRows adapts SDK rules for the renderer. +func lifecycleRuleRows(rules []components.LifecycleRuleData) []renderer.ResponseData { + out := make([]renderer.ResponseData, 0, len(rules)) + for _, r := range rules { + out = append(out, LifecycleRule{LifecycleRuleData: r}) + } + return out +} + +// LifecycleMutation is the structured result of create/update/delete +// (create/update render the resulting rule; delete and dry-runs render this). +type LifecycleMutation struct { + // Action is create, update or delete. + Action string `json:"action"` + Bucket string `json:"bucket"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + Error string `json:"error,omitempty"` +} + +func (m LifecycleMutation) TableRow() table.Row { + status := "ok" + switch { + case m.Error != "": + status = "error: " + m.Error + case m.DryRun: + status = "dryrun" + } + return table.Row{ + "action": {Label: "Action", Value: m.Action}, + "bucket": {Label: "Bucket", Value: m.Bucket}, + "id": {Label: "ID", Value: m.ID}, + "name": {Label: "Name", Value: m.Name}, + "status": {Label: "Status", Value: status}, + } +} + +// HumanLine renders the aws-style single line: `lifecycle: created ()`, +// or `(dryrun) lifecycle: create on s3://bucket` for a dry-run. +func (m LifecycleMutation) HumanLine() string { + subject := m.Name + if m.ID != "" && m.Name != "" { + subject = fmt.Sprintf("%s (%s)", m.ID, m.Name) + } else if m.ID != "" { + subject = m.ID + } + if m.DryRun { + return fmt.Sprintf("(dryrun) lifecycle: %s %s on %s", m.Action, subject, m.Bucket) + } + if m.Error != "" { + return fmt.Sprintf("lifecycle: failed to %s %s: %s", m.Action, subject, m.Error) + } + return fmt.Sprintf("lifecycle: %s %s", pastTense(m.Action), subject) +} + +func pastTense(action string) string { + switch action { + case "create": + return "created" + case "update": + return "updated" + case "delete": + return "deleted" + } + return action +} + +// ruleID / ruleName read the optional SDK fields. +func ruleID(r components.LifecycleRuleData) string { + if r.ID == nil { + return "" + } + return *r.ID +} + +func ruleName(r components.LifecycleRuleData) string { + if r.Attributes == nil || r.Attributes.Name == nil { + return "" + } + return *r.Attributes.Name +} + +// daysLabel renders "7d", or the shared table placeholder when unset. +func daysLabel(v *int64) string { + if v == nil { + return emptyCell + } + return fmt.Sprintf("%dd", *v) +} + +// strOrDash renders the string, or the shared table placeholder when unset. +func strOrDash(v *string) string { + if v == nil { + return emptyCell + } + return orEmptyCell(*v) +} + +// boolLabel renders true/false, or the shared table placeholder when unset. +func boolLabel(v *bool) string { + if v == nil { + return emptyCell + } + if *v { + return "true" + } + return "false" +} + +// findLifecycleRule picks the rule addressed by token: an exact id match wins, +// otherwise the rule whose name equals token. It fails with exit 3 when +// nothing matches and exit 2 when several rules share the name. +func findLifecycleRule(rules []components.LifecycleRuleData, token string) (*components.LifecycleRuleData, error) { + token = strings.TrimSpace(token) + if token == "" { + return nil, exitcode.Errorf(exitcode.Usage, "missing lifecycle rule: pass the rule id (lifecycle_...) or its name") + } + for i := range rules { + if ruleID(rules[i]) == token { + return &rules[i], nil + } + } + var matches []int + for i := range rules { + if ruleName(rules[i]) == token { + matches = append(matches, i) + } + } + switch len(matches) { + case 0: + return nil, exitcode.Errorf(exitcode.NotFound, "lifecycle rule %q not found on this bucket; run 'lsh s3 lifecycle list' to see the rules", token) + case 1: + return &rules[matches[0]], nil + } + ids := make([]string, 0, len(matches)) + for _, i := range matches { + ids = append(ids, " "+ruleID(rules[i])) + } + sort.Strings(ids) + return nil, exitcode.Errorf(exitcode.Usage, "lifecycle rule name %q matches several rules:\n%s\nuse the rule id instead", token, strings.Join(ids, "\n")) +} + +// defaultLifecycleRuleName builds the default name of a new rule: +// `expire-d`, plus `-` (sanitized to [a-z0-9-]) when a prefix is set. +func defaultLifecycleRuleName(expirationDays int64, prefix string) string { + name := fmt.Sprintf("expire-%dd", expirationDays) + if s := sanitizeRuleNamePart(prefix); s != "" { + name += "-" + s + } + return name +} + +// sanitizeRuleNamePart lowercases s and collapses every run of characters +// outside [a-z0-9] into a single "-", trimming dashes at both ends. +func sanitizeRuleNamePart(s string) string { + var b strings.Builder + dash := false + for _, r := range strings.ToLower(s) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + dash = false + default: + if b.Len() > 0 && !dash { + b.WriteByte('-') + } + dash = true + } + } + return strings.Trim(b.String(), "-") +} + +// addLifecycleRuleFlags registers the attribute flags shared by create/update. +func addLifecycleRuleFlags(cmd *cobra.Command) { + f := cmd.Flags() + f.String(flagRuleName, "", "rule name (default: expire-d[-])") + f.String(flagRulePrefix, "", "only apply to objects whose key starts with this prefix (default: whole bucket)") + f.Int64(flagExpirationDays, 0, "delete objects this many days after creation (required by the API)") + f.Int64(flagExpireDays, 0, "alias of --expiration-days") + f.Int64(flagNoncurrentDays, 0, "delete noncurrent versions this many days after they become noncurrent (versioned buckets)") + f.Int64(flagAbortMpuDays, 0, "abort incomplete multipart uploads this many days after initiation") + _ = f.MarkHidden(flagExpireDays) +} + +// expirationDaysFlag returns --expiration-days (or its alias --expire-days) +// and whether either was given. +func expirationDaysFlag(cmd *cobra.Command) (int64, bool) { + if cmd.Flags().Changed(flagExpirationDays) { + v, _ := cmd.Flags().GetInt64(flagExpirationDays) + return v, true + } + if cmd.Flags().Changed(flagExpireDays) { + v, _ := cmd.Flags().GetInt64(flagExpireDays) + return v, true + } + return 0, false +} + +// lifecycleAPI bundles the API client and the resolved bucket for one command. +type lifecycleAPI struct { + api *sdk.Latitudesh + bucket *objectstorage.Bucket + opts []operations.Option +} + +// openLifecycleAPI parses the bucket argument, resolves it through the API +// and returns the client used by every lifecycle subcommand. +func openLifecycleAPI(ctx context.Context, cmd *cobra.Command, arg string) (*lifecycleAPI, error) { + ref, err := objectstorage.ParseBucketOnly(arg) + if err != nil { + return nil, err + } + if endpointOverride(cmd) != "" { + return nil, exitcode.Errorf(exitcode.Usage, "lifecycle rules are managed through the Latitude API; unset --endpoint-url / %s to use this command", objectstorage.EnvEndpointURL) + } + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return nil, err + } + if b.EndpointOverride || b.ID == "" { + return nil, exitcode.Errorf(exitcode.Usage, "bucket %s has no API id; lifecycle rules need the Latitude API", b.Display()) + } + return &lifecycleAPI{ + api: apiClient(), + bucket: b, + opts: []operations.Option{operations.WithRetries(lsh.RetryConfig())}, + }, nil +} + +// uri returns the s3:// form of the bucket for messages. +func (l *lifecycleAPI) uri() string { return "s3://" + l.bucket.Name } + +// list returns every rule of the bucket (empty slice when there is none). +func (l *lifecycleAPI) list(ctx context.Context) ([]components.LifecycleRuleData, error) { + resp, err := l.api.ObjectStorage.GetStorageBucketLifecycleRules(ctx, l.bucket.ID, l.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("bucket %s", l.bucket.Display())) + } + if resp.LifecycleRules == nil { + return []components.LifecycleRuleData{}, nil + } + return resp.LifecycleRules.Data, nil +} + +// get resolves token (lifecycle_ id or name) to a rule. Ids are fetched +// directly; names are looked up in the rule list. +func (l *lifecycleAPI) get(ctx context.Context, token string) (*components.LifecycleRuleData, error) { + token = strings.TrimSpace(token) + if strings.HasPrefix(token, lifecycleIDPrefix) { + resp, err := l.api.ObjectStorage.GetStorageBucketLifecycleRule(ctx, l.bucket.ID, token, l.opts...) + switch { + case err != nil && exitcode.Of(objectstorage.HumanizeAPI(err, "")) != exitcode.NotFound: + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("lifecycle rule %q on bucket %s", token, l.bucket.Display())) + case err == nil && resp.Object != nil && resp.Object.Data != nil: + return resp.Object.Data, nil + } + // Rule names are free text, so a name may start with the ID prefix: + // fall through to the name lookup instead of reporting "not found". + } + rules, err := l.list(ctx) + if err != nil { + return nil, err + } + return findLifecycleRule(rules, token) +} + +func (l *lifecycleAPI) create(ctx context.Context, body operations.PostStorageBucketLifecycleRulesRequestBody) (*components.LifecycleRuleData, error) { + resp, err := l.api.ObjectStorage.PostStorageBucketLifecycleRules(ctx, l.bucket.ID, body, l.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("bucket %s", l.bucket.Display())) + } + if resp.Object == nil || resp.Object.Data == nil { + return nil, exitcode.Errorf(exitcode.Generic, "the API returned no lifecycle rule; run 'lsh s3 lifecycle list %s' to check", l.uri()) + } + return resp.Object.Data, nil +} + +func (l *lifecycleAPI) update(ctx context.Context, id string, body operations.PatchStorageBucketLifecycleRuleRequestBody) (*components.LifecycleRuleData, error) { + resp, err := l.api.ObjectStorage.PatchStorageBucketLifecycleRule(ctx, l.bucket.ID, id, body, l.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("lifecycle rule %q on bucket %s", id, l.bucket.Display())) + } + if resp.Object == nil || resp.Object.Data == nil { + return nil, exitcode.Errorf(exitcode.Generic, "the API returned no lifecycle rule; run 'lsh s3 lifecycle get %s %s' to check", l.uri(), id) + } + return resp.Object.Data, nil +} + +func (l *lifecycleAPI) delete(ctx context.Context, id string) error { + _, err := l.api.ObjectStorage.DeleteStorageBucketLifecycleRule(ctx, l.bucket.ID, id, l.opts...) + if err != nil { + return objectstorage.HumanizeAPI(err, fmt.Sprintf("lifecycle rule %q on bucket %s", id, l.bucket.Display())) + } + return nil +} diff --git a/cmd/s3/lifecycle_create.go b/cmd/s3/lifecycle_create.go new file mode 100644 index 0000000..1bd251f --- /dev/null +++ b/cmd/s3/lifecycle_create.go @@ -0,0 +1,156 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +const flagRuleDisabled = "disabled" + +// lifecycleCreateInput is the flag set of `lifecycle create`, kept separate +// from cobra so the request builder is testable. +type lifecycleCreateInput struct { + Name string + Prefix string + ExpirationDays int64 + NoncurrentDays int64 + AbortMpuDays int64 + Disabled bool +} + +// buildLifecycleCreateRequest validates the input and builds the POST body. +// --expiration-days is mandatory on the API side, so it is checked here with +// a clear message instead of a 422. +func buildLifecycleCreateRequest(in lifecycleCreateInput) (operations.PostStorageBucketLifecycleRulesRequestBody, error) { + var body operations.PostStorageBucketLifecycleRulesRequestBody + if in.ExpirationDays <= 0 { + return body, exitcode.Errorf(exitcode.Usage, "the API requires --expiration-days on every rule (a positive number of days after which objects are deleted)") + } + if in.NoncurrentDays < 0 { + return body, exitcode.Errorf(exitcode.Usage, "--%s must be a positive number of days", flagNoncurrentDays) + } + if in.AbortMpuDays < 0 { + return body, exitcode.Errorf(exitcode.Usage, "--%s must be a positive number of days", flagAbortMpuDays) + } + name := strings.TrimSpace(in.Name) + if name == "" { + name = defaultLifecycleRuleName(in.ExpirationDays, in.Prefix) + } + enabled := !in.Disabled + attrs := operations.PostStorageBucketLifecycleRulesAttributes{ + Name: name, + Enabled: &enabled, + ExpirationDays: in.ExpirationDays, + } + if p := in.Prefix; p != "" { + attrs.Prefix = &p + } + if in.NoncurrentDays > 0 { + v := in.NoncurrentDays + attrs.NoncurrentDays = &v + } + if in.AbortMpuDays > 0 { + v := in.AbortMpuDays + attrs.AbortMpuDaysAfterInitiation = &v + } + body.Data = operations.PostStorageBucketLifecycleRulesData{ + Type: operations.PostStorageBucketLifecycleRulesTypeLifecycleRules, + Attributes: attrs, + } + return body, nil +} + +// plannedLifecycleRule turns a create request into a renderable rule for +// --dry-run (no id, since nothing was created). +func plannedLifecycleRule(body operations.PostStorageBucketLifecycleRulesRequestBody) LifecycleRule { + a := body.Data.Attributes + name := a.Name + exp := a.ExpirationDays + typ := components.LifecycleRuleDataTypeLifecycleRules + return LifecycleRule{LifecycleRuleData: components.LifecycleRuleData{ + Type: &typ, + Attributes: &components.LifecycleRuleDataAttributes{ + Name: &name, + Enabled: a.Enabled, + Prefix: a.Prefix, + ExpirationDays: &exp, + NoncurrentDays: a.NoncurrentDays, + AbortMpuDaysAfterInitiation: a.AbortMpuDaysAfterInitiation, + }, + }} +} + +// newLifecycleCreateCmd builds `lsh s3 lifecycle create s3://bucket`. +func newLifecycleCreateCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "create ", + Aliases: []string{"add"}, + Short: "Create a lifecycle rule", + Long: `Create a lifecycle rule on a bucket. + +--expiration-days is required by the API on every rule, even when the goal is +only to abort incomplete multipart uploads or expire noncurrent versions. The +default name is expire-d, with - appended when --prefix is given.`, + Example: ` lsh s3 lifecycle create s3://logs --prefix tmp/ --expiration-days 7 + lsh s3 lifecycle create s3://logs --name abort-mpu --expiration-days 3650 --abort-incomplete-multipart-days 2 + lsh s3 lifecycle create s3://backups --expiration-days 90 --noncurrent-days 30 --disabled`, + Args: cobra.ExactArgs(1), + RunE: runLifecycleCreate, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + addLifecycleRuleFlags(cmd) + cmd.Flags().Bool(flagRuleDisabled, false, "create the rule disabled") + return cmd +} + +func runLifecycleCreate(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + in := lifecycleCreateInput{} + in.Name, _ = cmd.Flags().GetString(flagRuleName) + in.Prefix, _ = cmd.Flags().GetString(flagRulePrefix) + in.ExpirationDays, _ = expirationDaysFlag(cmd) + in.NoncurrentDays, _ = cmd.Flags().GetInt64(flagNoncurrentDays) + in.AbortMpuDays, _ = cmd.Flags().GetInt64(flagAbortMpuDays) + in.Disabled, _ = cmd.Flags().GetBool(flagRuleDisabled) + + // Validate the flags before touching the API so usage errors are cheap. + body, err := buildLifecycleCreateRequest(in) + if err != nil { + return printErr(err) + } + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + + if dryRun() { + if isHuman() { + fmt.Println(LifecycleMutation{Action: "create", Bucket: lc.uri(), Name: body.Data.Attributes.Name, DryRun: true}.HumanLine()) + return nil + } + render([]renderer.ResponseData{plannedLifecycleRule(body)}) + return nil + } + + created, err := lc.create(ctx, body) + if err != nil { + return printErr(err) + } + if isHuman() { + fmt.Println(LifecycleMutation{Action: "create", Bucket: lc.uri(), ID: ruleID(*created), Name: ruleName(*created)}.HumanLine()) + return nil + } + render([]renderer.ResponseData{LifecycleRule{LifecycleRuleData: *created}}) + return nil +} diff --git a/cmd/s3/lifecycle_create_test.go b/cmd/s3/lifecycle_create_test.go new file mode 100644 index 0000000..68bb259 --- /dev/null +++ b/cmd/s3/lifecycle_create_test.go @@ -0,0 +1,109 @@ +package s3 + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func TestBuildLifecycleCreateRequest(t *testing.T) { + body, err := buildLifecycleCreateRequest(lifecycleCreateInput{Prefix: "tmp/", ExpirationDays: 7}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := body.Data.Attributes + if body.Data.Type != operations.PostStorageBucketLifecycleRulesTypeLifecycleRules { + t.Errorf("type = %q", body.Data.Type) + } + if a.Name != "expire-7d-tmp" { + t.Errorf("default name = %q, want expire-7d-tmp", a.Name) + } + if a.ExpirationDays != 7 { + t.Errorf("expiration_days = %d", a.ExpirationDays) + } + if a.Prefix == nil || *a.Prefix != "tmp/" { + t.Errorf("prefix = %v", a.Prefix) + } + if a.Enabled == nil || !*a.Enabled { + t.Errorf("enabled should default to true, got %v", a.Enabled) + } + if a.NoncurrentDays != nil || a.AbortMpuDaysAfterInitiation != nil { + t.Errorf("optional days must be omitted when zero: %v %v", a.NoncurrentDays, a.AbortMpuDaysAfterInitiation) + } + + // Explicit name, all optional attributes and --disabled. + body, err = buildLifecycleCreateRequest(lifecycleCreateInput{Name: " abort-mpu ", ExpirationDays: 3650, NoncurrentDays: 30, AbortMpuDays: 2, Disabled: true}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a = body.Data.Attributes + if a.Name != "abort-mpu" { + t.Errorf("name = %q", a.Name) + } + if a.Prefix != nil { + t.Errorf("prefix should be omitted, got %q", *a.Prefix) + } + if a.NoncurrentDays == nil || *a.NoncurrentDays != 30 { + t.Errorf("noncurrent_days = %v", a.NoncurrentDays) + } + if a.AbortMpuDaysAfterInitiation == nil || *a.AbortMpuDaysAfterInitiation != 2 { + t.Errorf("abort_mpu_days_after_initiation = %v", a.AbortMpuDaysAfterInitiation) + } + if a.Enabled == nil || *a.Enabled { + t.Errorf("enabled should be false with --disabled, got %v", a.Enabled) + } + + // The wire format uses the API attribute names. + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"type":"lifecycle_rules"`, `"expiration_days":3650`, `"noncurrent_days":30`, `"abort_mpu_days_after_initiation":2`, `"enabled":false`} { + if !strings.Contains(string(raw), want) { + t.Errorf("payload %s lacks %s", raw, want) + } + } +} + +func TestBuildLifecycleCreateRequestValidation(t *testing.T) { + cases := []struct { + name string + in lifecycleCreateInput + want string + }{ + {"missing expiration", lifecycleCreateInput{AbortMpuDays: 2}, "the API requires --expiration-days on every rule"}, + {"negative expiration", lifecycleCreateInput{ExpirationDays: -1}, "the API requires --expiration-days on every rule"}, + {"negative noncurrent", lifecycleCreateInput{ExpirationDays: 1, NoncurrentDays: -1}, "--noncurrent-days"}, + {"negative abort", lifecycleCreateInput{ExpirationDays: 1, AbortMpuDays: -5}, "--abort-incomplete-multipart-days"}, + } + for _, c := range cases { + _, err := buildLifecycleCreateRequest(c.in) + if err == nil { + t.Errorf("%s: expected error", c.name) + continue + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("%s: exit %d, want %d", c.name, exitcode.Of(err), exitcode.Usage) + } + if !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: error %q should mention %q", c.name, err.Error(), c.want) + } + } +} + +func TestPlannedLifecycleRule(t *testing.T) { + body, err := buildLifecycleCreateRequest(lifecycleCreateInput{Prefix: "tmp/", ExpirationDays: 7, AbortMpuDays: 2}) + if err != nil { + t.Fatal(err) + } + row := plannedLifecycleRule(body).TableRow() + if row["id"].Value != "" { + t.Errorf("dry-run rule must have no id, got %q", row["id"].Value) + } + if row["name"].Value != "expire-7d-tmp" || row["expiration_days"].Value != "7d" || row["abort_mpu_days"].Value != "2d" || row["noncurrent_days"].Value != emptyCell { // unset days render the shared "-" placeholder + t.Errorf("unexpected planned row: %+v", row) + } +} diff --git a/cmd/s3/lifecycle_delete.go b/cmd/s3/lifecycle_delete.go new file mode 100644 index 0000000..2c6d922 --- /dev/null +++ b/cmd/s3/lifecycle_delete.go @@ -0,0 +1,133 @@ +package s3 + +import ( + "context" + "fmt" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +const flagRuleAll = "all" + +// newLifecycleDeleteCmd builds `lsh s3 lifecycle delete s3://bucket `. +func newLifecycleDeleteCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "delete []", + Aliases: []string{"rm", "remove"}, + Short: "Delete a lifecycle rule (or all of them with --all)", + Long: `Delete one lifecycle rule, addressed by id (lifecycle_...) or by name, or +every rule of the bucket with --all. + +Deletion asks for confirmation; pass --yes in scripts (a non-interactive +session without --yes fails with exit 7 instead of hanging).`, + Example: ` lsh s3 lifecycle delete s3://logs expire-7d-tmp + lsh s3 lifecycle rm s3://logs lifecycle_9x4kQ --yes + lsh s3 lifecycle delete s3://logs --all --yes`, + Args: cobra.RangeArgs(1, 2), + RunE: runLifecycleDelete, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + addYesFlag(cmd) + cmd.Flags().Bool(flagRuleAll, false, "delete every lifecycle rule of the bucket") + return cmd +} + +func runLifecycleDelete(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + all, _ := cmd.Flags().GetBool(flagRuleAll) + yes, _ := cmd.Flags().GetBool(flagYes) + switch { + case all && len(args) == 2: + return printErr(exitcode.Errorf(exitcode.Usage, "pass either a rule () or --all, not both")) + case !all && len(args) == 1: + return printErr(exitcode.Errorf(exitcode.Usage, "missing lifecycle rule: pass , or --all to delete every rule of the bucket")) + } + + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + + var targets []components.LifecycleRuleData + if all { + targets, err = lc.list(ctx) + if err != nil { + return printErr(err) + } + if len(targets) == 0 { + objectstorage.Hintf("no lifecycle rules on %s; nothing to delete", lc.uri()) + if !isHuman() { + render(nil) + } + return nil + } + } else { + rule, err := lc.get(ctx, args[1]) + if err != nil { + return printErr(err) + } + targets = []components.LifecycleRuleData{*rule} + } + + if dryRun() { + rows := make([]renderer.ResponseData, 0, len(targets)) + for _, r := range targets { + m := LifecycleMutation{Action: "delete", Bucket: lc.uri(), ID: ruleID(r), Name: ruleName(r), DryRun: true} + if isHuman() { + fmt.Println(m.HumanLine()) + } + rows = append(rows, m) + } + if !isHuman() { + render(rows) + } + return nil + } + + question := fmt.Sprintf("Delete lifecycle rule %s (%s) from %s?", ruleName(targets[0]), ruleID(targets[0]), lc.uri()) + if all { + question = fmt.Sprintf("Delete all %d lifecycle rules from %s?", len(targets), lc.uri()) + } + if err := objectstorage.ConfirmOrRefuse(cmd, yes, question); err != nil { + return printErr(err) + } + + rows := make([]renderer.ResponseData, 0, len(targets)) + var failed int + var firstErr error + for _, r := range targets { + m := LifecycleMutation{Action: "delete", Bucket: lc.uri(), ID: ruleID(r), Name: ruleName(r)} + if err := lc.delete(ctx, m.ID); err != nil { + failed++ + if firstErr == nil { + firstErr = err + } + m.Error = err.Error() + if isHuman() { + objectstorage.Warnf("%s", m.HumanLine()) + } + } else if isHuman() { + fmt.Println(m.HumanLine()) + } + rows = append(rows, m) + } + if !isHuman() { + render(rows) + } + switch { + case failed == 0: + return nil + case failed == len(targets) && len(targets) == 1: + return printErr(firstErr) + case failed == len(targets): + return printErr(exitcode.Errorf(exitcode.Of(firstErr), "none of the %d lifecycle rules could be deleted: %v", failed, firstErr)) + } + return printErr(exitcode.Errorf(exitcode.Partial, "%d of %d lifecycle rules could not be deleted", failed, len(targets))) +} diff --git a/cmd/s3/lifecycle_get.go b/cmd/s3/lifecycle_get.go new file mode 100644 index 0000000..fbe271e --- /dev/null +++ b/cmd/s3/lifecycle_get.go @@ -0,0 +1,43 @@ +package s3 + +import ( + "context" + + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +// newLifecycleGetCmd builds `lsh s3 lifecycle get s3://bucket `. +func newLifecycleGetCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "get ", + Aliases: []string{"show", "describe"}, + Short: "Show one lifecycle rule", + Long: `Show one lifecycle rule of a bucket, addressed by id (lifecycle_...) or by +name. A name shared by several rules is rejected (exit 2): use the id.`, + Example: ` lsh s3 lifecycle get s3://logs expire-7d-tmp + lsh s3 lifecycle get s3://logs lifecycle_9x4kQ -o json`, + Args: cobra.ExactArgs(2), + RunE: runLifecycleGet, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + return cmd +} + +func runLifecycleGet(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + rule, err := lc.get(ctx, args[1]) + if err != nil { + return printErr(err) + } + render([]renderer.ResponseData{LifecycleRule{LifecycleRuleData: *rule}}) + return nil +} diff --git a/cmd/s3/lifecycle_list.go b/cmd/s3/lifecycle_list.go new file mode 100644 index 0000000..67b7334 --- /dev/null +++ b/cmd/s3/lifecycle_list.go @@ -0,0 +1,49 @@ +package s3 + +import ( + "context" + + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/spf13/cobra" +) + +// newLifecycleListCmd builds `lsh s3 lifecycle list s3://bucket`. +func newLifecycleListCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "list ", + Aliases: []string{"ls"}, + Short: "List the lifecycle rules of a bucket", + Long: `List the lifecycle rules of a bucket. + +A bucket without rules prints an empty list and exits 0.`, + Example: ` lsh s3 lifecycle list s3://logs + lsh s3 lifecycle ls logs -o json`, + Args: cobra.ExactArgs(1), + RunE: runLifecycleList, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + return cmd +} + +func runLifecycleList(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + rules, err := lc.list(ctx) + if err != nil { + return printErr(err) + } + if len(rules) == 0 && isHuman() { + // Empty is a normal outcome (exit 0): say so on stderr and keep stdout + // clean instead of printing the renderer's "No results found" box. + objectstorage.Hintf("no lifecycle rules on %s", lc.uri()) + return nil + } + render(lifecycleRuleRows(rules)) + return nil +} diff --git a/cmd/s3/lifecycle_test.go b/cmd/s3/lifecycle_test.go new file mode 100644 index 0000000..658f94f --- /dev/null +++ b/cmd/s3/lifecycle_test.go @@ -0,0 +1,159 @@ +package s3 + +import ( + "strings" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func ptrS(s string) *string { return &s } +func ptrI(i int64) *int64 { return &i } +func ptrB(b bool) *bool { return &b } + +func rule(id, name string, exp *int64) components.LifecycleRuleData { + return components.LifecycleRuleData{ + ID: ptrS(id), + Attributes: &components.LifecycleRuleDataAttributes{ + Name: ptrS(name), + ExpirationDays: exp, + Enabled: ptrB(true), + }, + } +} + +func fixtureRules() []components.LifecycleRuleData { + return []components.LifecycleRuleData{ + rule("lifecycle_a", "expire-7d-tmp", ptrI(7)), + rule("lifecycle_b", "abort-mpu", ptrI(3650)), + rule("lifecycle_c", "dup", ptrI(1)), + rule("lifecycle_d", "dup", ptrI(2)), + rule("lifecycle_e", "lifecycle_a", ptrI(9)), // a name that looks like an id + } +} + +func TestFindLifecycleRule(t *testing.T) { + rules := fixtureRules() + + cases := []struct { + token string + wantID string + wantCode int + }{ + {"lifecycle_a", "lifecycle_a", 0}, // id wins over the rule named lifecycle_a + {"lifecycle_b", "lifecycle_b", 0}, + {"expire-7d-tmp", "lifecycle_a", 0}, + {" abort-mpu ", "lifecycle_b", 0}, + {"dup", "", exitcode.Usage}, + {"missing", "", exitcode.NotFound}, + {"", "", exitcode.Usage}, + } + for _, c := range cases { + got, err := findLifecycleRule(rules, c.token) + if c.wantCode != 0 { + if err == nil { + t.Errorf("findLifecycleRule(%q): expected error", c.token) + continue + } + if exitcode.Of(err) != c.wantCode { + t.Errorf("findLifecycleRule(%q): exit %d, want %d (%v)", c.token, exitcode.Of(err), c.wantCode, err) + } + if c.token == "dup" && (!strings.Contains(err.Error(), "lifecycle_c") || !strings.Contains(err.Error(), "lifecycle_d")) { + t.Errorf("ambiguous error should list the candidate ids, got %q", err.Error()) + } + continue + } + if err != nil { + t.Errorf("findLifecycleRule(%q): unexpected error %v", c.token, err) + continue + } + if ruleID(*got) != c.wantID { + t.Errorf("findLifecycleRule(%q) = %s, want %s", c.token, ruleID(*got), c.wantID) + } + } + + if _, err := findLifecycleRule(nil, "anything"); exitcode.Of(err) != exitcode.NotFound { + t.Errorf("empty rule list: exit %d, want %d", exitcode.Of(err), exitcode.NotFound) + } +} + +func TestDefaultLifecycleRuleName(t *testing.T) { + cases := []struct { + days int64 + prefix string + want string + }{ + {7, "", "expire-7d"}, + {7, "tmp/", "expire-7d-tmp"}, + {30, "Logs/2026/", "expire-30d-logs-2026"}, + {1, "a_b c", "expire-1d-a-b-c"}, + {365, "///", "expire-365d"}, + {14, "Ünïcode/ok", "expire-14d-n-code-ok"}, + } + for _, c := range cases { + if got := defaultLifecycleRuleName(c.days, c.prefix); got != c.want { + t.Errorf("defaultLifecycleRuleName(%d, %q) = %q, want %q", c.days, c.prefix, got, c.want) + } + } +} + +func TestLifecycleRuleTableRow(t *testing.T) { + r := LifecycleRule{LifecycleRuleData: components.LifecycleRuleData{ + ID: ptrS("lifecycle_9x4kQ"), + Attributes: &components.LifecycleRuleDataAttributes{ + Name: ptrS("expire-7d-tmp"), + Prefix: ptrS("tmp/"), + ExpirationDays: ptrI(7), + Enabled: ptrB(true), + }, + }} + row := r.TableRow() + want := map[string]string{ + "id": "lifecycle_9x4kQ", + "name": "expire-7d-tmp", + "prefix": "tmp/", + "expiration_days": "7d", + "noncurrent_days": "-", + "abort_mpu_days": "-", + "enabled": "true", + } + for k, v := range want { + if row[k].Value != v { + t.Errorf("row[%q] = %q, want %q", k, row[k].Value, v) + } + } + + // A rule with no attributes must not panic and renders the shared + // placeholder ("-", the same one every s3 table uses for empty cells). + empty := LifecycleRule{LifecycleRuleData: components.LifecycleRuleData{ID: ptrS("lifecycle_x")}} + for _, k := range []string{"enabled", "prefix", "expiration_days"} { + if got := empty.TableRow()[k].Value; got != emptyCell { + t.Errorf("empty rule %s = %q, want %q", k, got, emptyCell) + } + } + // An empty prefix string is also a placeholder, not a blank cell. + blank := LifecycleRule{LifecycleRuleData: components.LifecycleRuleData{Attributes: &components.LifecycleRuleDataAttributes{Prefix: ptrS("")}}} + if got := blank.TableRow()["prefix"].Value; got != emptyCell { + t.Errorf("blank prefix = %q, want %q", got, emptyCell) + } +} + +func TestLifecycleMutationHumanLine(t *testing.T) { + cases := []struct { + m LifecycleMutation + want string + }{ + {LifecycleMutation{Action: "create", Bucket: "s3://logs", ID: "lifecycle_1", Name: "expire-7d"}, "lifecycle: created lifecycle_1 (expire-7d)"}, + {LifecycleMutation{Action: "update", Bucket: "s3://logs", ID: "lifecycle_1", Name: "expire-7d"}, "lifecycle: updated lifecycle_1 (expire-7d)"}, + {LifecycleMutation{Action: "delete", Bucket: "s3://logs", ID: "lifecycle_1", Name: "expire-7d"}, "lifecycle: deleted lifecycle_1 (expire-7d)"}, + {LifecycleMutation{Action: "create", Bucket: "s3://logs", Name: "expire-7d", DryRun: true}, "(dryrun) lifecycle: create expire-7d on s3://logs"}, + {LifecycleMutation{Action: "delete", Bucket: "s3://logs", ID: "lifecycle_1", Name: "n", DryRun: true}, "(dryrun) lifecycle: delete lifecycle_1 (n) on s3://logs"}, + {LifecycleMutation{Action: "delete", Bucket: "s3://logs", ID: "lifecycle_1", Name: "n", Error: "boom"}, "lifecycle: failed to delete lifecycle_1 (n): boom"}, + } + for _, c := range cases { + if got := c.m.HumanLine(); got != c.want { + t.Errorf("HumanLine() = %q, want %q", got, c.want) + } + } +} diff --git a/cmd/s3/lifecycle_update.go b/cmd/s3/lifecycle_update.go new file mode 100644 index 0000000..832e246 --- /dev/null +++ b/cmd/s3/lifecycle_update.go @@ -0,0 +1,210 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +const ( + flagRuleEnable = "enable" + flagRuleDisable = "disable" +) + +// lifecycleUpdateInput holds only the attributes the user asked to change +// (nil = untouched), so the PATCH carries just those. +type lifecycleUpdateInput struct { + Name *string + Prefix *string + ExpirationDays *int64 + NoncurrentDays *int64 + AbortMpuDays *int64 + Enabled *bool +} + +func (in lifecycleUpdateInput) empty() bool { + return in.Name == nil && in.Prefix == nil && in.ExpirationDays == nil && + in.NoncurrentDays == nil && in.AbortMpuDays == nil && in.Enabled == nil +} + +// buildLifecycleUpdateRequest builds the PATCH body from the current rule and +// the changed attributes. The API requires Name on every PATCH, so the current +// name is sent when --name was not given. +func buildLifecycleUpdateRequest(current components.LifecycleRuleData, in lifecycleUpdateInput) (operations.PatchStorageBucketLifecycleRuleRequestBody, error) { + var body operations.PatchStorageBucketLifecycleRuleRequestBody + if in.empty() { + return body, exitcode.Errorf(exitcode.Usage, "nothing to update: pass at least one of --%s, --%s, --%s, --%s, --%s, --%s or --%s", + flagRuleName, flagRulePrefix, flagExpirationDays, flagNoncurrentDays, flagAbortMpuDays, flagRuleEnable, flagRuleDisable) + } + name := ruleName(current) + if in.Name != nil { + name = strings.TrimSpace(*in.Name) + } + if name == "" { + return body, exitcode.Errorf(exitcode.Usage, "the rule needs a name: pass --%s", flagRuleName) + } + if in.ExpirationDays != nil && *in.ExpirationDays <= 0 { + return body, exitcode.Errorf(exitcode.Usage, "the API requires --expiration-days on every rule (a positive number of days after which objects are deleted)") + } + if in.NoncurrentDays != nil && *in.NoncurrentDays < 0 { + return body, exitcode.Errorf(exitcode.Usage, "--%s must be a positive number of days", flagNoncurrentDays) + } + if in.AbortMpuDays != nil && *in.AbortMpuDays < 0 { + return body, exitcode.Errorf(exitcode.Usage, "--%s must be a positive number of days", flagAbortMpuDays) + } + body.Data = operations.PatchStorageBucketLifecycleRuleData{ + Type: operations.PatchStorageBucketLifecycleRuleTypeLifecycleRules, + Attributes: operations.PatchStorageBucketLifecycleRuleAttributes{ + Name: name, + Enabled: in.Enabled, + Prefix: in.Prefix, + ExpirationDays: in.ExpirationDays, + NoncurrentDays: in.NoncurrentDays, + AbortMpuDaysAfterInitiation: in.AbortMpuDays, + }, + } + return body, nil +} + +// plannedLifecycleUpdate merges the PATCH into the current rule for --dry-run. +func plannedLifecycleUpdate(current components.LifecycleRuleData, body operations.PatchStorageBucketLifecycleRuleRequestBody) LifecycleRule { + out := current + attrs := components.LifecycleRuleDataAttributes{} + if current.Attributes != nil { + attrs = *current.Attributes + } + a := body.Data.Attributes + name := a.Name + attrs.Name = &name + if a.Enabled != nil { + attrs.Enabled = a.Enabled + } + if a.Prefix != nil { + attrs.Prefix = a.Prefix + } + if a.ExpirationDays != nil { + attrs.ExpirationDays = a.ExpirationDays + } + if a.NoncurrentDays != nil { + attrs.NoncurrentDays = a.NoncurrentDays + } + if a.AbortMpuDaysAfterInitiation != nil { + attrs.AbortMpuDaysAfterInitiation = a.AbortMpuDaysAfterInitiation + } + out.Attributes = &attrs + return LifecycleRule{LifecycleRuleData: out} +} + +// newLifecycleUpdateCmd builds `lsh s3 lifecycle update s3://bucket `. +func newLifecycleUpdateCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "update ", + Aliases: []string{"edit", "set"}, + Short: "Update a lifecycle rule", + Long: `Update a lifecycle rule, addressed by id (lifecycle_...) or by name. + +Only the flags you pass are changed; --enable/--disable toggle the rule.`, + Example: ` lsh s3 lifecycle update s3://logs expire-7d-tmp --expiration-days 14 + lsh s3 lifecycle update s3://logs lifecycle_9x4kQ --disable + lsh s3 lifecycle update s3://logs expire-7d-tmp --name expire-14d-tmp --prefix tmp/`, + Args: cobra.ExactArgs(2), + RunE: runLifecycleUpdate, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + addLifecycleRuleFlags(cmd) + cmd.Flags().Bool(flagRuleEnable, false, "enable the rule") + cmd.Flags().Bool(flagRuleDisable, false, "disable the rule") + return cmd +} + +// lifecycleUpdateInputFromFlags collects only the flags that were set. +func lifecycleUpdateInputFromFlags(cmd *cobra.Command) lifecycleUpdateInput { + in := lifecycleUpdateInput{} + f := cmd.Flags() + if f.Changed(flagRuleName) { + v, _ := f.GetString(flagRuleName) + in.Name = &v + } + if f.Changed(flagRulePrefix) { + v, _ := f.GetString(flagRulePrefix) + in.Prefix = &v + } + if v, ok := expirationDaysFlag(cmd); ok { + in.ExpirationDays = &v + } + if f.Changed(flagNoncurrentDays) { + v, _ := f.GetInt64(flagNoncurrentDays) + in.NoncurrentDays = &v + } + if f.Changed(flagAbortMpuDays) { + v, _ := f.GetInt64(flagAbortMpuDays) + in.AbortMpuDays = &v + } + if v, _ := f.GetBool(flagRuleEnable); v { + t := true + in.Enabled = &t + } + if v, _ := f.GetBool(flagRuleDisable); v { + fl := false + in.Enabled = &fl + } + return in +} + +func runLifecycleUpdate(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + if en, _ := cmd.Flags().GetBool(flagRuleEnable); en { + if dis, _ := cmd.Flags().GetBool(flagRuleDisable); dis { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s and --%s are mutually exclusive", flagRuleEnable, flagRuleDisable)) + } + } + in := lifecycleUpdateInputFromFlags(cmd) + if in.empty() { + // Fail before any API call; the builder produces the same message. + _, err := buildLifecycleUpdateRequest(components.LifecycleRuleData{}, in) + return printErr(err) + } + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + current, err := lc.get(ctx, args[1]) + if err != nil { + return printErr(err) + } + body, err := buildLifecycleUpdateRequest(*current, in) + if err != nil { + return printErr(err) + } + id := ruleID(*current) + + if dryRun() { + if isHuman() { + fmt.Println(LifecycleMutation{Action: "update", Bucket: lc.uri(), ID: id, Name: body.Data.Attributes.Name, DryRun: true}.HumanLine()) + return nil + } + render([]renderer.ResponseData{plannedLifecycleUpdate(*current, body)}) + return nil + } + + updated, err := lc.update(ctx, id, body) + if err != nil { + return printErr(err) + } + if isHuman() { + fmt.Println(LifecycleMutation{Action: "update", Bucket: lc.uri(), ID: ruleID(*updated), Name: ruleName(*updated)}.HumanLine()) + return nil + } + render([]renderer.ResponseData{LifecycleRule{LifecycleRuleData: *updated}}) + return nil +} diff --git a/cmd/s3/lifecycle_update_test.go b/cmd/s3/lifecycle_update_test.go new file mode 100644 index 0000000..fa26742 --- /dev/null +++ b/cmd/s3/lifecycle_update_test.go @@ -0,0 +1,112 @@ +package s3 + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func TestBuildLifecycleUpdateRequestPartial(t *testing.T) { + current := components.LifecycleRuleData{ + ID: ptrS("lifecycle_a"), + Attributes: &components.LifecycleRuleDataAttributes{ + Name: ptrS("expire-7d-tmp"), + Prefix: ptrS("tmp/"), + ExpirationDays: ptrI(7), + Enabled: ptrB(true), + }, + } + + // Only --expiration-days changed: the name is filled from the rule and + // nothing else is sent. + body, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{ExpirationDays: ptrI(14)}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := body.Data.Attributes + if a.Name != "expire-7d-tmp" { + t.Errorf("name = %q, want the current name", a.Name) + } + if a.ExpirationDays == nil || *a.ExpirationDays != 14 { + t.Errorf("expiration_days = %v", a.ExpirationDays) + } + if a.Prefix != nil || a.Enabled != nil || a.NoncurrentDays != nil || a.AbortMpuDaysAfterInitiation != nil { + t.Errorf("untouched attributes must stay nil: %+v", a) + } + raw, _ := json.Marshal(body) + if strings.Contains(string(raw), `"prefix"`) || strings.Contains(string(raw), `"enabled"`) { + t.Errorf("payload should only carry name and expiration_days: %s", raw) + } + + // --disable plus a new name and an empty prefix (clears the filter). + body, err = buildLifecycleUpdateRequest(current, lifecycleUpdateInput{Name: ptrS(" new-name "), Prefix: ptrS(""), Enabled: ptrB(false)}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a = body.Data.Attributes + if a.Name != "new-name" { + t.Errorf("name = %q", a.Name) + } + if a.Enabled == nil || *a.Enabled { + t.Errorf("enabled = %v, want false", a.Enabled) + } + if a.Prefix == nil || *a.Prefix != "" { + t.Errorf("prefix = %v, want empty string", a.Prefix) + } + raw, _ = json.Marshal(body) + if !strings.Contains(string(raw), `"prefix":""`) || !strings.Contains(string(raw), `"enabled":false`) { + t.Errorf("payload should carry prefix and enabled: %s", raw) + } +} + +func TestBuildLifecycleUpdateRequestValidation(t *testing.T) { + current := components.LifecycleRuleData{ID: ptrS("lifecycle_a"), Attributes: &components.LifecycleRuleDataAttributes{Name: ptrS("n")}} + + if _, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{}); err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "nothing to update") { + t.Errorf("empty input: got %v", err) + } + if _, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{ExpirationDays: ptrI(0)}); err == nil || !strings.Contains(err.Error(), "the API requires --expiration-days on every rule") { + t.Errorf("zero expiration: got %v", err) + } + if _, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{NoncurrentDays: ptrI(-1)}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("negative noncurrent: got %v", err) + } + // A rule without a name needs --name. + nameless := components.LifecycleRuleData{ID: ptrS("lifecycle_b"), Attributes: &components.LifecycleRuleDataAttributes{}} + if _, err := buildLifecycleUpdateRequest(nameless, lifecycleUpdateInput{Enabled: ptrB(true)}); err == nil || !strings.Contains(err.Error(), "--name") { + t.Errorf("nameless rule: got %v", err) + } + if body, err := buildLifecycleUpdateRequest(nameless, lifecycleUpdateInput{Name: ptrS("fixed"), Enabled: ptrB(true)}); err != nil || body.Data.Attributes.Name != "fixed" { + t.Errorf("nameless rule with --name: %v %+v", err, body) + } +} + +func TestPlannedLifecycleUpdate(t *testing.T) { + current := components.LifecycleRuleData{ + ID: ptrS("lifecycle_a"), + Attributes: &components.LifecycleRuleDataAttributes{ + Name: ptrS("expire-7d-tmp"), + Prefix: ptrS("tmp/"), + ExpirationDays: ptrI(7), + Enabled: ptrB(true), + }, + } + body, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{ExpirationDays: ptrI(14), Enabled: ptrB(false)}) + if err != nil { + t.Fatal(err) + } + row := plannedLifecycleUpdate(current, body).TableRow() + if row["id"].Value != "lifecycle_a" || row["name"].Value != "expire-7d-tmp" || row["prefix"].Value != "tmp/" { + t.Errorf("planned update lost current values: %+v", row) + } + if row["expiration_days"].Value != "14d" || row["enabled"].Value != "false" { + t.Errorf("planned update did not apply the patch: %+v", row) + } + // The original must not be mutated. + if *current.Attributes.ExpirationDays != 7 || !*current.Attributes.Enabled { + t.Errorf("plannedLifecycleUpdate mutated the current rule") + } +} diff --git a/cmd/s3/ls.go b/cmd/s3/ls.go new file mode 100644 index 0000000..c7c093e --- /dev/null +++ b/cmd/s3/ls.go @@ -0,0 +1,453 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/cli" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/pagination" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" +) + +// Local flag names of `ls`. +const ( + flagRecursive = "recursive" + flagHumanReadable = "human-readable" + flagSummarize = "summarize" + flagVersions = "versions" + flagStartingToken = "starting-token" + flagStream = "stream" + flagAllProjects = "all-projects" + flagStorageClass = "storage-class" +) + +// maxListPageSize is the largest max-keys S3 accepts per request. +const maxListPageSize = 1000 + +// NewLsCmd builds `lsh s3 ls [s3://bucket[/prefix]]`. +func NewLsCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "ls [s3://bucket[/prefix]]", + Aliases: []string{"list"}, + Short: "List buckets, or the objects under a prefix", + Long: `List buckets, or the objects under a prefix. + +Without an argument, lists buckets through the Latitude API. Pick a project +with --project (or LSH_PROJECT), list every project with --all-projects, or — +in a terminal — choose from the project picker (which includes "All projects"). +--storage-class and --site narrow the listing. With an +argument, lists the objects and common prefixes under s3://[/] +straight from the bucket's S3 endpoint: the prefix is +matched literally ('s3://b/2026' matches every key starting with "2026"), +sub-prefixes are shown as PRE entries unless --recursive is given, and an empty +listing prints nothing and exits 0. + +Pagination follows the global flags: --page-size (default and maximum 1000 +keys per request), --max-items to cap the total number of entries and +--no-paginate to stop after one page; the token to resume from is printed on +stderr and accepted by --starting-token. + +Structured output (-o json|yaml|csv) emits one row per object or prefix; +--stream with -o json prints one JSON object per line as the listing arrives +instead of buffering everything.`, + Example: ` lsh s3 ls + lsh s3 ls --project my-project --storage-class high_performance + lsh s3 ls s3://backups/2026/09/ + lsh s3 ls s3://backups --recursive --human-readable --summarize + lsh s3 ls s3://backups/logs/ --versions + lsh s3 ls s3://backups --recursive -o json --stream | jq -r .key + lsh s3 ls s3://backups --no-paginate --page-size 100 + lsh s3 ls s3://backups --starting-token `, + Args: cobra.MaximumNArgs(1), + RunE: runLs, + }) + + addProjectFlag(cmd, true, "only list buckets of this project (ID or slug)") + cmd.Flags().Bool(flagAllProjects, false, "list buckets across every project (skip the project picker)") + cmd.Flags().StringP(flagStorageClass, "c", "", "only list buckets of this storage class (standard or high_performance)") + cmd.Flags().String(flagSite, "", "only list buckets in this Latitude site (e.g. DAL, TYO4)") + cmd.Flags().BoolP(flagRecursive, "r", false, "list every object under the prefix instead of stopping at the next '/'") + cmd.Flags().BoolP(flagHumanReadable, "H", false, "print sizes in human readable units (KiB, MiB…)") + cmd.Flags().Bool(flagSummarize, false, "append the total number of objects and their size") + cmd.Flags().Bool(flagVersions, false, "list every object version (versioned buckets)") + cmd.Flags().String(flagStartingToken, "", "resume the listing from the token printed by a previous --no-paginate or --max-items run") + cmd.Flags().Bool(flagStream, false, "with -o json, print one JSON object per line as the listing arrives (NDJSON)") + unsupportedAWSBoolFlags(cmd, map[string]string{ + "request-payer": "there is no requester-pays billing on Latitude object storage", + }) + rejectRegionFlag(cmd) + + return cmd +} + +func runLs(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + if len(args) == 0 { + return runLsBuckets(ctx, cmd) + } + return runLsObjects(ctx, cmd, args[0]) +} + +// --------------------------------------------------------------------------- +// Buckets +// --------------------------------------------------------------------------- + +func runLsBuckets(ctx context.Context, cmd *cobra.Command) error { + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "listing buckets needs the Latitude API and cannot be combined with --endpoint-url; pass s3:// to list objects on that endpoint")) + } + classFlag, _ := cmd.Flags().GetString(flagStorageClass) + // Shared alias table (standard|std|wasabi, high_performance|high-performance| + // hp|high|vast); an empty value means "no filter". + class, err := objectstorage.ParseStorageClass(classFlag) + if err != nil { + return printErr(fmt.Errorf("--%s: %w", flagStorageClass, err)) + } + + site, _ := cmd.Flags().GetString(flagSite) + + // Pick the project the same way as the rest of the CLI: --project / + // LSH_PROJECT, or --all-projects, otherwise an interactive picker with an + // "All projects" entry (and a usage error in a non-interactive session). + project, _, err := cli.PickProjectForList(cmd) + if err != nil { + return printErr(err) + } + + r := newResolver(cmd) + r.Project = project // "" when the user chose all projects + data, err := r.ListBuckets(ctx) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + data = filterBucketsByClass(data, class) + + // The SDK model drops the site slug, so fetch it best-effort for the Site + // column (and for the --site filter), scoped to the same --project filter + // as the listing. + sites, siteErr := objectstorage.RawBucketSitesForProject(ctx, "", project) + if siteErr != nil { + sites = nil + } + if s := strings.TrimSpace(site); s != "" { + // The filter depends entirely on that lookup (the SDK model drops the + // slug), so a failure would silently drop every bucket and exit 0. + if siteErr != nil { + return printErr(exitcode.Errorf(exitcode.Generic, "--site %s needs the sites of the listed buckets, which could not be fetched: %v", s, siteErr)) + } + data = filterBucketsBySite(data, s, sites) + } + + // Render through the shared renderer, exactly like every other `list` + // command (servers, volume, filesystems…): the default table, or + // -o json|yaml|csv|text with --query. + render(BucketRows(data, sites)) + return nil +} + +// filterBucketsBySite keeps the buckets in one Latitude site ("" keeps all). +// The site slug comes from the SDK model when present, otherwise from the +// site map fetched alongside the listing. +func filterBucketsBySite(data []components.ObjectStorageData, site string, sites map[string]string) []components.ObjectStorageData { + out := make([]components.ObjectStorageData, 0, len(data)) + for _, d := range data { + s := objectstorage.BucketFromData(d).Site + if s == "" && d.ID != nil { + s = sites[*d.ID] + } + if strings.EqualFold(s, site) { + out = append(out, d) + } + } + return out +} + +// filterBucketsByClass keeps the buckets of one storage class ("" keeps all). +func filterBucketsByClass(data []components.ObjectStorageData, class string) []components.ObjectStorageData { + if class == "" { + return data + } + out := make([]components.ObjectStorageData, 0, len(data)) + for _, d := range data { + if strings.EqualFold(objectstorage.BucketFromData(d).StorageClass, class) { + out = append(out, d) + } + } + return out +} + +// --------------------------------------------------------------------------- +// Objects +// --------------------------------------------------------------------------- + +// listOptions controls listObjects. +type listOptions struct { + // Prefix is matched literally (no trailing slash is added). + Prefix string + // Recursive drops the "/" delimiter. + Recursive bool + // Versions lists object versions instead of current objects. + Versions bool + // PageSize is the max-keys sent per request (1..1000; 0 means 1000). + PageSize int + // MaxItems caps the total number of entries (0 = unlimited). + MaxItems int64 + // NoPaginate stops after the first page. + NoPaginate bool + // StartingToken resumes a ListObjectsV2 listing from a continuation token. + StartingToken string + // OnEntry, when set, receives every entry as it arrives instead of it + // being accumulated in listResult.Entries (streaming output). + OnEntry func(renderer.ResponseData) +} + +// listResult is the outcome of listObjects. +type listResult struct { + // Entries holds prefixes and objects in listing order (empty when + // listOptions.OnEntry streamed them). + Entries []renderer.ResponseData + // Count and Bytes summarize the listed objects (prefixes excluded). + Count int + Bytes int64 + // Total is the number of entries emitted (prefixes included). + Total int64 + // NextToken is the continuation token to resume from when the listing + // stopped before the end ("" when exhausted). + NextToken string +} + +func (r *listResult) emit(opts listOptions, e renderer.ResponseData) { + r.Total++ + if o, ok := e.(objectstorage.Object); ok { + r.Count++ + r.Bytes += o.Size + } + if opts.OnEntry != nil { + opts.OnEntry(e) + return + } + r.Entries = append(r.Entries, e) +} + +// listObjects lists the objects (or versions) of bucketName under the prefix, +// paginating with continuation tokens and honoring the page-size, max-items, +// no-paginate and starting-token controls. It only issues GET requests. +func listObjects(ctx context.Context, client *minio.Client, bucketName string, opts listOptions) (listResult, error) { + if opts.PageSize <= 0 || opts.PageSize > maxListPageSize { + opts.PageSize = maxListPageSize + } + if opts.Versions { + return listVersions(ctx, client, bucketName, opts) + } + + var res listResult + delimiter := "/" + if opts.Recursive { + delimiter = "" + } + core := minio.Core{Client: client} + token := opts.StartingToken + for { + if err := ctx.Err(); err != nil { + return res, err + } + maxKeys := opts.PageSize + if opts.MaxItems > 0 { + remaining := opts.MaxItems - res.Total + if remaining <= 0 { + return res, nil + } + if remaining < int64(maxKeys) { + maxKeys = int(remaining) + } + } + page, err := listPage(ctx, core, bucketName, opts.Prefix, token, delimiter, maxKeys) + if err != nil { + return res, err + } + for _, cp := range page.CommonPrefixes { + res.emit(opts, objectstorage.NewPrefix(cp.Prefix)) + } + for _, info := range page.Contents { + res.emit(opts, objectstorage.ObjectFromInfo(info, false)) + } + next := page.NextContinuationToken + if !page.IsTruncated || next == "" { + res.NextToken = "" + return res, nil + } + res.NextToken = next + if opts.NoPaginate { + return res, nil + } + if opts.MaxItems > 0 && res.Total >= opts.MaxItems { + return res, nil + } + token = next + } +} + +// listPage issues one ListObjectsV2 request. minio.Core.ListObjectsV2 takes no +// context (it is the only entry point that exposes continuation tokens, which +// --starting-token / --no-paginate need), so the call runs in a goroutine and +// the caller stops waiting as soon as ctx is cancelled: Ctrl-C then returns +// ctx.Err(), which Humanize maps to exit 130. An abandoned request finishes in +// the background and its result is dropped (the channel is buffered). +func listPage(ctx context.Context, core minio.Core, bucketName, prefix, token, delimiter string, maxKeys int) (minio.ListBucketV2Result, error) { + type pageResult struct { + page minio.ListBucketV2Result + err error + } + done := make(chan pageResult, 1) + go func() { + page, err := core.ListObjectsV2(bucketName, prefix, "", token, delimiter, maxKeys) + done <- pageResult{page: page, err: err} + }() + select { + case <-ctx.Done(): + return minio.ListBucketV2Result{}, ctx.Err() + case r := <-done: + return r.page, r.err + } +} + +// listVersions lists object versions (ListObjectVersions) through the minio +// iterator. Versions listings have no continuation token to expose, so +// --no-paginate and --starting-token do not apply; --max-items does. +func listVersions(ctx context.Context, client *minio.Client, bucketName string, opts listOptions) (listResult, error) { + var res listResult + ctx, cancel := context.WithCancel(ctx) + defer cancel() + ch := client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{ + Prefix: opts.Prefix, + Recursive: opts.Recursive, + WithVersions: true, + MaxKeys: opts.PageSize, + }) + for info := range ch { + if info.Err != nil { + return res, info.Err + } + if opts.MaxItems > 0 && res.Total >= opts.MaxItems { + cancel() + // Drain so the producer goroutine exits. + for range ch { + } + break + } + // minio reports common prefixes of a non-recursive listing as entries + // with a key ending in "/" and no version. + if !opts.Recursive && strings.HasSuffix(info.Key, "/") && info.VersionID == "" && info.Size == 0 { + res.emit(opts, objectstorage.NewPrefix(info.Key)) + continue + } + res.emit(opts, objectstorage.ObjectFromInfo(info, true)) + } + return res, nil +} + +// streamJSON writes one entry as a single JSON line (NDJSON). +func streamJSON(w io.Writer, e renderer.ResponseData) error { + b, err := json.Marshal(e) + if err != nil { + return err + } + _, err = fmt.Fprintln(w, string(b)) + return err +} + +// resolvePageSize returns the max-keys per request: --page-size when given +// explicitly (capped at 1000), else 1000. +func resolvePageSize(cmd *cobra.Command, resolved pagination.Options) int { + if !cmd.Flags().Changed("page-size") { + return maxListPageSize + } + if resolved.PageSize <= 0 || resolved.PageSize > maxListPageSize { + return maxListPageSize + } + return int(resolved.PageSize) +} + +func runLsObjects(ctx context.Context, cmd *cobra.Command, arg string) error { + ref, err := objectstorage.ParseRemote(arg) + if err != nil { + return printErr(err) + } + recursive, _ := cmd.Flags().GetBool(flagRecursive) + human, _ := cmd.Flags().GetBool(flagHumanReadable) + summarize, _ := cmd.Flags().GetBool(flagSummarize) + versions, _ := cmd.Flags().GetBool(flagVersions) + stream, _ := cmd.Flags().GetBool(flagStream) + startingToken, _ := cmd.Flags().GetString(flagStartingToken) + + pg := pagination.Resolve() + opts := listOptions{ + Prefix: ref.Key, + Recursive: recursive, + Versions: versions, + PageSize: resolvePageSize(cmd, pg), + MaxItems: pg.MaxItems, + NoPaginate: pg.NoPaginate, + StartingToken: startingToken, + } + if versions && (pg.NoPaginate || startingToken != "") { + objectstorage.Warnf("--no-paginate and --starting-token do not apply to --versions listings; ignoring them") + opts.NoPaginate = false + opts.StartingToken = "" + } + + format := renderer.ResolveFormat() + if stream && format != renderer.FormatJSON { + objectstorage.Warnf("--stream only applies to -o json; ignoring it") + stream = false + } + + if dryRun() { + objectstorage.Hintf("(dryrun) ls is read-only; listing %s", ref) + } + + sess, err := openBucket(ctx, cmd, ref.Bucket, false) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + + // Objects render through the shared renderer like every other list command + // (default table, or -o json|yaml|csv|text with --query). --stream keeps + // the NDJSON fast path for -o json over very large listings. + if stream { + opts.OnEntry = func(e renderer.ResponseData) { + if err := streamJSON(os.Stdout, e); err != nil { + objectstorage.Warnf("could not write entry: %v", err) + } + } + } + + res, err := listObjects(ctx, sess.Client, sess.Bucket.BucketName, opts) + if err != nil { + return printErr(sess.humanize(err)) + } + + if !stream { + render(res.Entries) + if summarize && renderer.ResolveFormat() == renderer.FormatTable { + fmt.Print(objectstorage.LsSummary(res.Count, res.Bytes, human)) + fmt.Println() + } + } + + if res.NextToken != "" { + objectstorage.Hintf("Next token: %s (use --starting-token)", res.NextToken) + } + return nil +} diff --git a/cmd/s3/ls_test.go b/cmd/s3/ls_test.go new file mode 100644 index 0000000..9d20f3d --- /dev/null +++ b/cmd/s3/ls_test.go @@ -0,0 +1,485 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" + "github.com/latitudesh/lsh/internal/renderer" +) + +const lsTestBucket = "backups-7f3a" + +// newLsTestSession builds a Session against the fake server. +func newLsTestSession(t *testing.T, srv *s3test.Server, bucket string) *Session { + t.Helper() + b := &objectstorage.Bucket{ + ID: "bkt_1", + Name: "backups", + BucketName: bucket, + Endpoint: srv.URL(), + StorageClass: "standard", + SigningRegion: "us-east-1", + } + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +// seedLsBucket fills the bucket with objects under two prefixes plus one at +// the root and forces two-key pages so continuation tokens are exercised. +func seedLsBucket(t *testing.T) (*s3test.Server, *Session) { + t.Helper() + srv := s3test.New() + t.Cleanup(srv.Close) + srv.MaxListKeys = 2 + srv.CreateBucket(lsTestBucket) + srv.AddObject(lsTestBucket, "2026/09/dump.sql", make([]byte, 1258291), "application/sql") + srv.AddObject(lsTestBucket, "2026/09/notes.txt", []byte("notes"), "text/plain") + srv.AddObject(lsTestBucket, "logs/app.log", []byte("log line"), "text/plain") + srv.AddObject(lsTestBucket, "readme.md", []byte("# hi"), "text/markdown") + srv.ResetRequests() + return srv, newLsTestSession(t, srv, lsTestBucket) +} + +func entryNames(entries []renderer.ResponseData) []string { + out := make([]string, 0, len(entries)) + for _, e := range entries { + switch v := e.(type) { + case objectstorage.Prefix: + out = append(out, "PRE "+v.Prefix) + case objectstorage.Object: + out = append(out, v.Key) + } + } + return out +} + +func assertNoWrites(t *testing.T, srv *s3test.Server) { + t.Helper() + if w := srv.WriteRequests(); len(w) != 0 { + t.Fatalf("ls issued %d write requests: %+v", len(w), w) + } +} + +func listRequests(srv *s3test.Server) []s3test.Request { + var out []s3test.Request + for _, r := range srv.Requests() { + if r.Method == http.MethodGet && r.Query.Get("list-type") == "2" { + out = append(out, r) + } + } + return out +} + +func TestListObjectsDelimiter(t *testing.T) { + srv, sess := seedLsBucket(t) + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + got := strings.Join(entryNames(res.Entries), ",") + want := "PRE 2026/,PRE logs/,readme.md" + if got != want { + t.Fatalf("entries = %q, want %q", got, want) + } + if res.NextToken != "" { + t.Fatalf("NextToken = %q, want empty (listing exhausted)", res.NextToken) + } + if res.Count != 1 || res.Bytes != 4 { + t.Fatalf("summary = %d objects / %d bytes, want 1 / 4", res.Count, res.Bytes) + } + reqs := listRequests(srv) + if len(reqs) < 2 { + t.Fatalf("expected the listing to paginate with tokens, got %d list requests", len(reqs)) + } + if reqs[1].Query.Get("continuation-token") == "" { + t.Fatalf("second page did not carry a continuation token: %v", reqs[1].Query) + } + for _, r := range reqs { + if r.Query.Get("delimiter") != "/" { + t.Fatalf("delimiter = %q, want /", r.Query.Get("delimiter")) + } + if r.Query.Get("max-keys") != "1000" { + t.Fatalf("max-keys = %q, want 1000 by default", r.Query.Get("max-keys")) + } + } + assertNoWrites(t, srv) +} + +func TestListObjectsRecursive(t *testing.T) { + srv, sess := seedLsBucket(t) + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + got := strings.Join(entryNames(res.Entries), ",") + want := "2026/09/dump.sql,2026/09/notes.txt,logs/app.log,readme.md" + if got != want { + t.Fatalf("entries = %q, want %q", got, want) + } + if res.Count != 4 || res.Bytes != 1258291+5+8+4 { + t.Fatalf("summary = %d objects / %d bytes", res.Count, res.Bytes) + } + for _, r := range listRequests(srv) { + if r.Query.Get("delimiter") != "" { + t.Fatalf("recursive listing sent delimiter %q", r.Query.Get("delimiter")) + } + } + assertNoWrites(t, srv) +} + +func TestListObjectsLiteralPrefix(t *testing.T) { + _, sess := seedLsBucket(t) + // No trailing slash is added: "2026" matches keys starting with "2026". + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Prefix: "2026"}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if got := strings.Join(entryNames(res.Entries), ","); got != "PRE 2026/" { + t.Fatalf("entries = %q, want PRE 2026/", got) + } + res, err = listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Prefix: "2026/09/"}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if got := strings.Join(entryNames(res.Entries), ","); got != "2026/09/dump.sql,2026/09/notes.txt" { + t.Fatalf("entries = %q", got) + } +} + +func TestListObjectsMaxItemsAndStartingToken(t *testing.T) { + srv, sess := seedLsBucket(t) + first, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true, MaxItems: 3}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if len(first.Entries) != 3 { + t.Fatalf("got %d entries, want 3 (max-items)", len(first.Entries)) + } + if first.NextToken == "" { + t.Fatalf("expected a NextToken when max-items stops early") + } + // max-items never over-fetches: the last request asks only for what is left. + reqs := listRequests(srv) + if last := reqs[len(reqs)-1]; last.Query.Get("max-keys") != "1" { + t.Fatalf("last max-keys = %q, want 1", last.Query.Get("max-keys")) + } + + rest, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true, StartingToken: first.NextToken}) + if err != nil { + t.Fatalf("listObjects (resume): %v", err) + } + if got := strings.Join(entryNames(rest.Entries), ","); got != "readme.md" { + t.Fatalf("resumed entries = %q, want readme.md", got) + } + if rest.NextToken != "" { + t.Fatalf("NextToken after the last page = %q", rest.NextToken) + } + assertNoWrites(t, srv) +} + +func TestListObjectsNoPaginate(t *testing.T) { + srv, sess := seedLsBucket(t) + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true, NoPaginate: true, PageSize: 2}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if len(res.Entries) != 2 { + t.Fatalf("got %d entries, want one page of 2", len(res.Entries)) + } + if res.NextToken == "" { + t.Fatalf("expected a NextToken after one page") + } + if reqs := listRequests(srv); len(reqs) != 1 { + t.Fatalf("--no-paginate issued %d list requests, want 1", len(reqs)) + } +} + +func TestListObjectsStreamsEntries(t *testing.T) { + _, sess := seedLsBucket(t) + var buf bytes.Buffer + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{ + Recursive: true, + OnEntry: func(e renderer.ResponseData) { + if err := streamJSON(&buf, e); err != nil { + t.Fatalf("streamJSON: %v", err) + } + }, + }) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if len(res.Entries) != 0 { + t.Fatalf("streamed entries must not be accumulated, got %d", len(res.Entries)) + } + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + if len(lines) != 4 { + t.Fatalf("got %d NDJSON lines, want 4:\n%s", len(lines), buf.String()) + } + var first map[string]interface{} + if err := json.Unmarshal([]byte(lines[0]), &first); err != nil { + t.Fatalf("line 0 is not JSON: %v", err) + } + if first["key"] != "2026/09/dump.sql" || first["type"] != "object" { + t.Fatalf("unexpected first line: %v", first) + } + if res.Count != 4 { + t.Fatalf("Count = %d, want 4 even when streaming", res.Count) + } +} + +func TestListObjectsErrorsSurface(t *testing.T) { + srv, sess := seedLsBucket(t) + srv.DenyAll = true + _, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{}) + if err == nil { + t.Fatalf("expected an error from a denied listing") + } + herr := sess.humanize(err) + if !strings.Contains(herr.Error(), "access denied") { + t.Fatalf("humanized error = %q", herr) + } +} + +func TestListObjectsVersions(t *testing.T) { + srv := s3test.New() + t.Cleanup(srv.Close) + b := srv.CreateBucket(lsTestBucket) + b.Versioned = true + srv.AddObject(lsTestBucket, "cfg/app.yaml", []byte("v1"), "text/yaml") + srv.AddObject(lsTestBucket, "cfg/app.yaml", []byte("v2!"), "text/yaml") + sess := newLsTestSession(t, srv, lsTestBucket) + + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Prefix: "cfg/", Recursive: true, Versions: true}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if len(res.Entries) != 2 { + t.Fatalf("got %d version entries, want 2: %v", len(res.Entries), entryNames(res.Entries)) + } + // The fake server assigns version IDs before archiving the previous + // version, so only the ordering, sizes and IsLatest are asserted here. + latest, ok := res.Entries[0].(objectstorage.Object) + if !ok || latest.Size != 3 || latest.VersionID == "" || latest.IsLatest == nil || !*latest.IsLatest { + t.Fatalf("first entry should be the latest version (3 bytes): %+v", res.Entries[0]) + } + older := res.Entries[1].(objectstorage.Object) + if older.Size != 2 || older.VersionID == "" || older.IsLatest == nil || *older.IsLatest { + t.Fatalf("second entry should be the older version (2 bytes): %+v", older) + } + + // --max-items applies to version listings too. + capped, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true, Versions: true, MaxItems: 1}) + if err != nil { + t.Fatalf("listObjects (capped): %v", err) + } + if len(capped.Entries) != 1 { + t.Fatalf("got %d entries, want 1", len(capped.Entries)) + } + assertNoWrites(t, srv) +} + +// TestLsObjectEntries lists a seeded prefix and checks the renderable entries +// (one PRE prefix + the objects with their sizes), which the shared renderer +// then prints as the table / json / csv the rest of the CLI uses. +func TestLsObjectEntries(t *testing.T) { + _, sess := seedLsBucket(t) + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Prefix: "2026/09/"}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if res.Count != 2 || res.Bytes != 1258296 { + t.Fatalf("count=%d bytes=%d, want 2 / 1258296", res.Count, res.Bytes) + } + var sizes []int64 + for _, e := range res.Entries { + if o, ok := e.(objectstorage.Object); ok { + sizes = append(sizes, o.Size) + } + } + if len(sizes) != 2 || sizes[0] != 1258291 { + t.Fatalf("object sizes = %v", sizes) + } +} + +// TestLsSummary checks the --summarize footer helper (used only in table mode). +func TestLsSummary(t *testing.T) { + if got := objectstorage.LsSummary(3, 3*1024*1024, true); got != "\nTotal Objects: 3\n Total Size: 3.0 MiB" { + t.Fatalf("human summary = %q", got) + } + if got := objectstorage.LsSummary(2, 1258296, false); got != "\nTotal Objects: 2\n Total Size: 1258296" { + t.Fatalf("byte summary = %q", got) + } +} + +func TestBucketRowsSortedAndFiltered(t *testing.T) { + created := time.Date(2026, 9, 7, 15, 12, 1, 0, time.Local) + mk := func(id, name string, class components.StorageClass) components.ObjectStorageData { + n := name + i := id + c := class + t := created + return components.ObjectStorageData{ID: &i, Attributes: &components.ObjectStorageDataAttributes{Name: &n, StorageClass: &c, CreatedAt: &t}} + } + data := []components.ObjectStorageData{ + mk("bkt_2", "logs", components.StorageClass("high_performance")), + mk("bkt_1", "backups", components.StorageClass("standard")), + mk("bkt_3", "media", components.StorageClass("standard")), + } + // BucketRows renders through the shared renderer; the row for a bucket keeps + // its name so the table shows it. + rows := BucketRows(data, nil) + if len(rows) != 3 || rows[0].TableRow()["name"].Value != "logs" { + t.Fatalf("unexpected rows: %d", len(rows)) + } + + // --storage-class goes through the shared alias table so "high-performance" + // (and hp, vast…) mean the same thing here as in mb and access-keys. + class, err := objectstorage.ParseStorageClass("high-performance") + if err != nil || class != objectstorage.ClassHighPerformance { + t.Fatalf("ParseStorageClass = %q, %v", class, err) + } + filtered := filterBucketsByClass(data, class) + if len(filtered) != 1 || *filtered[0].Attributes.Name != "logs" { + t.Fatalf("filtered = %v", entryBucketNames(filtered)) + } + // An empty value is "no filter", not an error. + if class, err := objectstorage.ParseStorageClass(""); err != nil || class != "" { + t.Fatalf("ParseStorageClass(\"\") = %q, %v; want no filter", class, err) + } + if len(filterBucketsByClass(data, "")) != len(data) { + t.Fatalf("an empty class must keep every bucket") + } + if _, err := objectstorage.ParseStorageClass("glacier"); exitcode.Of(err) != exitcode.Usage { + t.Fatalf("expected a usage error for an unknown storage class, got %v", err) + } +} + +// TestListObjectsCancelBetweenPages cancels the context while the first page +// is being consumed: the loop must stop before requesting the second page and +// surface context.Canceled, which the session maps to exit 130. +func TestListObjectsCancelBetweenPages(t *testing.T) { + srv, sess := seedLsBucket(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var seen int + res, err := listObjects(ctx, sess.Client, lsTestBucket, listOptions{ + Recursive: true, + OnEntry: func(renderer.ResponseData) { + seen++ + cancel() + }, + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if seen == 0 || seen > srv.MaxListKeys { + t.Fatalf("saw %d entries, want only the first page (<= %d)", seen, srv.MaxListKeys) + } + if got := len(listRequests(srv)); got != 1 { + t.Fatalf("issued %d list requests after cancellation, want 1", got) + } + if res.Total != int64(seen) { + t.Fatalf("Total = %d, want %d", res.Total, seen) + } + if code := exitcode.Of(sess.humanize(err)); code != exitcode.Interrupted { + t.Fatalf("humanized exit code = %d, want %d (interrupted)", code, exitcode.Interrupted) + } +} + +// TestListObjectsCancelDuringRequest cancels the context while a list request +// is still in flight: minio.Core.ListObjectsV2 has no context, so listObjects +// must stop waiting on its own instead of hanging until the server answers. +func TestListObjectsCancelDuringRequest(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + var once bool + blocking := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !once { + once = true + close(started) + } + <-release + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(blocking.Close) + + b := &objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: lsTestBucket, Endpoint: blocking.URL, SigningRegion: "us-east-1"} + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + sess := &Session{Bucket: b, Cred: cred, Client: client} + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-started + cancel() + }() + done := make(chan error, 1) + go func() { + _, err := listObjects(ctx, sess.Client, lsTestBucket, listOptions{Recursive: true}) + done <- err + }() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if code := exitcode.Of(sess.humanize(err)); code != exitcode.Interrupted { + t.Fatalf("humanized exit code = %d, want %d (interrupted)", code, exitcode.Interrupted) + } + case <-time.After(5 * time.Second): + t.Fatalf("listObjects did not return after the context was cancelled while the request was in flight") + } + // Let the abandoned request finish so the server can shut down cleanly. + close(release) +} + +// TestLsInvalidBackendNameExitsUsage drives the real openBucket path with an +// endpoint override: a backend bucket name S3 rejects ("b" is shorter than the +// three characters the protocol requires) must be a usage error (exit 2) +// before any request or credential lookup happens. +func TestLsInvalidBackendNameExitsUsage(t *testing.T) { + srv := s3test.New() + t.Cleanup(srv.Close) + t.Setenv(objectstorage.EnvEndpointURL, srv.URL()) + + cmd := NewLsCmd() + err := runLsObjects(context.Background(), cmd, "s3://b/") + if err == nil { + t.Fatalf("expected an error for the one-letter backend bucket name") + } + if code := exitcode.Of(err); code != exitcode.Usage { + t.Fatalf("exit code = %d, want %d; err=%v", code, exitcode.Usage, err) + } + if !strings.Contains(err.Error(), "invalid bucket name") { + t.Fatalf("error should name the invalid bucket: %v", err) + } + if reqs := srv.Requests(); len(reqs) != 0 { + t.Fatalf("no request should reach the endpoint, got %d", len(reqs)) + } +} + +func entryBucketNames(data []components.ObjectStorageData) []string { + out := make([]string, 0, len(data)) + for _, d := range data { + out = append(out, objectstorage.BucketFromData(d).Name) + } + return out +} diff --git a/cmd/s3/mb.go b/cmd/s3/mb.go new file mode 100644 index 0000000..8ed0a03 --- /dev/null +++ b/cmd/s3/mb.go @@ -0,0 +1,613 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "strings" + "time" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +// Flag names of mb. +const ( + flagMbName = "name" + flagMbRegion = "region" + flagMbStorageClass = "storage-class" + flagMbVersioning = "versioning" + flagMbLocking = "locking" + flagMbRetentionMode = "retention-mode" + flagMbRetentionDays = "retention-days" + flagMbCreateAccessKey = "create-access-key" + flagMbNoAccessKey = "no-access-key" +) + +// mbOptions are the validated inputs of `mb`. +type mbOptions struct { + Name string + Project string + Region string + StorageClass string + // Versioning is the --versioning value; VersioningSet records whether the + // flag was given explicitly (so --locking --versioning=false is caught). + Versioning bool + VersioningSet bool + Locking bool + RetentionMode string + RetentionDays int64 +} + +// NewMbCmd builds `lsh s3 mb s3://bucket`. +func NewMbCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "mb s3://bucket", + Aliases: []string{"create"}, + Short: "Create a bucket", + Long: `Create an object storage bucket through the Latitude API. + +The bucket is created in a Latitude site (--region DAL, NYC, TYO4…). The +storage class picks the backend tier: standard (default) or +high_performance (select sites only). Versioning, object lock and the default +retention cannot be changed after creation. + +In an interactive session missing --project and --region are asked for, and +right after the bucket is created the CLI offers to create and save an S3 +access key so cp/ls/rm work immediately; use --create-access-key or +--no-access-key in scripts.`, + Example: ` lsh s3 mb s3://backups --region DAL --project my-project + lsh s3 mb s3://fast-cache --region TYO4 --storage-class high_performance + lsh s3 mb s3://audit --region NYC --locking --retention-mode COMPLIANCE --retention-days 30 + lsh s3 mb s3://ci-artifacts --region DAL --create-access-key + lsh s3 mb s3://backups --region DAL -o json`, + Args: cobra.MaximumNArgs(1), + RunE: runMb, + }) + // Registered as required so the root pre-run opens the shared project + // picker (or fails with exit 2 in a non-interactive session), like every + // other project-scoped command. + addProjectFlag(cmd, false, "project ID or slug that owns the bucket (required; a picker opens when omitted in a terminal)") + cmd.Flags().String(flagMbName, "", "bucket name (legacy alternative to the positional s3://bucket)") + _ = cmd.Flags().MarkHidden(flagMbName) + cmd.Flags().String(flagMbRegion, "", "Latitude site slug where the bucket lives (e.g. DAL, NYC, TYO4); see 'lsh regions list'") + cmd.Flags().StringP(flagMbStorageClass, "c", objectstorage.ClassStandard, "storage class: standard or high_performance (aliases: std, hp)") + cmd.Flags().Bool(flagMbVersioning, false, "enable object versioning") + cmd.Flags().Bool(flagMbLocking, false, "enable object lock (WORM); implies --versioning and cannot be added later") + cmd.Flags().String(flagMbRetentionMode, "", "default object lock retention mode: NONE, GOVERNANCE or COMPLIANCE (requires --locking)") + cmd.Flags().Int64(flagMbRetentionDays, 0, "default object lock retention period in days (requires --locking)") + cmd.Flags().Bool(flagMbCreateAccessKey, false, "create and save an access key for the new bucket's class without asking") + cmd.Flags().Bool(flagMbNoAccessKey, false, "never offer to create an access key") + unsupportedAWSFlags(cmd, map[string]string{ + "tags": "buckets have no tags; use the bucket name or project to organise them", + }) + return cmd +} + +func runMb(cmd *cobra.Command, args []string) error { + o, err := mbOptionsFromCmd(cmd, args) + if err != nil { + return printErr(err) + } + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "mb creates buckets through the Latitude API; unset --endpoint-url / %s", objectstorage.EnvEndpointURL)) + } + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + if o.Region == "" { + region, err := pickRegion(ctx, cmd) + if err != nil { + return printErr(err) + } + o.Region = region + } + req, err := buildCreateBucketRequest(o) + if err != nil { + return printErr(err) + } + a := req.Data.Attributes + + if dryRun() { + if isHuman() { + fmt.Printf("(dryrun) make_bucket: s3://%s\n", o.Name) + fmt.Println(" " + describeCreateRequest(a)) + } else { + render([]renderer.ResponseData{mbPlan{Attributes: a, DryRun: true}}) + } + return nil + } + + if o.Locking { + objectstorage.Warnf("object lock, versioning and the default retention cannot be changed after the bucket is created") + if strings.EqualFold(o.RetentionMode, string(operations.RetentionModeCompliance)) && objectstorage.CanPrompt(cmd) { + ok, err := objectstorage.Confirm(fmt.Sprintf("COMPLIANCE retention of %d days cannot be shortened or bypassed by anyone. Create bucket %s?", o.RetentionDays, o.Name)) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + if !ok { + return printErr(exitcode.Errorf(exitcode.Refused, "cancelled")) + } + } + } + + client := apiClient() + resp, err := client.ObjectStorage.PostStorageBuckets(ctx, req, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + if resp == nil || resp.Object == nil || resp.Object.Data == nil { + return printErr(exitcode.Errorf(exitcode.Generic, "the API returned no bucket for %s", o.Name)) + } + b := objectstorage.BucketFromData(*resp.Object.Data) + if b.Name == "" { + b.Name = o.Name + } + if b.Site == "" { + b.Site = strings.ToUpper(o.Region) + } + if b.StorageClass == "" { + b.StorageClass = string(*a.StorageClass) + } + if b.ProjectID == "" && b.ProjectSlug == "" { + b.ProjectSlug = o.Project + } + + if isHuman() { + fmt.Printf("make_bucket: s3://%s\n", b.Name) + for _, line := range describeCreatedBucket(b) { + fmt.Println(" " + line) + } + } else { + render([]renderer.ResponseData{NewBucketRow(b)}) + } + + offerAccessKey(ctx, cmd, b, o) + if isHuman() { + objectstorage.Hintf("Try: lsh s3 cp ./file s3://%s/", b.Name) + } + return nil +} + +// mbOptionsFromCmd reads the flags and positional argument. +func mbOptionsFromCmd(cmd *cobra.Command, args []string) (mbOptions, error) { + f := cmd.Flags() + o := mbOptions{Project: projectFlag(cmd)} + legacy, _ := f.GetString(flagMbName) + switch { + case len(args) == 1 && legacy != "" && !sameBucketToken(args[0], legacy): + return o, exitcode.Errorf(exitcode.Usage, "bucket given twice (%q and --name %q); pass only one", args[0], legacy) + case len(args) == 1: + ref, err := objectstorage.ParseBucketOnly(args[0]) + if err != nil { + return o, err + } + o.Name = ref.Bucket + case legacy != "": + ref, err := objectstorage.ParseBucketOnly(legacy) + if err != nil { + return o, err + } + o.Name = ref.Bucket + default: + return o, exitcode.Errorf(exitcode.Usage, "missing bucket: usage 'lsh s3 mb s3:// --region '") + } + o.Region, _ = f.GetString(flagMbRegion) + o.StorageClass, _ = f.GetString(flagMbStorageClass) + o.Versioning, _ = f.GetBool(flagMbVersioning) + o.VersioningSet = f.Changed(flagMbVersioning) + o.Locking, _ = f.GetBool(flagMbLocking) + o.RetentionMode, _ = f.GetString(flagMbRetentionMode) + o.RetentionDays, _ = f.GetInt64(flagMbRetentionDays) + return o, nil +} + +func sameBucketToken(a, b string) bool { + ra, errA := objectstorage.ParseBucketOnly(a) + rb, errB := objectstorage.ParseBucketOnly(b) + return errA == nil && errB == nil && ra.Bucket == rb.Bucket +} + +// buildCreateBucketRequest validates the options and builds the API request. +// It is pure so the rules can be unit-tested without cobra or the API. +func buildCreateBucketRequest(o mbOptions) (operations.PostStorageBucketsRequestBody, error) { + var req operations.PostStorageBucketsRequestBody + name := strings.TrimSpace(o.Name) + if name == "" { + return req, exitcode.Errorf(exitcode.Usage, "missing bucket name: usage 'lsh s3 mb s3:// --region '") + } + if strings.ContainsAny(name, " /\\") { + return req, exitcode.Errorf(exitcode.Usage, "invalid bucket name %q: it cannot contain spaces or slashes", name) + } + project := strings.TrimSpace(o.Project) + if project == "" { + return req, exitcode.Errorf(exitcode.Usage, "--project is required (or set LSH_PROJECT): the bucket must belong to a project") + } + region := strings.TrimSpace(o.Region) + if region == "" { + return req, exitcode.Errorf(exitcode.Usage, "--region is required; regions are Latitude sites such as DAL, NYC or TYO4 (run 'lsh regions list')") + } + if looksLikeAWSRegion(region) { + return req, exitcode.Errorf(exitcode.Usage, "%q is not a Latitude site; --region takes a site slug such as DAL, NYC or TYO4 (run 'lsh regions list')", region) + } + + // The shared alias table (std, hp, high-performance…) applies here too. + class, err := objectstorage.ParseStorageClass(o.StorageClass) + if err != nil { + return req, exitcode.Errorf(exitcode.Usage, "invalid --storage-class %q: use standard or high_performance", o.StorageClass) + } + if class == "" { + class = objectstorage.ClassStandard + } + var storageClass operations.StorageClass + switch class { + case objectstorage.ClassHighPerformance: + storageClass = operations.StorageClassHighPerformance + default: + storageClass = operations.StorageClassStandard + } + + mode := strings.ToUpper(strings.TrimSpace(o.RetentionMode)) + var retentionMode operations.RetentionMode + switch mode { + case "", string(operations.RetentionModeNone): + retentionMode = operations.RetentionModeNone + case string(operations.RetentionModeGovernance): + retentionMode = operations.RetentionModeGovernance + case string(operations.RetentionModeCompliance): + retentionMode = operations.RetentionModeCompliance + default: + return req, exitcode.Errorf(exitcode.Usage, "invalid --retention-mode %q: use NONE, GOVERNANCE or COMPLIANCE", o.RetentionMode) + } + if o.RetentionDays < 0 { + return req, exitcode.Errorf(exitcode.Usage, "--retention-days must be a positive number of days") + } + hasRetention := retentionMode != operations.RetentionModeNone || o.RetentionDays > 0 + if hasRetention && !o.Locking { + return req, exitcode.Errorf(exitcode.Usage, "--retention-mode and --retention-days require --locking (object lock must be enabled at creation)") + } + if retentionMode != operations.RetentionModeNone && o.RetentionDays == 0 { + return req, exitcode.Errorf(exitcode.Usage, "--retention-mode %s requires --retention-days ", retentionMode) + } + if o.RetentionDays > 0 && retentionMode == operations.RetentionModeNone { + return req, exitcode.Errorf(exitcode.Usage, "--retention-days requires --retention-mode GOVERNANCE or COMPLIANCE") + } + + versioning := o.Versioning + if o.Locking { + if o.VersioningSet && !o.Versioning { + return req, exitcode.Errorf(exitcode.Usage, "--locking requires versioning; drop --versioning=false") + } + versioning = true + } + + attrs := operations.PostStorageBucketsAttributes{ + Project: project, + Name: name, + Region: strings.ToUpper(region), + StorageClass: &storageClass, + Versioning: &versioning, + Locking: mbPtr(o.Locking), + RetentionMode: &retentionMode, + } + if o.RetentionDays > 0 { + attrs.RetentionPeriod = mbPtr(o.RetentionDays) + } + req.Data = operations.PostStorageBucketsData{ + Type: operations.PostStorageBucketsTypeObjects, + Attributes: attrs, + } + return req, nil +} + +func mbPtr[T any](v T) *T { return &v } + +// describeCreateRequest renders the request for dry-run output. +func describeCreateRequest(a operations.PostStorageBucketsAttributes) string { + parts := []string{ + "project: " + a.Project, + "region: " + a.Region, + "class: " + string(*a.StorageClass), + "versioning: " + yesNo(a.Versioning != nil && *a.Versioning), + } + if a.Locking != nil && *a.Locking { + lock := "yes" + if a.RetentionMode != nil && *a.RetentionMode != operations.RetentionModeNone { + lock = string(*a.RetentionMode) + if a.RetentionPeriod != nil { + lock = fmt.Sprintf("%s (%dd)", lock, *a.RetentionPeriod) + } + } + parts = append(parts, "locking: "+lock) + } else { + parts = append(parts, "locking: no") + } + return strings.Join(parts, " ") +} + +// describeCreatedBucket renders the detail lines under `make_bucket:`. +func describeCreatedBucket(b *objectstorage.Bucket) []string { + first := []string{} + if b.ID != "" { + first = append(first, "id: "+b.ID) + } + if b.BucketName != "" { + first = append(first, "bucket name on the endpoint: "+b.BucketName) + } + if b.StorageClass != "" { + first = append(first, "class: "+b.StorageClass) + } + if b.Site != "" { + first = append(first, "site: "+b.Site) + } + if b.Endpoint != "" { + first = append(first, "endpoint: "+b.Endpoint) + } + lines := []string{strings.Join(first, " ")} + if b.Versioning || b.Locking { + lines = append(lines, fmt.Sprintf("versioning: %s locking: %s", yesNo(b.Versioning), lockingLabel(b))) + } + return lines +} + +// mbPlan is the structured dry-run row of mb. +type mbPlan struct { + Attributes operations.PostStorageBucketsAttributes + DryRun bool +} + +func (p mbPlan) TableRow() table.Row { + a := p.Attributes + op := "make_bucket" + if p.DryRun { + op = "(dryrun) make_bucket" + } + retention := "" + if a.RetentionMode != nil && *a.RetentionMode != operations.RetentionModeNone { + retention = string(*a.RetentionMode) + if a.RetentionPeriod != nil { + retention = fmt.Sprintf("%s (%dd)", retention, *a.RetentionPeriod) + } + } + return table.Row{ + "op": {Label: "Op", Value: op}, + "name": {Label: "Name", Value: a.Name}, + "project": {Label: "Project", Value: a.Project}, + "region": {Label: "Site", Value: a.Region}, + "storage_class": {Label: "Class", Value: string(*a.StorageClass)}, + "versioning": {Label: "Versioning", Value: yesNo(a.Versioning != nil && *a.Versioning)}, + "locking": {Label: "Locking", Value: yesNo(a.Locking != nil && *a.Locking)}, + "retention": {Label: "Retention", Value: retention}, + } +} + +// MarshalJSON emits the plan as a flat document. +func (p mbPlan) MarshalJSON() ([]byte, error) { + a := p.Attributes + doc := map[string]interface{}{ + "op": "make_bucket", + "dry_run": p.DryRun, + "name": a.Name, + "project": a.Project, + "region": a.Region, + "storage_class": string(*a.StorageClass), + "versioning": a.Versioning != nil && *a.Versioning, + "locking": a.Locking != nil && *a.Locking, + } + if a.RetentionMode != nil && *a.RetentionMode != operations.RetentionModeNone { + doc["retention_mode"] = string(*a.RetentionMode) + if a.RetentionPeriod != nil { + doc["retention_days"] = *a.RetentionPeriod + } + } + return json.Marshal(doc) +} + +// pickRegion asks the user for a site when --region is missing, or fails +// with a usage error in non-interactive sessions. +func pickRegion(ctx context.Context, cmd *cobra.Command) (string, error) { + if !objectstorage.CanPrompt(cmd) { + return "", exitcode.Errorf(exitcode.Usage, "--region is required; regions are Latitude sites such as DAL, NYC or TYO4 (run 'lsh regions list')") + } + sites, err := mbListSites(ctx) + if err != nil { + return "", err + } + if len(sites) == 0 { + return "", exitcode.Errorf(exitcode.Usage, "--region is required and no sites were returned by the API (run 'lsh regions list')") + } + options := make([]string, 0, len(sites)) + for _, s := range sites { + options = append(options, s.label()) + } + idx, err := objectstorage.Choose("Which site should host the bucket? (--region)", options, 0) + if err != nil { + return "", objectstorage.Humanize(err, nil, nil) + } + if idx < 0 { + return "", exitcode.Errorf(exitcode.Refused, "no site selected; pass --region ") + } + return sites[idx].Slug, nil +} + +type mbSite struct { + Slug string + Name string +} + +func (s mbSite) label() string { + if s.Name == "" || strings.EqualFold(s.Name, s.Slug) { + return s.Slug + } + return s.Slug + " - " + s.Name +} + +// mbListSites fetches the site slugs from the API (custom storage-only sites +// included), sorted by slug. +func mbListSites(ctx context.Context) ([]mbSite, error) { + client := apiClient() + resp, err := client.Regions.Get(ctx, operations.GetRegionsRequest{ + IncludeCustom: mbPtr(true), + PageSize: mbPtr(int64(100)), + }, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return nil, objectstorage.Humanize(err, nil, nil) + } + var out []mbSite + if resp != nil && resp.Regions != nil { + for _, r := range resp.Regions.Data { + if r.Attributes == nil || r.Attributes.Slug == nil || *r.Attributes.Slug == "" { + continue + } + s := mbSite{Slug: *r.Attributes.Slug} + if r.Attributes.Name != nil { + s.Name = *r.Attributes.Name + } + out = append(out, s) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Slug < out[j].Slug }) + return out, nil +} + +// offerAccessKey implements the post-creation access key offer (J1). Any +// failure here is a warning: the bucket already exists and the command +// succeeds. +func offerAccessKey(ctx context.Context, cmd *cobra.Command, b *objectstorage.Bucket, o mbOptions) { + f := cmd.Flags() + noKey, _ := f.GetBool(flagMbNoAccessKey) + createKey, _ := f.GetBool(flagMbCreateAccessKey) + if noKey || endpointOverride(cmd) != "" { + return + } + if !isHuman() { + // The key would have to be printed to stay usable, and the structured + // document of mb describes a bucket; say so instead of doing nothing. + if createKey { + objectstorage.Warnf("--create-access-key is only offered with human output; create the key with: lsh s3 access-keys create --bucket %s --save", b.Name) + } + return + } + if !createKey && !objectstorage.CanPrompt(cmd) { + return + } + _, profileName, profile, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + objectstorage.Warnf("not saving an access key: %v", err) + return + } + keys := profile.ObjectStorageKeys() + if name, _, ok := objectstorage.SelectKey(keys, profile.DefaultObjectStorageKey(), b, true); ok { + objectstorage.Hintf("Saved access key %q already covers this bucket; the CLI will use it automatically.", name) + return + } + + scope := config.ScopeFullAccess + if !createKey { + allLabel := fmt.Sprintf("key for all %s buckets of project %s", b.StorageClass, b.ProjectRef()) + if b.StorageClass == objectstorage.ClassHighPerformance && b.Site != "" { + allLabel += " in site " + b.Site + } + idx, err := objectstorage.Choose( + "To upload and download files the CLI needs an S3 access key (separate from your API token). Create one now?", + []string{allLabel + " (recommended)", "key for this bucket only (rw)", "skip"}, 0) + if err != nil { + objectstorage.Warnf("could not read the answer: %v", err) + return + } + switch idx { + case 0: + scope = config.ScopeFullAccess + case 1: + scope = config.ScopeLimitedAccess + default: + objectstorage.Hintf("Skipped. Create a key later with: lsh s3 configure (or: lsh s3 access-keys create --bucket %s --save)", b.Name) + return + } + } + + if b.StorageClass == objectstorage.ClassHighPerformance && b.Site == "" { + if err := newResolver(cmd).FillSite(ctx, b); err != nil { + objectstorage.Warnf("could not determine the bucket's site: %v", err) + } + } + // The name is generated the way the dashboard does it (see petname.go); a + // clash with a saved key is handled by saveNewKey when storing. + req := accessKeyRequest{ + Project: firstNonEmptyStr(b.ProjectID, b.ProjectSlug, o.Project), + StorageClass: b.StorageClass, + Site: b.Site, + Scope: scope, + } + var perms map[string]string + switch scope { + case config.ScopeLimitedAccess: + perms = map[string]string{b.ID: config.PermissionRW} + req.Buckets = perms + } + req.Name = generateKeyName(b.StorageClass) + + created, err := createAccessKeyRetrying(ctx, cmd, req, true) + if err != nil { + objectstorage.Warnf("bucket created, but the access key was not: %v", err) + objectstorage.Hintf("Create one later with: lsh s3 access-keys create --bucket %s --save", b.Name) + return + } + stored := created.stored(b.ProjectID, perms, config.KeySourceMakeBkt) + if stored.StorageClass == "" { + stored.StorageClass = b.StorageClass + } + if stored.Site == "" && b.StorageClass == objectstorage.ClassHighPerformance { + stored.Site = b.Site + } + if stored.Scope == "" { + stored.Scope = scope + } + stored.CreatedAt = time.Now().UTC() + name := created.Name + if name == "" { + name = req.Name + } + finalName, savedProfile, err := saveNewKey(cmd, name, stored) + if err != nil { + mbReportUnsavedKey(ctx, os.Stderr, created, req, b, name, err) + return + } + if savedProfile == "" { + savedProfile = profileName + } + objectstorage.Hintf("Saved to %s (0600) as %q in profile %s. The CLI uses it automatically.", mbConfigPathForHint(), finalName, savedProfile) + objectstorage.Hintf("For apps or CI create a separate key: lsh s3 access-keys create --bucket %s", b.Name) +} + +// mbReportUnsavedKey handles a key that was created but could not be stored in +// the profile. mb creates the key on the user's behalf, so the key is removed +// again rather than leaving a credential whose secret would have to be printed; +// the shared helper only prints when the removal fails. +func mbReportUnsavedKey(ctx context.Context, w io.Writer, created *createdAccessKey, req accessKeyRequest, b *objectstorage.Bucket, name string, saveErr error) { + created.Endpoint = firstNonEmptyStr(created.Endpoint, b.Endpoint) + created.SigningRegion = firstNonEmptyStr(created.SigningRegion, b.SigningRegion) + plan := &createPlan{Request: req} + if req.Scope == config.ScopeLimitedAccess { + plan.Buckets = []scopedBucket{{Bucket: b, Permission: config.PermissionRW}} + created.Buckets = map[string]string{b.Name: config.PermissionRW} + } + reportUnsavedKey(ctx, w, req, created, plan, name, false, saveErr) +} + +// mbConfigPathForHint renders the config location for messages. +func mbConfigPathForHint() string { + if p := os.Getenv("LSH_CONFIG_PATH"); p != "" { + return p + } + return "~/.config/lsh/config.json" +} diff --git a/cmd/s3/mb_test.go b/cmd/s3/mb_test.go new file mode 100644 index 0000000..526edd2 --- /dev/null +++ b/cmd/s3/mb_test.go @@ -0,0 +1,379 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cli" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func baseMbOptions() mbOptions { + return mbOptions{Name: "backups", Project: "my-project", Region: "DAL"} +} + +func TestBuildCreateBucketRequest_Defaults(t *testing.T) { + req, err := buildCreateBucketRequest(baseMbOptions()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := req.Data.Attributes + if req.Data.Type != operations.PostStorageBucketsTypeObjects { + t.Errorf("type = %q, want objects", req.Data.Type) + } + if a.Name != "backups" || a.Project != "my-project" || a.Region != "DAL" { + t.Errorf("attributes = %+v", a) + } + if a.StorageClass == nil || *a.StorageClass != operations.StorageClassStandard { + t.Errorf("storage class = %v, want standard", a.StorageClass) + } + if a.Versioning == nil || *a.Versioning { + t.Errorf("versioning should default to false") + } + if a.Locking == nil || *a.Locking { + t.Errorf("locking should default to false") + } + if a.RetentionMode == nil || *a.RetentionMode != operations.RetentionModeNone { + t.Errorf("retention mode = %v, want NONE", a.RetentionMode) + } + if a.RetentionPeriod != nil { + t.Errorf("retention period should be nil, got %d", *a.RetentionPeriod) + } +} + +func TestBuildCreateBucketRequest_RegionUpperCased(t *testing.T) { + o := baseMbOptions() + o.Region = "tyo4" + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if req.Data.Attributes.Region != "TYO4" { + t.Errorf("region = %q, want TYO4", req.Data.Attributes.Region) + } +} + +func TestBuildCreateBucketRequest_RequiredFields(t *testing.T) { + cases := []struct { + name string + mutate func(*mbOptions) + want string + }{ + {"missing name", func(o *mbOptions) { o.Name = "" }, "missing bucket name"}, + {"missing project", func(o *mbOptions) { o.Project = "" }, "--project is required"}, + {"missing region", func(o *mbOptions) { o.Region = "" }, "--region is required"}, + {"bad class", func(o *mbOptions) { o.StorageClass = "glacier" }, "invalid --storage-class"}, + {"bad retention mode", func(o *mbOptions) { o.Locking = true; o.RetentionMode = "FOREVER"; o.RetentionDays = 1 }, "invalid --retention-mode"}, + {"name with slash", func(o *mbOptions) { o.Name = "a/b" }, "cannot contain spaces or slashes"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + o := baseMbOptions() + tc.mutate(&o) + _, err := buildCreateBucketRequest(o) + if err == nil { + t.Fatalf("expected an error") + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("exit code = %d, want %d (usage)", exitcode.Of(err), exitcode.Usage) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error %q does not mention %q", err.Error(), tc.want) + } + }) + } +} + +func TestBuildCreateBucketRequest_AWSRegionDetected(t *testing.T) { + for _, region := range []string{"us-east-1", "eu-central-2", "ap-southeast-1"} { + o := baseMbOptions() + o.Region = region + _, err := buildCreateBucketRequest(o) + if err == nil { + t.Fatalf("%s: expected an error", region) + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("%s: exit code = %d, want usage", region, exitcode.Of(err)) + } + if !strings.Contains(err.Error(), "is not a Latitude site") || !strings.Contains(err.Error(), "DAL") { + t.Errorf("%s: error %q should explain that regions are Latitude sites", region, err.Error()) + } + } + // Latitude slugs with digits are not mistaken for AWS regions. + for _, region := range []string{"DAL", "TYO4", "SAO2", "nyc"} { + o := baseMbOptions() + o.Region = region + if _, err := buildCreateBucketRequest(o); err != nil { + t.Errorf("%s: unexpected error %v", region, err) + } + } +} + +func TestBuildCreateBucketRequest_LockingImpliesVersioning(t *testing.T) { + o := baseMbOptions() + o.Locking = true + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := req.Data.Attributes + if a.Locking == nil || !*a.Locking { + t.Errorf("locking should be true") + } + if a.Versioning == nil || !*a.Versioning { + t.Errorf("versioning should be implied by --locking") + } + + o.VersioningSet = true + o.Versioning = false + _, err = buildCreateBucketRequest(o) + if err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("--locking --versioning=false should be a usage error, got %v", err) + } + if !strings.Contains(err.Error(), "--locking requires versioning") { + t.Errorf("unexpected message %q", err.Error()) + } +} + +func TestBuildCreateBucketRequest_RetentionOnlyWithLocking(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*mbOptions) + }{ + {"mode without locking", func(o *mbOptions) { o.RetentionMode = "GOVERNANCE"; o.RetentionDays = 7 }}, + {"days without locking", func(o *mbOptions) { o.RetentionDays = 7 }}, + } { + t.Run(tc.name, func(t *testing.T) { + o := baseMbOptions() + tc.mutate(&o) + _, err := buildCreateBucketRequest(o) + if err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("expected a usage error, got %v", err) + } + if !strings.Contains(err.Error(), "require --locking") { + t.Errorf("unexpected message %q", err.Error()) + } + }) + } + + // Mode and days must come together. + o := baseMbOptions() + o.Locking = true + o.RetentionMode = "compliance" + if _, err := buildCreateBucketRequest(o); err == nil || !strings.Contains(err.Error(), "requires --retention-days") { + t.Errorf("mode without days should be rejected, got %v", err) + } + o = baseMbOptions() + o.Locking = true + o.RetentionDays = 30 + if _, err := buildCreateBucketRequest(o); err == nil || !strings.Contains(err.Error(), "requires --retention-mode") { + t.Errorf("days without mode should be rejected, got %v", err) + } + o = baseMbOptions() + o.Locking = true + o.RetentionMode = "GOVERNANCE" + o.RetentionDays = -1 + if _, err := buildCreateBucketRequest(o); err == nil || !strings.Contains(err.Error(), "positive") { + t.Errorf("negative days should be rejected, got %v", err) + } +} + +func TestBuildCreateBucketRequest_RetentionMapsToPeriod(t *testing.T) { + o := baseMbOptions() + o.Locking = true + o.RetentionMode = "compliance" + o.RetentionDays = 30 + o.StorageClass = "high_performance" + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := req.Data.Attributes + if a.RetentionMode == nil || *a.RetentionMode != operations.RetentionModeCompliance { + t.Errorf("retention mode = %v, want COMPLIANCE", a.RetentionMode) + } + if a.RetentionPeriod == nil || *a.RetentionPeriod != 30 { + t.Errorf("retention period = %v, want 30", a.RetentionPeriod) + } + if a.StorageClass == nil || *a.StorageClass != operations.StorageClassHighPerformance { + t.Errorf("storage class = %v, want high_performance", a.StorageClass) + } + + body, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, want := range []string{`"retention_period":30`, `"retention_mode":"COMPLIANCE"`, `"locking":true`, `"versioning":true`, `"storage_class":"high_performance"`} { + if !strings.Contains(string(body), want) { + t.Errorf("request body %s lacks %s", body, want) + } + } +} + +func TestDescribeCreateRequest(t *testing.T) { + o := baseMbOptions() + o.Locking = true + o.RetentionMode = "GOVERNANCE" + o.RetentionDays = 7 + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Fatal(err) + } + got := describeCreateRequest(req.Data.Attributes) + for _, want := range []string{"project: my-project", "region: DAL", "class: standard", "versioning: yes", "locking: GOVERNANCE (7d)"} { + if !strings.Contains(got, want) { + t.Errorf("%q lacks %q", got, want) + } + } +} + +func TestMbPlanJSON(t *testing.T) { + req, err := buildCreateBucketRequest(baseMbOptions()) + if err != nil { + t.Fatal(err) + } + body, err := json.Marshal(mbPlan{Attributes: req.Data.Attributes, DryRun: true}) + if err != nil { + t.Fatal(err) + } + var doc map[string]interface{} + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + if doc["op"] != "make_bucket" || doc["dry_run"] != true || doc["name"] != "backups" || doc["region"] != "DAL" { + t.Errorf("plan = %s", body) + } + if _, has := doc["retention_mode"]; has { + t.Errorf("retention_mode should be omitted when NONE: %s", body) + } +} + +// Default key names and the clash policy now come from the shared helpers +// (defaultKeyName in accesskeys_shared.go, saveNewKey in s3.go), which are +// covered by their own tests; the mb-specific copies were removed. + +func TestBuildCreateBucketRequest_StorageClassAliases(t *testing.T) { + cases := map[string]operations.StorageClass{ + "": operations.StorageClassStandard, + "standard": operations.StorageClassStandard, + "std": operations.StorageClassStandard, + "high_performance": operations.StorageClassHighPerformance, + "high-performance": operations.StorageClassHighPerformance, + "HP": operations.StorageClassHighPerformance, + } + for in, want := range cases { + o := baseMbOptions() + o.StorageClass = in + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Errorf("%q: unexpected error %v", in, err) + continue + } + if got := *req.Data.Attributes.StorageClass; got != want { + t.Errorf("%q: storage class = %q, want %q", in, got, want) + } + } +} + +func TestNewMbCmd_ProjectFlagRequiresRootPicker(t *testing.T) { + cmd := NewMbCmd() + if cmd.Flags().Lookup(flagProject) == nil { + t.Fatalf("mb must register --project") + } + // mb needs exactly one project, so --project is NOT marked optional: the + // root pre-run resolves it the same way as every other command (LSH_PROJECT, + // then the interactive project picker in a terminal, else an error). + if cmd.Annotations[cli.ProjectOptionalAnnotation] == "true" { + t.Errorf("mb must not carry %s: the root pre-run should own the project prompt", cli.ProjectOptionalAnnotation) + } + // buildCreateBucketRequest still guards against an empty project as a safety net. + o := baseMbOptions() + o.Project = "" + _, err := buildCreateBucketRequest(o) + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--project is required") { + t.Errorf("missing project must stay a usage error, got %v", err) + } +} + +// mbUnsavedKeyFixture builds the state of a key that was created for a new +// bucket but could not be stored in the profile. +func mbUnsavedKeyFixture() (*createdAccessKey, accessKeyRequest, *objectstorage.Bucket) { + created := &createdAccessKey{ + Name: "lsh-me-backups", AccessKeyID: "AKIAEXAMPLE", SecretAccessKey: "s3cr3t/value", + Username: "me+lsh-me-backups@latitude.sh", + StorageClass: "standard", Project: "my-project", Scope: config.ScopeLimitedAccess, + } + req := accessKeyRequest{Project: "my-project", StorageClass: "standard", Name: "lsh-me-backups", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": config.PermissionRW}} + b := &objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a", Endpoint: "https://s3.us-central-1.storage.sh", SigningRegion: "us-central-1"} + return created, req, b +} + +// stubDiscard replaces the API deletion for the duration of a test. +func stubDiscard(t *testing.T, err error) *int { + t.Helper() + calls := 0 + saved := discardKey + discardKey = func(context.Context, accessKeyRequest, *createdAccessKey) error { + calls++ + return err + } + t.Cleanup(func() { discardKey = saved }) + return &calls +} + +// TestMbReportUnsavedKey_RemovesInsteadOfPrinting covers the preferred +// recovery: a key that cannot be saved is deleted again, so no one-time secret +// is printed and no credential is left live on the API. +func TestMbReportUnsavedKey_RemovesInsteadOfPrinting(t *testing.T) { + created, req, b := mbUnsavedKeyFixture() + calls := stubDiscard(t, nil) + + var out bytes.Buffer + mbReportUnsavedKey(context.Background(), &out, created, req, b, "lsh-me-backups", errors.New("read-only file system")) + text := out.String() + if *calls != 1 { + t.Errorf("the key must be deleted exactly once, got %d calls", *calls) + } + if strings.Contains(text, "s3cr3t/value") { + t.Errorf("no secret may be printed when the key was removed:\n%s", text) + } + for _, want := range []string{"could not be saved", "read-only file system", "It was deleted again", "Retry once the profile is writable"} { + if !strings.Contains(text, want) { + t.Errorf("report lacks %q:\n%s", want, text) + } + } +} + +// TestMbReportUnsavedKey_PrintsSecretOnceWhenRemovalFails covers the fallback: +// the key is live and unusable unless its secret is shown, so it is printed — +// exactly once, with the command that stores it. +func TestMbReportUnsavedKey_PrintsSecretOnceWhenRemovalFails(t *testing.T) { + created, req, b := mbUnsavedKeyFixture() + stubDiscard(t, errors.New("403 forbidden")) + + var out bytes.Buffer + mbReportUnsavedKey(context.Background(), &out, created, req, b, "lsh-me-backups", errors.New("read-only file system")) + text := out.String() + for _, want := range []string{ + "could not be saved", "read-only file system", "nor deleted again", + "Access Key ID:", "AKIAEXAMPLE", + "Secret Access Key:", "s3cr3t/value", + "https://s3.us-central-1.storage.sh", "us-central-1", + "backups-7f3a (rw)", + "lsh s3 access-keys import --name lsh-me-backups --access-key-id AKIAEXAMPLE --project my-project --storage-class standard --bucket backups=rw", + } { + if !strings.Contains(text, want) { + t.Errorf("report lacks %q:\n%s", want, text) + } + } + if strings.Count(text, "s3cr3t/value") != 1 { + t.Errorf("the secret must be printed exactly once:\n%s", text) + } +} diff --git a/cmd/s3/metrics.go b/cmd/s3/metrics.go new file mode 100644 index 0000000..1cd5f7c --- /dev/null +++ b/cmd/s3/metrics.go @@ -0,0 +1,370 @@ +package s3 + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/spf13/cobra" +) + +// optAllProjects is --all-projects (same spelling as ls). +const optAllProjects = "all-projects" + +// metricsConcurrency bounds the parallel GetStorageBucketMetrics calls. +const metricsConcurrency = 8 + +// MetricsRow is one bucket's consumption and estimated cost for the current +// billing period, as returned by GET /storage/buckets/{id}/metrics. +type MetricsRow struct { + Bucket string `json:"bucket"` + ID string `json:"id"` + Project string `json:"project,omitempty"` + StorageClass string `json:"storage_class,omitempty"` + Site string `json:"site,omitempty"` + CurrentGB *int64 `json:"current_gb"` + ConsumedGB *int64 `json:"consumed_gb"` + Unit string `json:"unit,omitempty"` + CostAmount *float64 `json:"estimated_cost"` + Currency string `json:"currency,omitempty"` + PeriodStart *time.Time `json:"period_start,omitempty"` + PeriodEnd *time.Time `json:"period_end,omitempty"` + // Total marks the aggregated row appended in table mode. + Total bool `json:"-"` + // costs is the per-currency sum of a TOTAL row (several currencies may mix). + costs map[string]float64 +} + +// newMetricsRow converts an API payload for bucket b. +func newMetricsRow(b *objectstorage.Bucket, site string, data *operations.GetStorageBucketMetricsData) MetricsRow { + row := MetricsRow{Bucket: b.Name, ID: b.ID, Project: b.ProjectRef(), StorageClass: b.StorageClass, Site: site} + if row.Site == "" { + row.Site = b.Site + } + if data == nil || data.Attributes == nil { + return row + } + a := data.Attributes + if a.Storage != nil { + row.CurrentGB = a.Storage.Current + row.ConsumedGB = a.Storage.Consumed + if a.Storage.Unit != nil { + row.Unit = string(*a.Storage.Unit) + } + } + if a.EstimatedCost != nil { + row.CostAmount = a.EstimatedCost.Amount + if a.EstimatedCost.Currency != nil { + row.Currency = *a.EstimatedCost.Currency + } + } + if a.Period != nil { + row.PeriodStart = a.Period.Start + row.PeriodEnd = a.Period.End + } + return row +} + +func (m MetricsRow) TableRow() table.Row { + return table.Row{ + "id": {Label: "ID", Value: m.ID}, + "bucket": {Label: "Bucket", Value: m.Bucket}, + "project": {Label: "Project", Value: m.Project}, + "storage_class": {Label: "Class", Value: m.StorageClass}, + "region": {Label: "Site", Value: m.Site}, + "current_gb": {Label: "Current (GB)", Value: gbLabel(m.CurrentGB)}, + "consumed_gb": {Label: "Consumed (GB)", Value: gbLabel(m.ConsumedGB)}, + "estimated_cost": {Label: "Est. Cost", Value: m.costLabel()}, + "period_start": {Label: "Period Start", Value: dateLabel(m.PeriodStart)}, + "period_end": {Label: "Period End", Value: dateLabel(m.PeriodEnd)}, + } +} + +// costLabel renders "12.30 USD" (or the per-currency sums of a TOTAL row). +func (m MetricsRow) costLabel() string { + if m.Total && len(m.costs) > 0 { + currencies := make([]string, 0, len(m.costs)) + for c := range m.costs { + currencies = append(currencies, c) + } + sort.Strings(currencies) + parts := make([]string, 0, len(currencies)) + for _, c := range currencies { + parts = append(parts, formatCost(m.costs[c], c)) + } + return strings.Join(parts, " + ") + } + if m.CostAmount == nil { + return emptyCell + } + return formatCost(*m.CostAmount, m.Currency) +} + +func formatCost(amount float64, currency string) string { + return strings.TrimSpace(fmt.Sprintf("%.2f %s", amount, currency)) +} + +// gbLabel renders the GB figure, or the shared table placeholder when unset. +func gbLabel(v *int64) string { + if v == nil { + return emptyCell + } + return fmt.Sprintf("%d", *v) +} + +func dateLabel(t *time.Time) string { + if t == nil || t.IsZero() { + return "" + } + // The API reports billing-period boundaries in UTC; rendering them in the + // local zone shifts them a day west of Greenwich. + return t.UTC().Format("2006-01-02") +} + +// costOf returns the estimated cost or 0 when unknown. +func (m MetricsRow) costOf() float64 { + if m.CostAmount == nil { + return 0 + } + return *m.CostAmount +} + +// sortMetricsRows orders rows by estimated cost (desc), then consumed GB +// (desc), then bucket name so the output is stable. +func sortMetricsRows(rows []MetricsRow) { + sort.SliceStable(rows, func(i, j int) bool { + ci, cj := rows[i].costOf(), rows[j].costOf() + if ci != cj { + return ci > cj + } + gi, gj := int64Of(rows[i].ConsumedGB), int64Of(rows[j].ConsumedGB) + if gi != gj { + return gi > gj + } + return rows[i].Bucket < rows[j].Bucket + }) +} + +func int64Of(v *int64) int64 { + if v == nil { + return 0 + } + return *v +} + +// metricsTotal sums current/consumed GB and the cost per currency across rows. +// Period columns are filled when every row shares the same period. +func metricsTotal(rows []MetricsRow) MetricsRow { + total := MetricsRow{Bucket: "TOTAL", Total: true, costs: map[string]float64{}} + var current, consumed int64 + var haveCurrent, haveConsumed bool + var currency string + var samePeriod = true + for i, r := range rows { + if r.CurrentGB != nil { + current += *r.CurrentGB + haveCurrent = true + } + if r.ConsumedGB != nil { + consumed += *r.ConsumedGB + haveConsumed = true + } + if r.CostAmount != nil { + total.costs[r.Currency] += *r.CostAmount + } + if r.Unit != "" && total.Unit == "" { + total.Unit = r.Unit + } + if i == 0 { + total.PeriodStart, total.PeriodEnd = r.PeriodStart, r.PeriodEnd + } else if !sameTime(total.PeriodStart, r.PeriodStart) || !sameTime(total.PeriodEnd, r.PeriodEnd) { + samePeriod = false + } + } + if haveCurrent { + total.CurrentGB = ¤t + } + if haveConsumed { + total.ConsumedGB = &consumed + } + if len(total.costs) == 1 { + for c, amount := range total.costs { + currency = c + a := amount + total.CostAmount = &a + } + total.Currency = currency + } + if !samePeriod { + total.PeriodStart, total.PeriodEnd = nil, nil + } + return total +} + +func sameTime(a, b *time.Time) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return a.Equal(*b) +} + +// metricsResult pairs a bucket with its metrics or the error fetching them. +type metricsResult struct { + Bucket *objectstorage.Bucket + Row MetricsRow + Err error +} + +// fetchMetrics calls GetStorageBucketMetrics for every bucket with bounded +// concurrency. Per-bucket failures do not stop the others; results keep the +// input order. +func fetchMetrics(ctx context.Context, api *sdk.Latitudesh, buckets []*objectstorage.Bucket, sites map[string]string, opts []operations.Option) []metricsResult { + results := make([]metricsResult, len(buckets)) + sem := make(chan struct{}, metricsConcurrency) + var wg sync.WaitGroup + for i, b := range buckets { + wg.Add(1) + go func(i int, b *objectstorage.Bucket) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + res := metricsResult{Bucket: b} + resp, err := api.ObjectStorage.GetStorageBucketMetrics(ctx, b.ID, opts...) + if err != nil { + res.Err = objectstorage.HumanizeAPI(err, fmt.Sprintf("metrics for bucket %s", b.Display())) + } else { + var data *operations.GetStorageBucketMetricsData + if resp.Object != nil { + data = resp.Object.Data + } + res.Row = newMetricsRow(b, sites[b.ID], data) + } + results[i] = res + }(i, b) + } + wg.Wait() + return results +} + +// NewMetricsCmd builds `lsh s3 metrics [s3://bucket]`. +func NewMetricsCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "metrics [s3://bucket]", + Short: "Storage consumption and estimated cost", + Long: `Show the storage consumed in the current billing period and the estimated +cost, per bucket. Without a bucket every bucket of the project (or of the team +with --all-projects) is listed, sorted by estimated cost, with a TOTAL row in +table output. Structured output (-o json|yaml|csv) stays one record per bucket.`, + Example: ` lsh s3 metrics s3://backups + lsh s3 metrics --project my-project + lsh s3 metrics --all-projects -o json --query '[].{bucket:bucket,cost:estimated_cost}'`, + Args: cobra.MaximumNArgs(1), + RunE: runMetrics, + }) + addProjectFlag(cmd, true, "only buckets of this project (ID or slug)") + addBucketFilterFlags(cmd) + cmd.Flags().Bool(optAllProjects, false, "buckets of every project of the team") + return cmd +} + +func runMetrics(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "metrics come from the Latitude API; unset --endpoint-url / %s to use this command", objectstorage.EnvEndpointURL)) + } + allProjects, _ := cmd.Flags().GetBool(optAllProjects) + if allProjects && cmd.Flags().Changed(flagProject) { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s and --%s are mutually exclusive", flagProject, optAllProjects)) + } + + var buckets []*objectstorage.Bucket + var sites map[string]string + if len(args) == 1 { + ref, err := objectstorage.ParseBucketOnly(args[0]) + if err != nil { + return printErr(err) + } + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + if s, err := objectstorage.RawBucketSites(ctx, b.ID); err == nil { + sites = s + } else { + lsh.LogDebugf("metrics: could not fetch site for %s: %v", b.ID, err) + } + buckets = []*objectstorage.Bucket{b} + } else { + r := newResolver(cmd) + if allProjects { + r.Project = "" + } + list, err := r.ListBuckets(ctx) + if err != nil { + return printErr(err) + } + // Scope the site lookup to the project when one is known; only + // --all-projects needs the team-wide listing. + if s, err := objectstorage.RawBucketSitesForProject(ctx, "", r.Project); err == nil { + sites = s + } else { + lsh.LogDebugf("metrics: could not fetch bucket sites: %v", err) + } + // --storage-class/--site also narrow the multi-bucket listing, not just + // a single ambiguous name. + list = filterBucketsByClass(list, r.ClassFilter) + for _, d := range list { + b := objectstorage.BucketFromData(d) + if r.SiteFilter != "" { + site := b.Site + if site == "" && d.ID != nil { + site = sites[*d.ID] + } + if !strings.EqualFold(site, r.SiteFilter) { + continue + } + } + buckets = append(buckets, b) + } + } + + results := fetchMetrics(ctx, apiClient(), buckets, sites, []operations.Option{operations.WithRetries(lsh.RetryConfig())}) + rows := make([]MetricsRow, 0, len(results)) + var failed int + var firstErr error + for _, res := range results { + if res.Err != nil { + failed++ + if firstErr == nil { + firstErr = res.Err + } + objectstorage.Warnf("metrics unavailable for %s: %v", res.Bucket.Display(), res.Err) + continue + } + rows = append(rows, res.Row) + } + if failed > 0 && len(rows) == 0 { + return printErr(firstErr) + } + + sortMetricsRows(rows) + if isHuman() && len(rows) > 1 { + rows = append(rows, metricsTotal(rows)) + } + render(objectstorage.AsResponseData(rows)) + if failed > 0 { + return printErr(exitcode.Errorf(exitcode.Partial, "metrics unavailable for %d of %d buckets", failed, len(results))) + } + return nil +} diff --git a/cmd/s3/metrics_test.go b/cmd/s3/metrics_test.go new file mode 100644 index 0000000..f2b33fb --- /dev/null +++ b/cmd/s3/metrics_test.go @@ -0,0 +1,249 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func ptrF(f float64) *float64 { return &f } + +func metricsFixture() []MetricsRow { + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2026, 9, 30, 23, 59, 59, 0, time.UTC) + return []MetricsRow{ + {Bucket: "cheap", ID: "bkt_1", CurrentGB: ptrI(10), ConsumedGB: ptrI(12), Unit: "GB", CostAmount: ptrF(1.5), Currency: "USD", PeriodStart: &start, PeriodEnd: &end}, + {Bucket: "big", ID: "bkt_2", CurrentGB: ptrI(500), ConsumedGB: ptrI(480), Unit: "GB", CostAmount: ptrF(30), Currency: "USD", PeriodStart: &start, PeriodEnd: &end}, + {Bucket: "free", ID: "bkt_3", CurrentGB: ptrI(0), ConsumedGB: ptrI(0), Unit: "GB", CostAmount: ptrF(0), Currency: "USD", PeriodStart: &start, PeriodEnd: &end}, + {Bucket: "brl", ID: "bkt_4", CurrentGB: ptrI(100), ConsumedGB: ptrI(90), Unit: "GB", CostAmount: ptrF(30), Currency: "BRL", PeriodStart: &start, PeriodEnd: &end}, + {Bucket: "unknown", ID: "bkt_5"}, // metrics payload without attributes + } +} + +func TestSortMetricsRows(t *testing.T) { + rows := metricsFixture() + sortMetricsRows(rows) + var got []string + for _, r := range rows { + got = append(got, r.Bucket) + } + // Cost desc; ties (30 USD vs 30 BRL) by consumed desc; zero-cost rows by + // consumed, then name. + want := []string{"big", "brl", "cheap", "free", "unknown"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("order = %v, want %v", got, want) + } +} + +func TestMetricsTotal(t *testing.T) { + rows := metricsFixture() + total := metricsTotal(rows) + if total.Bucket != "TOTAL" || !total.Total { + t.Errorf("total row not marked: %+v", total) + } + if total.CurrentGB == nil || *total.CurrentGB != 610 { + t.Errorf("current_gb = %v, want 610", total.CurrentGB) + } + if total.ConsumedGB == nil || *total.ConsumedGB != 582 { + t.Errorf("consumed_gb = %v, want 582", total.ConsumedGB) + } + if total.costs["USD"] != 31.5 || total.costs["BRL"] != 30 { + t.Errorf("costs = %v", total.costs) + } + // Mixed currencies: no single amount, label lists both (sorted). + if total.CostAmount != nil { + t.Errorf("mixed currencies must not collapse into one amount: %v", *total.CostAmount) + } + if got := total.TableRow()["estimated_cost"].Value; got != "30.00 BRL + 31.50 USD" { + t.Errorf("cost label = %q", got) + } + // Same period on every row (rows without a period do not count). + if total.PeriodStart != nil && total.TableRow()["period_start"].Value == "" { + t.Errorf("period should be kept when uniform") + } + + // Single currency: amount and currency are filled. + single := metricsTotal(rows[:3]) + if single.CostAmount == nil || *single.CostAmount != 31.5 || single.Currency != "USD" { + t.Errorf("single-currency total = %+v", single) + } + if got := single.TableRow()["estimated_cost"].Value; got != "31.50 USD" { + t.Errorf("single-currency label = %q", got) + } + if single.TableRow()["period_start"].Value == "" || single.TableRow()["period_end"].Value == "" { + t.Errorf("uniform period should be shown on the TOTAL row") + } + + // Rows with no metrics at all yield a total without numbers. + empty := metricsTotal([]MetricsRow{{Bucket: "x"}, {Bucket: "y"}}) + if empty.CurrentGB != nil || empty.ConsumedGB != nil || empty.CostAmount != nil { + t.Errorf("empty total should have nil numbers: %+v", empty) + } + // Empty numbers render the shared "-" placeholder used by every s3 table. + if got := empty.TableRow()["estimated_cost"].Value; got != emptyCell { + t.Errorf("empty cost label = %q, want %q", got, emptyCell) + } + if got := empty.TableRow()["current_gb"].Value; got != emptyCell { + t.Errorf("empty current_gb label = %q, want %q", got, emptyCell) + } +} + +func TestMetricsRowJSONShape(t *testing.T) { + rows := metricsFixture() + raw, err := json.Marshal(rows[0]) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"bucket":"cheap"`, `"id":"bkt_1"`, `"current_gb":10`, `"consumed_gb":12`, `"estimated_cost":1.5`, `"currency":"USD"`, `"period_start":`} { + if !strings.Contains(string(raw), want) { + t.Errorf("json %s lacks %s", raw, want) + } + } + // The TOTAL marker never leaks into structured output. + total := metricsTotal(rows) + raw, _ = json.Marshal(total) + if strings.Contains(string(raw), "Total") || strings.Contains(string(raw), "costs") { + t.Errorf("total internals leaked: %s", raw) + } +} + +// fakeMetricsAPI serves GET /storage/buckets/{id}/metrics; bkt_err fails +// with 500 and bkt_missing with 404. It records the peak concurrency. +func fakeMetricsAPI(t *testing.T, peak *int32) *httptest.Server { + var inflight int32 + var mu sync.Mutex + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cur := atomic.AddInt32(&inflight, 1) + defer atomic.AddInt32(&inflight, -1) + mu.Lock() + if cur > *peak { + *peak = cur + } + mu.Unlock() + time.Sleep(20 * time.Millisecond) + + if r.Header.Get("Authorization") == "" { + t.Errorf("missing Authorization header") + } + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(parts) != 4 || parts[0] != "storage" || parts[1] != "buckets" || parts[3] != "metrics" { + http.Error(w, "unexpected path "+r.URL.Path, http.StatusNotFound) + return + } + id := parts[2] + w.Header().Set("Content-Type", "application/vnd.api+json") + switch id { + case "bkt_err": + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"errors":[{"status":"500","title":"Internal Server Error","detail":"boom"}]}`) + return + case "bkt_missing": + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, `{"errors":[{"status":"404","title":"Not Found","detail":"Couldn't find bucket"}]}`) + return + } + n := strings.TrimPrefix(id, "bkt_") + fmt.Fprintf(w, `{"data":{"id":%q,"type":"object_storage_metrics","attributes":{"period":{"start":"2026-09-01T00:00:00Z","end":"2026-09-30T23:59:59Z"},"storage":{"consumed":%s0,"current":%s,"unit":"GB"},"estimated_cost":{"amount":%s.5,"currency":"USD"}}}}`, id, n, n, n) + })) +} + +func TestFetchMetricsFanOut(t *testing.T) { + var peak int32 + srv := fakeMetricsAPI(t, &peak) + defer srv.Close() + + api := sdk.New(sdk.WithServerURL(srv.URL), sdk.WithSecurity("test-token")) + var buckets []*objectstorage.Bucket + for i := 1; i <= 20; i++ { + buckets = append(buckets, &objectstorage.Bucket{ID: fmt.Sprintf("bkt_%d", i), Name: fmt.Sprintf("b%d", i), StorageClass: "standard", ProjectSlug: "proj"}) + } + buckets = append(buckets, + &objectstorage.Bucket{ID: "bkt_err", Name: "broken"}, + &objectstorage.Bucket{ID: "bkt_missing", Name: "gone"}, + ) + sites := map[string]string{"bkt_1": "DAL", "bkt_2": "TYO4"} + + results := fetchMetrics(context.Background(), api, buckets, sites, nil) + if len(results) != len(buckets) { + t.Fatalf("got %d results, want %d", len(results), len(buckets)) + } + if peak > metricsConcurrency { + t.Errorf("peak concurrency %d exceeds %d", peak, metricsConcurrency) + } + if peak < 2 { + t.Errorf("expected parallel requests, peak was %d", peak) + } + + var failed int + for i, res := range results { + if res.Bucket != buckets[i] { + t.Errorf("result %d out of order: %s", i, res.Bucket.ID) + } + switch res.Bucket.ID { + case "bkt_err": + failed++ + if res.Err == nil || exitcode.Of(res.Err) != exitcode.Generic { + t.Errorf("bkt_err: err = %v", res.Err) + } + case "bkt_missing": + failed++ + if res.Err == nil || exitcode.Of(res.Err) != exitcode.NotFound { + t.Errorf("bkt_missing: err = %v (exit %d)", res.Err, exitcode.Of(res.Err)) + } else if !strings.Contains(res.Err.Error(), "gone (bkt_missing)") || !strings.Contains(res.Err.Error(), "Couldn't find bucket") { + t.Errorf("bkt_missing: message should name the bucket and carry the API detail: %v", res.Err) + } + default: + if res.Err != nil { + t.Errorf("%s: unexpected error %v", res.Bucket.ID, res.Err) + continue + } + n := strings.TrimPrefix(res.Bucket.ID, "bkt_") + if res.Row.Bucket != res.Bucket.Name || res.Row.ID != res.Bucket.ID || res.Row.Project != "proj" { + t.Errorf("row identity wrong: %+v", res.Row) + } + if res.Row.CurrentGB == nil || fmt.Sprint(*res.Row.CurrentGB) != n { + t.Errorf("%s current_gb = %v", res.Bucket.ID, res.Row.CurrentGB) + } + if res.Row.CostAmount == nil || res.Row.Currency != "USD" { + t.Errorf("%s cost = %v %s", res.Bucket.ID, res.Row.CostAmount, res.Row.Currency) + } + if res.Row.PeriodStart == nil || res.Row.PeriodEnd == nil { + t.Errorf("%s period missing", res.Bucket.ID) + } + } + } + if failed != 2 { + t.Errorf("failed = %d, want 2", failed) + } + if results[0].Row.Site != "DAL" || results[1].Row.Site != "TYO4" || results[2].Row.Site != "" { + t.Errorf("site mapping wrong: %q %q %q", results[0].Row.Site, results[1].Row.Site, results[2].Row.Site) + } +} + +// TestDateLabelIsUTC covers the billing-period boundary fix: the API reports +// them in UTC, so rendering them in the local zone moved them a day west of +// Greenwich (UTC-3 showed the previous month's last day). +func TestDateLabelIsUTC(t *testing.T) { + restore := time.Local + time.Local = time.FixedZone("UTC-3", -3*60*60) + defer func() { time.Local = restore }() + + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + if got := dateLabel(&start); got != "2026-09-01" { + t.Errorf("dateLabel = %q, want 2026-09-01 (the UTC date the API reported)", got) + } + if got := dateLabel(nil); got != "" { + t.Errorf("dateLabel(nil) = %q, want empty", got) + } +} diff --git a/cmd/s3/mv.go b/cmd/s3/mv.go new file mode 100644 index 0000000..a5f2e7a --- /dev/null +++ b/cmd/s3/mv.go @@ -0,0 +1,36 @@ +package s3 + +import ( + "github.com/spf13/cobra" +) + +// NewMvCmd builds `lsh s3 mv `. +func NewMvCmd() *cobra.Command { + var f transferFlags + cmd := newCmd(&cobra.Command{ + Use: "mv ", + Aliases: []string{"move"}, + Short: "Move files and objects", + Long: `Move a local file into a bucket, an object to the local disk, or an object to +another key (server-side, same endpoint; buckets that resolve different access +keys are streamed through this machine). Each source is deleted only after +its copy succeeded; a failed copy leaves the source untouched. Both operands +local, or source and destination naming the same object, is refused. + +` + destinationRulesHelp + ` + +All cp flags apply (--recursive, --exclude/--include, --content-type, +--metadata, --no-overwrite, --quiet, ...). Output lines use the "move:" +prefix. --dry-run (or --dryrun) prints the plan without copying or deleting.`, + Example: ` lsh s3 mv ./dump.sql s3://backups/2026/09/ + lsh s3 mv s3://backups/tmp/report.pdf ./reports/ + lsh s3 mv s3://backups/tmp/ s3://backups/archive/2026/ --recursive + lsh s3 mv ./logs s3://logs/host-1/ --recursive --exclude "*" --include "*.log" --dryrun`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return runCopyCommand(cmd, args, &f, true) + }, + }) + addTransferFlags(cmd, &f, true) + return cmd +} diff --git a/cmd/s3/petname.go b/cmd/s3/petname.go new file mode 100644 index 0000000..fec0047 --- /dev/null +++ b/cmd/s3/petname.go @@ -0,0 +1,56 @@ +package s3 + +import ( + "crypto/rand" + "math/big" + + "github.com/latitudesh/lsh/internal/objectstorage" +) + +// Access key names follow the same "pet name" pattern the dashboard uses when +// it auto-suggests a name: key---, e.g. key-brave-otter-std. +// There is no random numeric segment; uniqueness is enforced per project, so a +// candidate that collides is simply re-rolled (see createAccessKeyRetrying). +// The names satisfy the API rules (lowercase, 3-25 chars, start/end alphanumeric). + +var petAdjectives = []string{ + "amber", "azure", "bold", "brave", "brisk", "calm", "clever", "cosmic", + "crimson", "daring", "deep", "eager", "electric", "gentle", "golden", "happy", + "jolly", "keen", "lively", "lucky", "mellow", "merry", "mighty", "noble", + "plum", "proud", "quiet", "rapid", "royal", "shiny", "silent", "sleek", + "smart", "snappy", "solar", "spry", "stellar", "sturdy", "sunny", "swift", + "teal", "tidy", "vivid", "warm", "wise", "witty", "young", "zesty", +} + +var petNouns = []string{ + "otter", "falcon", "harbor", "forest", "river", "meadow", "comet", "canyon", + "maple", "willow", "cedar", "cobra", "lynx", "heron", "raven", "badger", + "marlin", "puma", "bison", "koala", "gecko", "panda", "tiger", "walrus", + "zephyr", "summit", "delta", "ember", "glacier", "harvest", "island", "jungle", + "lagoon", "monsoon", "nebula", "orchid", "pebble", "quartz", "ridge", "sequoia", + "tundra", "vortex", "wave", "beacon", "boulder", "cascade", "dune", "fjord", +} + +// tierAbbr maps a storage class to the dashboard's short suffix. +func tierAbbr(storageClass string) string { + if storageClass == objectstorage.ClassHighPerformance { + return "hp" + } + return "std" +} + +// generateKeyName returns a fresh pet name for the given storage class, capped +// at the API's 25-character limit. +func generateKeyName(storageClass string) string { + return capKeyName("key-" + pick(petAdjectives) + "-" + pick(petNouns) + "-" + tierAbbr(storageClass)) +} + +// pick returns a uniformly random element of list using crypto/rand, falling +// back to the first element only if the source ever fails. +func pick(list []string) string { + n, err := rand.Int(rand.Reader, big.NewInt(int64(len(list)))) + if err != nil { + return list[0] + } + return list[n.Int64()] +} diff --git a/cmd/s3/presign.go b/cmd/s3/presign.go new file mode 100644 index 0000000..43d66c3 --- /dev/null +++ b/cmd/s3/presign.go @@ -0,0 +1,219 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +// presignMaxExpiry is the SigV4 ceiling for pre-signed URLs (7 days). +const presignMaxExpiry = 7 * 24 * time.Hour + +// presignOptions are the parsed flags of `lsh s3 presign`. +type presignOptions struct { + Method string + Expires time.Duration + VersionID string + ContentType string +} + +// NewPresignCmd builds `lsh s3 presign s3://bucket/key`. +func NewPresignCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "presign s3://bucket/key", + Aliases: []string{"presigned-url", "share"}, + Short: "Generate a pre-signed URL", + Long: `Generate a pre-signed URL that grants temporary access to one object +without sharing credentials. + +The URL is signed locally with the access key the CLI selects for the bucket +(the only network call is the bucket lookup). --expires-in accepts seconds +(3600) or a duration with an s/m/h/d suffix (15m, 1h, 7d); the maximum is 7 +days (604800 seconds). --method PUT signs an upload URL and requires a key +with write permission. In the default output only the URL is printed, so the +command composes with $(...) and curl.`, + Example: ` lsh s3 presign s3://backups/report.pdf + lsh s3 presign s3://backups/report.pdf --expires-in 15m + curl -o report.pdf "$(lsh s3 presign s3://backups/report.pdf)" + lsh s3 presign s3://uploads/in.bin --method PUT --expires-in 1h + lsh s3 presign s3://backups/report.pdf -o json`, + Args: cobra.ExactArgs(1), + }) + f := cmd.Flags() + f.String("expires-in", "3600", "how long the URL stays valid: seconds or a duration (15m, 1h, 7d); max 7d") + f.String("method", "GET", "HTTP method to sign: GET, PUT or HEAD") + f.String("version-id", "", "sign a URL for this specific version (GET/HEAD only)") + f.String("content-type", "", "content type the uploader will send (PUT only; informational, see notes)") + addProjectFlag(cmd, true, "project the bucket belongs to (disambiguates buckets with the same name)") + addBucketFilterFlags(cmd) + rejectRegionFlag(cmd) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + expiresIn, _ := f.GetString("expires-in") + method, _ := f.GetString("method") + versionID, _ := f.GetString("version-id") + contentType, _ := f.GetString("content-type") + + // ObjectRef's shared message points at --recursive, a flag only rm and + // the transfer commands have; presign signs exactly one object. + if r, parseErr := objectstorage.ParseRemote(args[0]); parseErr == nil && strings.HasSuffix(r.Key, "/") { + return printErr(objectstorage.ErrUsagef("key %q ends with '/'; presign signs a single object (use 'lsh s3 ls %s' to list the prefix)", r.Key, r)) + } + ref, err := objectstorage.ObjectRef(args[0], false) + if err != nil { + return printErr(err) + } + opts, err := presignParseOptions(method, expiresIn, versionID, contentType) + if err != nil { + return printErr(err) + } + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + sess, err := openBucket(ctx, cmd, ref.Bucket, opts.Method == http.MethodPut) + if err != nil { + return printErr(err) + } + res, err := presignObject(ctx, sess, ref.Key, opts) + if err != nil { + return printErr(err) + } + if isHuman() { + presignWrite(os.Stdout, res) + return nil + } + render([]renderer.ResponseData{res}) + return nil + } + return cmd +} + +// presignParseOptions validates the flag values. +func presignParseOptions(method, expiresIn, versionID, contentType string) (presignOptions, error) { + opts := presignOptions{VersionID: versionID, ContentType: contentType} + opts.Method = strings.ToUpper(strings.TrimSpace(method)) + switch opts.Method { + case http.MethodGet, http.MethodPut, http.MethodHead: + case "": + opts.Method = http.MethodGet + default: + return opts, objectstorage.ErrUsagef("--method must be GET, PUT or HEAD (got %q)", method) + } + d, err := parseExpiresIn(expiresIn) + if err != nil { + return opts, err + } + opts.Expires = d + if opts.VersionID != "" && opts.Method == http.MethodPut { + return opts, objectstorage.ErrUsagef("--version-id only applies to GET and HEAD URLs") + } + if opts.ContentType != "" && opts.Method != http.MethodPut { + return opts, objectstorage.ErrUsagef("--content-type only applies to --method PUT") + } + return opts, nil +} + +var expiresInRe = regexp.MustCompile(`^(\d+)\s*([smhd])$`) + +// parseExpiresIn accepts a plain integer (seconds) or a duration with an +// s/m/h/d suffix (15m, 1h, 7d; composite Go durations such as 1h30m also +// work). Values above 7 days are a usage error rather than being clipped. +func parseExpiresIn(s string) (time.Duration, error) { + v := strings.TrimSpace(s) + if v == "" { + return 0, objectstorage.ErrUsagef("--expires-in cannot be empty") + } + var d time.Duration + switch { + case isDigits(v): + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return 0, objectstorage.ErrUsagef("invalid --expires-in %q: %v", s, err) + } + d = time.Duration(n) * time.Second + default: + if m := expiresInRe.FindStringSubmatch(v); m != nil { + n, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + return 0, objectstorage.ErrUsagef("invalid --expires-in %q: %v", s, err) + } + unit := map[string]time.Duration{"s": time.Second, "m": time.Minute, "h": time.Hour, "d": 24 * time.Hour}[m[2]] + d = time.Duration(n) * unit + } else { + parsed, err := time.ParseDuration(v) + if err != nil { + return 0, objectstorage.ErrUsagef("invalid --expires-in %q: use seconds (3600) or a duration such as 15m, 1h or 7d", s) + } + d = parsed + } + } + if d < time.Second { + return 0, objectstorage.ErrUsagef("--expires-in must be at least 1 second (got %q)", s) + } + if d > presignMaxExpiry { + return 0, objectstorage.ErrUsagef("--expires-in %q exceeds the 7 day maximum (604800 seconds) for pre-signed URLs", s) + } + return d, nil +} + +func isDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// presignObject signs the URL locally for the given method. +func presignObject(ctx context.Context, sess *Session, key string, opts presignOptions) (objectstorage.PresignResult, error) { + if opts.ContentType != "" && opts.Method == http.MethodPut { + objectstorage.Warnf("--content-type is not part of the signature; the uploader must send Content-Type: %s itself (a mismatch is accepted by most backends but not enforced)", opts.ContentType) + } + params := url.Values{} + if opts.VersionID != "" { + params.Set("versionId", opts.VersionID) + } + expiresAt := time.Now().Add(opts.Expires) + var ( + u *url.URL + err error + ) + switch opts.Method { + case http.MethodPut: + u, err = sess.Client.PresignedPutObject(ctx, sess.Bucket.BucketName, key, opts.Expires) + case http.MethodHead: + u, err = sess.Client.PresignedHeadObject(ctx, sess.Bucket.BucketName, key, opts.Expires, params) + default: + u, err = sess.Client.PresignedGetObject(ctx, sess.Bucket.BucketName, key, opts.Expires, params) + } + if err != nil { + return objectstorage.PresignResult{}, sess.humanize(err) + } + if u == nil { + return objectstorage.PresignResult{}, exitcode.Errorf(exitcode.Generic, "presigning returned no URL") + } + return objectstorage.PresignResult{URL: u.String(), Method: opts.Method, ExpiresAt: expiresAt}, nil +} + +// presignWrite prints the human output: only the URL, so the command composes +// with $(...) and pipes. +func presignWrite(w io.Writer, res objectstorage.PresignResult) { + fmt.Fprintln(w, res.URL) +} diff --git a/cmd/s3/presign_test.go b/cmd/s3/presign_test.go new file mode 100644 index 0000000..59f460a --- /dev/null +++ b/cmd/s3/presign_test.go @@ -0,0 +1,261 @@ +package s3 + +import ( + "bytes" + "context" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" +) + +func presignTestSession(t *testing.T, srv *s3test.Server, region string) *Session { + t.Helper() + b := &objectstorage.Bucket{ID: "bkt_1", Name: "b", BucketName: "backups-7f3a", Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: region} + cred := objectstorage.NewCredential("AKIAEXAMPLE", "topsecret", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +func TestParseExpiresIn(t *testing.T) { + cases := []struct { + in string + want time.Duration + wantErr string + }{ + {in: "3600", want: time.Hour}, + {in: "1", want: time.Second}, + {in: "604800", want: 7 * 24 * time.Hour}, + {in: "15m", want: 15 * time.Minute}, + {in: "90s", want: 90 * time.Second}, + {in: "1h", want: time.Hour}, + {in: "7d", want: 7 * 24 * time.Hour}, + {in: "1h30m", want: 90 * time.Minute}, + {in: " 2h ", want: 2 * time.Hour}, + {in: "8d", wantErr: "exceeds the 7 day maximum"}, + {in: "604801", wantErr: "exceeds the 7 day maximum"}, + {in: "0", wantErr: "at least 1 second"}, + {in: "junk", wantErr: "invalid --expires-in"}, + {in: "10x", wantErr: "invalid --expires-in"}, + {in: "-5", wantErr: "invalid --expires-in"}, + {in: "", wantErr: "cannot be empty"}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + got, err := parseExpiresIn(tc.in) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("expected error, got %v", got) + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("exit code = %d, want %d", exitcode.Of(err), exitcode.Usage) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("error %q does not mention %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("got %v, want %v", got, tc.want) + } + }) + } +} + +func TestPresignParseOptions(t *testing.T) { + if o, err := presignParseOptions("get", "3600", "", ""); err != nil || o.Method != http.MethodGet || o.Expires != time.Hour { + t.Errorf("lower-case get: %+v %v", o, err) + } + if o, err := presignParseOptions("", "60", "", ""); err != nil || o.Method != http.MethodGet { + t.Errorf("empty method defaults to GET: %+v %v", o, err) + } + if o, err := presignParseOptions("head", "60", "v1", ""); err != nil || o.Method != http.MethodHead || o.VersionID != "v1" { + t.Errorf("head with version: %+v %v", o, err) + } + if _, err := presignParseOptions("POST", "60", "", ""); exitcode.Of(err) != exitcode.Usage { + t.Errorf("POST must be a usage error, got %v", err) + } + if _, err := presignParseOptions("PUT", "60", "v1", ""); exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--version-id") { + t.Errorf("PUT with version-id must be a usage error, got %v", err) + } + if _, err := presignParseOptions("GET", "60", "", "text/plain"); exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--content-type") { + t.Errorf("GET with content-type must be a usage error, got %v", err) + } + if o, err := presignParseOptions("put", "60", "", "text/plain"); err != nil || o.ContentType != "text/plain" { + t.Errorf("PUT with content-type: %+v %v", o, err) + } + if _, err := presignParseOptions("GET", "8d", "", ""); exitcode.Of(err) != exitcode.Usage { + t.Errorf("8d must be a usage error, got %v", err) + } +} + +func TestPresignGetURL(t *testing.T) { + srv := s3test.New() + defer srv.Close() + sess := presignTestSession(t, srv, "eu-west-2") + + before := time.Now() + res, err := presignObject(context.Background(), sess, "2026/09/report.pdf", presignOptions{Method: http.MethodGet, Expires: 15 * time.Minute}) + if err != nil { + t.Fatal(err) + } + if res.Method != http.MethodGet { + t.Errorf("method = %q", res.Method) + } + if res.ExpiresAt.Before(before.Add(15*time.Minute-time.Second)) || res.ExpiresAt.After(time.Now().Add(15*time.Minute+time.Second)) { + t.Errorf("expires_at = %v", res.ExpiresAt) + } + u, err := url.Parse(res.URL) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(res.URL, srv.URL()) { + t.Errorf("url %q not on endpoint %s", res.URL, srv.URL()) + } + if u.Path != "/backups-7f3a/2026/09/report.pdf" { + t.Errorf("path = %q, want path-style //", u.Path) + } + q := u.Query() + if q.Get("X-Amz-Expires") != "900" { + t.Errorf("X-Amz-Expires = %q", q.Get("X-Amz-Expires")) + } + if cred := q.Get("X-Amz-Credential"); !strings.Contains(cred, "/eu-west-2/s3/aws4_request") || !strings.HasPrefix(cred, "AKIAEXAMPLE/") { + t.Errorf("X-Amz-Credential = %q", cred) + } + if q.Get("X-Amz-Algorithm") != "AWS4-HMAC-SHA256" || q.Get("X-Amz-Signature") == "" { + t.Errorf("missing SigV4 query parameters: %v", q) + } + if q.Has("versionId") { + t.Error("versionId must be absent when not requested") + } + if strings.Contains(res.URL, "topsecret") { + t.Error("URL leaked the secret") + } + // Presigning is local: the bucket is never contacted. + if n := len(srv.Requests()); n != 0 { + t.Errorf("%d requests sent while presigning, want 0", n) + } + + var out bytes.Buffer + presignWrite(&out, res) + if out.String() != res.URL+"\n" { + t.Errorf("human output = %q", out.String()) + } +} + +func TestPresignVersionIDAndHead(t *testing.T) { + srv := s3test.New() + defer srv.Close() + sess := presignTestSession(t, srv, "us-east-1") + + res, err := presignObject(context.Background(), sess, "k", presignOptions{Method: http.MethodHead, Expires: time.Hour, VersionID: "3HL4kqtJlcpXroDTDmJ"}) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(res.URL) + if err != nil { + t.Fatal(err) + } + if u.Query().Get("versionId") != "3HL4kqtJlcpXroDTDmJ" { + t.Errorf("versionId missing: %s", res.URL) + } + if u.Query().Get("X-Amz-Expires") != "3600" { + t.Errorf("X-Amz-Expires = %q", u.Query().Get("X-Amz-Expires")) + } + if res.Method != http.MethodHead { + t.Errorf("method = %q", res.Method) + } +} + +func TestPresignPutURL(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + sess := presignTestSession(t, srv, "us-east-1") + + res, err := presignObject(context.Background(), sess, "in/upload.bin", presignOptions{Method: http.MethodPut, Expires: 7 * 24 * time.Hour}) + if err != nil { + t.Fatal(err) + } + if res.Method != http.MethodPut { + t.Errorf("method = %q", res.Method) + } + u, err := url.Parse(res.URL) + if err != nil { + t.Fatal(err) + } + if u.Path != "/backups-7f3a/in/upload.bin" || u.Query().Get("X-Amz-Expires") != "604800" { + t.Errorf("url = %s", res.URL) + } + if len(srv.Requests()) != 0 { + t.Error("presigning must not contact the backend") + } + + // The URL actually works against the fake server without credentials. + req, err := http.NewRequest(http.MethodPut, res.URL, strings.NewReader("payload")) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("PUT with presigned URL = %d", resp.StatusCode) + } + if o := srv.Object("backups-7f3a", "in/upload.bin"); o == nil || string(o.Data) != "payload" { + t.Errorf("uploaded object = %+v", o) + } +} + +// TestPresignCommandRejectsRegionFlag guards the aws-compat shim on presign: +// --region must fail with exit 2 and point to --signing-region, before the +// bucket is looked up. +func TestPresignCommandRejectsRegionFlag(t *testing.T) { + cmd := NewPresignCmd() + Finalize(cmd) // production installs this in build_s3.go + cmd.SetArgs([]string{"s3://backups/report.pdf", "--region", "us-east-1"}) + var err error + _, stderr := rmCaptureOutput(t, func() { err = cmd.Execute() }) + if exitcode.Of(err) != exitcode.Usage { + t.Fatalf("exit code = %d, want %d (%v)", exitcode.Of(err), exitcode.Usage, err) + } + if !strings.Contains(stderr, "--region is not supported") || !strings.Contains(stderr, "--signing-region") { + t.Errorf("stderr lacks the directed --region explanation:\n%s", stderr) + } +} + +// TestPresignRejectsPrefixWithoutRecursiveHint covers the message fix: the +// shared object parser points at --recursive, a flag presign does not have. +func TestPresignRejectsPrefixWithoutRecursiveHint(t *testing.T) { + cmd := NewPresignCmd() + Finalize(cmd) + cmd.SetArgs([]string{"s3://backups/2026/"}) + var errBuf bytes.Buffer + cmd.SetErr(&errBuf) + err := cmd.Execute() + if err == nil { + t.Fatal("expected a usage error for a prefix") + } + if code := exitcode.Of(err); code != exitcode.Usage { + t.Errorf("exit code = %d, want %d", code, exitcode.Usage) + } + if strings.Contains(err.Error(), "--recursive") { + t.Errorf("presign has no --recursive; message was %q", err) + } + if !strings.Contains(err.Error(), "single object") { + t.Errorf("message %q should say presign signs one object", err) + } +} diff --git a/cmd/s3/rb.go b/cmd/s3/rb.go new file mode 100644 index 0000000..94dad6a --- /dev/null +++ b/cmd/s3/rb.go @@ -0,0 +1,358 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + cobra "github.com/spf13/cobra" +) + +// Flag names of rb. +const ( + flagRbForce = "force" + flagRbVersions = "versions" + flagRbBypass = "bypass-governance-retention" + flagRbMaxDelete = "max-delete" +) + +// rbDeleteBatch caps the objects sent per multi-delete request. +const rbDeleteBatch = 1000 + +// rbDeleteBucketAPI deletes the bucket record through the Latitude API. It is +// a variable so tests can stub the control plane. +var rbDeleteBucketAPI = func(ctx context.Context, b *objectstorage.Bucket) error { + client := apiClient() + _, err := client.ObjectStorage.DeleteStorageBuckets(ctx, b.ID, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return rbHumanizeAPIDelete(objectstorage.Humanize(err, nil, nil), b) + } + return nil +} + +// rbHumanizeAPIDelete adds the --force hint to "not empty" refusals from the +// API (409/422 mentioning not empty, or an S3 BucketNotEmpty passthrough). +func rbHumanizeAPIDelete(err error, b *objectstorage.Bucket) error { + if err == nil { + return nil + } + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "not empty") || strings.Contains(msg, "bucketnotempty") { + return exitcode.Errorf(exitcode.Refused, "bucket %s is not empty; re-run with --force to delete its objects first (add --versions for versioned buckets)", b.Display()) + } + return err +} + +// rbOptions are the inputs of the --force flow. +type rbOptions struct { + Versions bool + Bypass bool + Yes bool + MaxDelete int64 + DryRun bool + Human bool +} + +// NewRbCmd builds `lsh s3 rb s3://bucket`. +func NewRbCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "rb s3://bucket", + Aliases: []string{"delete"}, + Short: "Delete a bucket", + Long: `Delete a bucket through the Latitude API. + +Without --force the bucket must be empty; the API refuses otherwise (exit 7). +With --force the CLI lists the objects, asks for confirmation (or --yes), +deletes them and then removes the bucket. Versioned buckets also need +--versions so every object version and delete marker is removed. Buckets with COMPLIANCE object lock cannot be +force-deleted; GOVERNANCE retention needs --bypass-governance-retention. + +--force never skips the confirmation: pass --yes in scripts. Without a TTY and +without --yes the command fails fast with exit 7.`, + Example: ` lsh s3 rb s3://backups + lsh s3 rb s3://backups --force + lsh s3 rb s3://backups --force --versions --yes + lsh s3 rb s3://backups --force --max-delete 100 --dryrun`, + Args: cobra.ExactArgs(1), + RunE: runRb, + }) + addProjectFlag(cmd, true, "project ID or slug to disambiguate the bucket name") + addYesFlag(cmd) + addBucketFilterFlags(cmd) + cmd.Flags().Bool(flagRbForce, false, "delete all objects in the bucket first") + cmd.Flags().Bool(flagRbVersions, false, "with --force: also delete every object version and delete marker") + cmd.Flags().Bool(flagRbBypass, false, "with --force: bypass GOVERNANCE object lock retention") + cmd.Flags().Int64(flagRbMaxDelete, 0, "with --force: refuse when more than N objects would be deleted (0 = no limit)") + return cmd +} + +func runRb(cmd *cobra.Command, args []string) error { + ref, err := objectstorage.ParseBucketOnly(args[0]) + if err != nil { + return printErr(err) + } + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "rb deletes the bucket through the Latitude API and cannot work with --endpoint-url / %s; unset it and address the bucket by name or bkt_ ID", objectstorage.EnvEndpointURL)) + } + f := cmd.Flags() + force, _ := f.GetBool(flagRbForce) + yes, _ := f.GetBool(flagYes) + o := rbOptions{Yes: yes, DryRun: dryRun(), Human: isHuman()} + o.Versions, _ = f.GetBool(flagRbVersions) + o.Bypass, _ = f.GetBool(flagRbBypass) + o.MaxDelete, _ = f.GetInt64(flagRbMaxDelete) + if o.MaxDelete < 0 { + return printErr(exitcode.Errorf(exitcode.Usage, "--max-delete must be zero or a positive number")) + } + if !force && (o.Versions || o.Bypass || o.MaxDelete > 0) { + return printErr(exitcode.Errorf(exitcode.Usage, "--versions, --bypass-governance-retention and --max-delete only apply with --force")) + } + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + + if !force { + row := objectstorage.DeleteResult{Bucket: b.Name, DryRun: o.DryRun} + if !o.DryRun { + if err := rbDeleteBucketAPI(ctx, b); err != nil { + return printErr(err) + } + row.Deleted = true + } + rbEmitResults(o.Human, os.Stdout, []objectstorage.DeleteResult{row}) + return nil + } + + sess, err := openResolved(cmd, b, true) + if err != nil { + return printErr(err) + } + rows, err := forceRemoveBucket(ctx, cmd, sess, o, os.Stdout) + if !o.Human && len(rows) > 0 { + render(objectstorage.AsResponseData(rows)) + } + if err != nil { + return printErr(err) + } + return nil +} + +// forceRemoveBucket empties the bucket and deletes it (or prints the plan in +// dry-run mode). In human mode the aws-style lines are written to w as they +// happen; otherwise the rows are returned for the caller to render. The +// returned rows are also populated on partial failure so structured output +// still lists what was deleted. +func forceRemoveBucket(ctx context.Context, cmd *cobra.Command, sess *Session, o rbOptions, w io.Writer) ([]objectstorage.DeleteResult, error) { + b := sess.Bucket + if b.Locking && strings.EqualFold(b.RetentionMode, string(operations.RetentionModeCompliance)) { + return nil, exitcode.Errorf(exitcode.Refused, "bucket %s has COMPLIANCE object lock: its objects cannot be deleted until their retention expires, so the bucket cannot be force-deleted", b.Display()) + } + if b.Versioning && !o.Versions { + return nil, exitcode.Errorf(exitcode.Refused, "bucket %s is versioned; add --versions to delete all object versions and delete markers as well", b.Display()) + } + if b.Locking && !o.Bypass && !o.DryRun { + objectstorage.Hintf("note: bucket %s has %s object lock; objects still under retention will fail to delete unless you pass --bypass-governance-retention", b.Display(), strings.ToUpper(b.RetentionMode)) + } + + objects, err := rbListAllObjects(ctx, sess.Client, b.BucketName, o.Versions) + if err != nil { + return nil, sess.humanize(err) + } + if o.MaxDelete > 0 && int64(len(objects)) > o.MaxDelete { + return nil, exitcode.Errorf(exitcode.Refused, "bucket %s holds %s %s, more than --max-delete %d; nothing was deleted", b.Display(), rbCommaInt(int64(len(objects))), rbNoun(o.Versions, len(objects)), o.MaxDelete) + } + + if o.DryRun { + rows := make([]objectstorage.DeleteResult, 0, len(objects)+1) + for _, obj := range objects { + rows = append(rows, rbDeleteRow(b.Name, obj, o.Versions, true)) + } + rows = append(rows, objectstorage.DeleteResult{Bucket: b.Name, DryRun: true}) + if o.Human { + rbEmitResults(true, w, rows) + } + return rows, nil + } + + question := fmt.Sprintf("Delete bucket %s (%s, project %s, backend %s) and its %s %s?", + b.Name, b.ID, b.ProjectRef(), b.BucketName, rbCommaInt(int64(len(objects))), rbNoun(o.Versions, len(objects))) + if len(objects) == 0 { + question = fmt.Sprintf("Delete bucket %s (%s, project %s, backend %s)? It is empty.", b.Name, b.ID, b.ProjectRef(), b.BucketName) + } + if err := objectstorage.ConfirmOrRefuse(cmd, o.Yes, question); err != nil { + return nil, err + } + + rows, deleted, failed := rbDeleteObjects(ctx, sess, objects, o, w) + // An interrupt aborts the batches mid-flight and inflates failed; report + // the signal instead of a partial run (same contract as rm). + if ctx.Err() != nil { + return rows, objectstorage.Humanize(ctx.Err(), sess.Bucket, &sess.Cred) + } + if failed > 0 { + fmt.Fprintf(os.Stderr, "%d deleted, %d failed; bucket %s was not removed\n", deleted, failed, b.Display()) + return rows, exitcode.Errorf(exitcode.Partial, "could not delete every object of bucket %s (%d deleted, %d failed)", b.Display(), deleted, failed) + } + + if err := rbDeleteBucketAPI(ctx, b); err != nil { + return rows, err + } + final := objectstorage.DeleteResult{Bucket: b.Name, Deleted: true} + rows = append(rows, final) + if o.Human { + fmt.Fprintln(w, final.HumanLine()) + } + return rows, nil +} + +// rbListAllObjects enumerates every object (or every version and delete marker +// when withVersions is set) in the bucket. +func rbListAllObjects(ctx context.Context, client *minio.Client, bucket string, withVersions bool) ([]minio.ObjectInfo, error) { + var out []minio.ObjectInfo + for info := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Recursive: true, WithVersions: withVersions}) { + if info.Err != nil { + return nil, info.Err + } + out = append(out, info) + } + return out, nil +} + +// rbDeleteObjects removes the objects in batches. It stops at the first batch +// with a failure. Human mode prints one line per object to w. +func rbDeleteObjects(ctx context.Context, sess *Session, objects []minio.ObjectInfo, o rbOptions, w io.Writer) (rows []objectstorage.DeleteResult, deleted, failed int) { + b := sess.Bucket + for start := 0; start < len(objects); start += rbDeleteBatch { + end := start + rbDeleteBatch + if end > len(objects) { + end = len(objects) + } + batch := objects[start:end] + if ctx.Err() != nil { + return rows, deleted, failed + } + + ch := make(chan minio.ObjectInfo, len(batch)) + for _, obj := range batch { + item := minio.ObjectInfo{Key: obj.Key} + if o.Versions && obj.VersionID != "" && obj.VersionID != "null" { + item.VersionID = obj.VersionID + } + ch <- item + } + close(ch) + + results := sess.Client.RemoveObjectsWithResult(ctx, b.BucketName, ch, minio.RemoveObjectsOptions{GovernanceBypass: o.Bypass}) + seen := map[string]bool{} + for res := range results { + row := objectstorage.DeleteResult{Bucket: b.Name, Key: res.ObjectName} + if o.Versions { + row.VersionID = res.ObjectVersionID + } + if res.Err != nil { + row.Error = sess.humanize(res.Err).Error() + failed++ + } else { + row.Deleted = true + deleted++ + } + seen[row.Key+"\x00"+row.VersionID] = true + rows = append(rows, row) + if o.Human { + fmt.Fprintln(w, row.HumanLine()) + } + } + // Backends in quiet mode report nothing for successes: account for the + // objects of a batch that produced no result as deleted. + for _, obj := range batch { + vid := "" + if o.Versions && obj.VersionID != "" && obj.VersionID != "null" { + vid = obj.VersionID + } + if seen[obj.Key+"\x00"+vid] { + continue + } + row := objectstorage.DeleteResult{Bucket: b.Name, Key: obj.Key, VersionID: vid, Deleted: true} + deleted++ + rows = append(rows, row) + if o.Human { + fmt.Fprintln(w, row.HumanLine()) + } + } + if failed > 0 { + return rows, deleted, failed + } + } + return rows, deleted, failed +} + +// rbDeleteRow builds the row for one listed object. +func rbDeleteRow(bucket string, obj minio.ObjectInfo, withVersions, dry bool) objectstorage.DeleteResult { + row := objectstorage.DeleteResult{Bucket: bucket, Key: obj.Key, DryRun: dry} + if withVersions && obj.VersionID != "" && obj.VersionID != "null" { + row.VersionID = obj.VersionID + } + return row +} + +// rbEmitResults prints human lines to w, or renders the rows. +func rbEmitResults(human bool, w io.Writer, rows []objectstorage.DeleteResult) { + if human { + for _, r := range rows { + fmt.Fprintln(w, r.HumanLine()) + } + return + } + out := make([]renderer.ResponseData, 0, len(rows)) + for _, r := range rows { + out = append(out, r) + } + render(out) +} + +func rbNoun(versions bool, n int) string { + switch { + case versions && n == 1: + return "object version" + case versions: + return "object versions" + case n == 1: + return "object" + default: + return "objects" + } +} + +// rbCommaInt renders 1204 as "1,204". +func rbCommaInt(n int64) string { + s := strconv.FormatInt(n, 10) + neg := strings.HasPrefix(s, "-") + if neg { + s = s[1:] + } + var sb strings.Builder + for i, r := range s { + if i > 0 && (len(s)-i)%3 == 0 { + sb.WriteByte(',') + } + sb.WriteRune(r) + } + if neg { + return "-" + sb.String() + } + return sb.String() +} diff --git a/cmd/s3/rb_test.go b/cmd/s3/rb_test.go new file mode 100644 index 0000000..3753379 --- /dev/null +++ b/cmd/s3/rb_test.go @@ -0,0 +1,309 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" +) + +// newRbStatSession builds a Session against the fake S3 server for the rb and +// stat tests. +func newRbStatSession(t *testing.T, srv *s3test.Server, backendName string) *Session { + t.Helper() + b := &objectstorage.Bucket{ + ID: "bkt_1", + Name: "backups", + BucketName: backendName, + Endpoint: srv.URL(), + StorageClass: "standard", + SigningRegion: "us-east-1", + ProjectSlug: "my-project", + } + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +// stubRbDeleteAPI replaces the API delete for the test and reports calls. +func stubRbDeleteAPI(t *testing.T, fail error) *int { + t.Helper() + calls := 0 + prev := rbDeleteBucketAPI + rbDeleteBucketAPI = func(ctx context.Context, b *objectstorage.Bucket) error { + calls++ + return fail + } + t.Cleanup(func() { rbDeleteBucketAPI = prev }) + return &calls +} + +func rbSeedBucket(srv *s3test.Server, name string, n int) { + srv.CreateBucket(name) + for i := 0; i < n; i++ { + srv.AddObject(name, fmt.Sprintf("dir/file-%02d.txt", i), []byte("payload"), "text/plain") + } +} + +func TestForceRemoveBucket_DeletesObjectsThenBucket(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 3) + sess := newRbStatSession(t, srv, "backups-7f3a") + calls := stubRbDeleteAPI(t, nil) + + var out bytes.Buffer + rows, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true}, &out) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if *calls != 1 { + t.Errorf("API delete called %d times, want 1", *calls) + } + if keys := srv.Keys("backups-7f3a"); len(keys) != 0 { + t.Errorf("objects left on the backend: %v", keys) + } + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(lines) != 4 { + t.Fatalf("expected 3 delete lines + remove_bucket, got %d:\n%s", len(lines), out.String()) + } + for _, l := range lines[:3] { + if !strings.HasPrefix(l, "delete: s3://backups/dir/file-") { + t.Errorf("unexpected line %q", l) + } + } + if lines[3] != "remove_bucket: s3://backups" { + t.Errorf("last line = %q", lines[3]) + } + if len(rows) != 4 || !rows[3].Deleted || rows[3].Key != "" { + t.Errorf("rows = %+v", rows) + } + // Deletes go through multi-object POSTs, never one DELETE per object. + for _, r := range srv.WriteRequests() { + if r.Method == "DELETE" { + t.Errorf("unexpected single DELETE request %s", r.Path) + } + } +} + +func TestForceRemoveBucket_MaxDeleteRefuses(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 5) + sess := newRbStatSession(t, srv, "backups-7f3a") + calls := stubRbDeleteAPI(t, nil) + + var out bytes.Buffer + _, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true, MaxDelete: 2}, &out) + if err == nil { + t.Fatalf("expected a refusal") + } + if exitcode.Of(err) != exitcode.Refused { + t.Errorf("exit code = %d, want %d", exitcode.Of(err), exitcode.Refused) + } + if !strings.Contains(err.Error(), "--max-delete 2") || !strings.Contains(err.Error(), "5 objects") { + t.Errorf("message %q should mention the limit and the count", err.Error()) + } + if *calls != 0 { + t.Errorf("API delete must not be called, got %d", *calls) + } + if len(srv.WriteRequests()) != 0 { + t.Errorf("no write must reach the backend, got %d", len(srv.WriteRequests())) + } + if len(srv.Keys("backups-7f3a")) != 5 { + t.Errorf("objects were deleted despite the refusal") + } + if out.Len() != 0 { + t.Errorf("nothing should be printed on stdout, got %q", out.String()) + } +} + +func TestForceRemoveBucket_DryRunWritesNothing(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 2) + sess := newRbStatSession(t, srv, "backups-7f3a") + calls := stubRbDeleteAPI(t, nil) + + var out bytes.Buffer + // Yes is deliberately false: dry-run never prompts. + rows, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{DryRun: true, Human: true}, &out) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if *calls != 0 { + t.Errorf("API delete must not be called in dry-run, got %d", *calls) + } + if len(srv.WriteRequests()) != 0 { + t.Errorf("dry-run must not write, got %d write requests", len(srv.WriteRequests())) + } + if len(srv.Keys("backups-7f3a")) != 2 { + t.Errorf("dry-run deleted objects") + } + want := "(dryrun) delete: s3://backups/dir/file-00.txt\n(dryrun) delete: s3://backups/dir/file-01.txt\n(dryrun) remove_bucket: s3://backups\n" + if out.String() != want { + t.Errorf("output:\n%s\nwant:\n%s", out.String(), want) + } + if len(rows) != 3 || !rows[0].DryRun || !rows[2].DryRun { + t.Errorf("rows = %+v", rows) + } +} + +func TestForceRemoveBucket_VersionedNeedsVersionsFlag(t *testing.T) { + srv := s3test.New() + defer srv.Close() + bkt := srv.CreateBucket("backups-7f3a") + bkt.Versioned = true + // The fake server derives version IDs from the history length before + // appending, so pin distinct IDs explicitly. + srv.AddObject("backups-7f3a", "a.txt", []byte("v1"), "text/plain").VersionID = "v1" + srv.AddObject("backups-7f3a", "a.txt", []byte("v2"), "text/plain").VersionID = "v2" + sess := newRbStatSession(t, srv, "backups-7f3a") + sess.Bucket.Versioning = true + calls := stubRbDeleteAPI(t, nil) + + var out bytes.Buffer + _, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true}, &out) + if err == nil || exitcode.Of(err) != exitcode.Refused { + t.Fatalf("expected exit 7 refusal, got %v", err) + } + if !strings.Contains(err.Error(), "--versions") || !strings.Contains(err.Error(), "delete markers") { + t.Errorf("message %q should point to --versions", err.Error()) + } + if *calls != 0 || len(srv.Requests()) != 0 { + t.Errorf("refusal must happen before any request (api=%d, s3=%d)", *calls, len(srv.Requests())) + } + + // With --versions every version goes away. + out.Reset() + rows, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true, Versions: true}, &out) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if *calls != 1 { + t.Errorf("API delete called %d times, want 1", *calls) + } + if len(srv.Keys("backups-7f3a")) != 0 { + t.Errorf("versions left: %v", srv.Keys("backups-7f3a")) + } + if len(rows) != 3 { // v1, v2 and the bucket + t.Errorf("rows = %+v", rows) + } + if !strings.Contains(out.String(), "(version v1)") || !strings.Contains(out.String(), "(version v2)") { + t.Errorf("output should list versions:\n%s", out.String()) + } +} + +func TestForceRemoveBucket_ComplianceLockRefused(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 1) + sess := newRbStatSession(t, srv, "backups-7f3a") + sess.Bucket.Locking = true + sess.Bucket.Versioning = true + sess.Bucket.RetentionMode = "COMPLIANCE" + stubRbDeleteAPI(t, nil) + + _, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Versions: true, Human: true}, &bytes.Buffer{}) + if err == nil || exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "COMPLIANCE") { + t.Fatalf("expected COMPLIANCE refusal, got %v", err) + } +} + +func TestForceRemoveBucket_NonInteractiveWithoutYesRefuses(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 1) + sess := newRbStatSession(t, srv, "backups-7f3a") + calls := stubRbDeleteAPI(t, nil) + + _, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Human: true}, &bytes.Buffer{}) + if err == nil || exitcode.Of(err) != exitcode.Refused { + t.Fatalf("expected exit 7 without --yes in a non-TTY, got %v", err) + } + for _, want := range []string{"backups", "bkt_1", "my-project", "backups-7f3a", "1 object"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("question %q should include %q", err.Error(), want) + } + } + if *calls != 0 || len(srv.WriteRequests()) != 0 { + t.Errorf("nothing must be deleted before confirmation") + } +} + +func TestForceRemoveBucket_APIDeleteFailureKeepsExitCode(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 1) + sess := newRbStatSession(t, srv, "backups-7f3a") + stubRbDeleteAPI(t, exitcode.Errorf(exitcode.Permission, "your API token does not have permission")) + + var out bytes.Buffer + rows, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true}, &out) + if err == nil || exitcode.Of(err) != exitcode.Permission { + t.Fatalf("expected the API error to propagate, got %v", err) + } + if len(rows) != 1 || !rows[0].Deleted { + t.Errorf("object rows should still be returned: %+v", rows) + } + if strings.Contains(out.String(), "remove_bucket") { + t.Errorf("remove_bucket must not be printed when the API delete failed") + } +} + +func TestRbDeleteObjects_BackendFailureIsPartial(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 2) + sess := newRbStatSession(t, srv, "backups-7f3a") + objects, err := rbListAllObjects(context.Background(), sess.Client, "backups-7f3a", false) + if err != nil { + t.Fatal(err) + } + srv.FailNext = &s3test.ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: 403} + + var out bytes.Buffer + rows, deleted, failed := rbDeleteObjects(context.Background(), sess, objects, rbOptions{Human: true}, &out) + if deleted != 0 || failed == 0 { + t.Fatalf("deleted=%d failed=%d, want a failed batch", deleted, failed) + } + if len(rows) == 0 || rows[0].Error == "" { + t.Errorf("rows should carry the error: %+v", rows) + } + if !strings.Contains(out.String(), "delete failed:") { + t.Errorf("human output should report the failure:\n%s", out.String()) + } +} + +func TestRbHumanizeAPIDelete(t *testing.T) { + b := &objectstorage.Bucket{ID: "bkt_1", Name: "backups"} + err := rbHumanizeAPIDelete(exitcode.Errorf(exitcode.Usage, "the API rejected the request: bucket is not empty"), b) + if exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "--force") { + t.Errorf("not-empty should become exit 7 with a --force hint, got %v", err) + } + other := exitcode.Errorf(exitcode.NotFound, "bucket not found") + if got := rbHumanizeAPIDelete(other, b); !errors.Is(got, other) { + t.Errorf("unrelated errors must pass through, got %v", got) + } + if rbHumanizeAPIDelete(nil, b) != nil { + t.Errorf("nil must stay nil") + } +} + +func TestRbCommaInt(t *testing.T) { + cases := map[int64]string{0: "0", 7: "7", 999: "999", 1000: "1,000", 1204: "1,204", 1234567: "1,234,567", -1204: "-1,204"} + for in, want := range cases { + if got := rbCommaInt(in); got != want { + t.Errorf("rbCommaInt(%d) = %q, want %q", in, got, want) + } + } +} diff --git a/cmd/s3/rm.go b/cmd/s3/rm.go new file mode 100644 index 0000000..c054073 --- /dev/null +++ b/cmd/s3/rm.go @@ -0,0 +1,435 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" +) + +// rmBatchSize is the S3 multi-delete limit. +const rmBatchSize = 1000 + +// rmOptions are the parsed flags of `lsh s3 rm`. +type rmOptions struct { + Recursive bool + All bool + MaxDelete int + Versions bool + VersionID string + BypassGovernance bool + Quiet bool + OnlyShowErrors bool + DryRun bool + Filters *objectstorage.Filters +} + +// NewRmCmd builds `lsh s3 rm s3://bucket/key`. +func NewRmCmd() *cobra.Command { + filters := &objectstorage.Filters{} + cmd := newCmd(&cobra.Command{ + Use: "rm s3://bucket/key", + Aliases: []string{"delete-object", "remove"}, + Short: "Delete objects", + Long: `Delete one object, or every object under a prefix with --recursive. + +A single object is deleted without confirmation and the command exits 0 even +when the key does not exist (S3 deletes are idempotent). With --recursive the +prefix is normalized to end with "/", the matching objects are enumerated and +counted, and you are asked to confirm (or pass --yes). Deleting every object +in a bucket (s3://bucket with no prefix) additionally requires --all. + +Filters (--exclude/--include) are shell globs evaluated against the key +relative to the prefix, in the order given; the last matching rule wins. + +Object lock: buckets in COMPLIANCE mode are refused; GOVERNANCE mode requires +--bypass-governance-retention. Use --dry-run to print the plan without +deleting anything.`, + Example: ` lsh s3 rm s3://backups/2026/09/dump.sql + lsh s3 rm s3://logs/tmp/ --recursive --dryrun + lsh s3 rm s3://logs/tmp/ --recursive --exclude "*" --include "*.log" --yes + lsh s3 rm s3://logs --recursive --all --max-delete 5000 --yes + lsh s3 rm s3://backups/dump.sql --version-id 3HL4kqtJlcpXroDTDmJ`, + Args: cobra.ExactArgs(1), + }) + f := cmd.Flags() + f.BoolP("recursive", "r", false, "delete every object under the prefix (asks for confirmation)") + f.Bool("all", false, "allow deleting every object in the bucket (required when no prefix is given)") + f.Int("max-delete", 0, "refuse to delete more than N objects (0 = unlimited)") + f.Bool("versions", false, "with --recursive: delete all versions and delete markers, not only the current objects") + f.String("version-id", "", "delete this specific version of the object") + f.Bool("bypass-governance-retention", false, "bypass GOVERNANCE object lock retention (requires a key allowed to do so)") + f.BoolP("quiet", "q", false, "do not print the deleted objects") + f.Bool("only-show-errors", false, "print only failures") + objectstorage.FilterFlags(f, filters) + addYesFlag(cmd) + addProjectFlag(cmd, true, "project the bucket belongs to (disambiguates buckets with the same name)") + addBucketFilterFlags(cmd) + unsupportedAWSBoolFlags(cmd, map[string]string{ + "request-payer": "Latitude buckets have no requester-pays mode", + }) + rejectRegionFlag(cmd) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + opts := rmOptions{Filters: filters, DryRun: dryRun()} + opts.Recursive, _ = f.GetBool("recursive") + opts.All, _ = f.GetBool("all") + opts.MaxDelete, _ = f.GetInt("max-delete") + opts.Versions, _ = f.GetBool("versions") + opts.VersionID, _ = f.GetString("version-id") + opts.BypassGovernance, _ = f.GetBool("bypass-governance-retention") + opts.Quiet, _ = f.GetBool("quiet") + opts.OnlyShowErrors, _ = f.GetBool("only-show-errors") + yes, _ := f.GetBool(flagYes) + + ref, err := rmParseArgs(args[0], opts) + if err != nil { + return printErr(err) + } + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + sess, err := openBucket(ctx, cmd, ref.Bucket, true) + if err != nil { + return printErr(err) + } + confirm := func(question string) error { + return objectstorage.ConfirmOrRefuse(cmd, yes, question) + } + rows, err := runRm(ctx, sess, ref, opts, isHuman(), os.Stdout, confirm) + if !isHuman() && len(rows) > 0 { + render(objectstorage.AsResponseData(rows)) + } + if err != nil { + return printErr(err) + } + return nil + } + return cmd +} + +// rmParseArgs parses the target and validates flag combinations. +func rmParseArgs(arg string, opts rmOptions) (objectstorage.Ref, error) { + if opts.MaxDelete < 0 { + return objectstorage.Ref{}, objectstorage.ErrUsagef("--max-delete must be zero or positive") + } + if !opts.Recursive { + if opts.All { + return objectstorage.Ref{}, objectstorage.ErrUsagef("--all only makes sense with --recursive") + } + if opts.Versions { + return objectstorage.Ref{}, objectstorage.ErrUsagef("--versions requires --recursive; use --version-id to delete one version of an object") + } + // ObjectRef rejects bucket-only targets and keys ending in "/" with + // the hint to add --recursive. A bare bucket gets a longer message: + // the legacy `storage-objects rm ` deleted the bucket, and + // that alias now lands here, so both destinations are spelled out. + ref, err := objectstorage.ObjectRef(arg, false) + if err != nil { + if bucket, bucketErr := objectstorage.ParseBucketOnly(arg); bucketErr == nil { + return ref, objectstorage.ErrUsagef("%s names a bucket, not an object; to delete its objects use 'lsh s3 rm s3://%s --recursive --all', to delete the bucket itself use 'lsh s3 rb s3://%s'", arg, bucket.Bucket, bucket.Bucket) + } + } + return ref, err + } + if opts.VersionID != "" { + return objectstorage.Ref{}, objectstorage.ErrUsagef("--version-id cannot be combined with --recursive; use --versions to delete every version under the prefix") + } + ref, err := objectstorage.ParseRemote(arg) + if err != nil { + return ref, err + } + ref.Key = objectstorage.NormalizePrefix(strings.TrimLeft(ref.Key, "/")) + if ref.Key == "" && !opts.All { + return ref, objectstorage.ErrUsagef("refusing to delete every object in s3://%s without --all", ref.Bucket) + } + return ref, nil +} + +// runRm executes rm against an open session. Human lines go to out; the +// structured rows are returned for the caller to render. Errors carry exit +// codes (2 usage, 6 partial, 7 refused). +func runRm(ctx context.Context, sess *Session, ref objectstorage.Ref, opts rmOptions, human bool, out io.Writer, confirm func(question string) error) ([]objectstorage.DeleteResult, error) { + if !opts.Recursive { + return rmSingle(ctx, sess, ref.Key, opts, human, out) + } + return rmRecursive(ctx, sess, ref.Key, opts, human, out, confirm) +} + +// rmBucketLabel is the bucket part of `s3:///` in the output: +// the display name, or the backend name when addressed by --endpoint-url. +func rmBucketLabel(b *objectstorage.Bucket) string { + if b.Name != "" && !b.EndpointOverride { + return b.Name + } + return b.BucketName +} + +// rmCheckObjectLock enforces the object lock rules before any write. +func rmCheckObjectLock(b *objectstorage.Bucket, bypass bool) error { + if !b.Locking { + return nil + } + switch strings.ToUpper(b.RetentionMode) { + case "COMPLIANCE": + return exitcode.Errorf(exitcode.Refused, "bucket %s has object lock in COMPLIANCE mode; objects cannot be deleted until their retention expires", b.Display()) + case "GOVERNANCE": + if !bypass { + return exitcode.Errorf(exitcode.Refused, "bucket %s has object lock in GOVERNANCE mode; re-run with --bypass-governance-retention to delete retained objects", b.Display()) + } + } + return nil +} + +// rmSingle deletes one object (or one version) without HEAD and without a +// prompt. A missing key is not an error. +func rmSingle(ctx context.Context, sess *Session, key string, opts rmOptions, human bool, out io.Writer) ([]objectstorage.DeleteResult, error) { + if opts.VersionID != "" { + // Deleting a specific version is permanent, so the lock rules apply; + // without --version-id a versioned bucket only gets a delete marker. + if err := rmCheckObjectLock(sess.Bucket, opts.BypassGovernance); err != nil { + return nil, err + } + } + res := objectstorage.DeleteResult{Bucket: rmBucketLabel(sess.Bucket), Key: key, VersionID: opts.VersionID, DryRun: opts.DryRun} + if opts.DryRun { + rmEmit(res, opts, human, out) + return []objectstorage.DeleteResult{res}, nil + } + err := sess.Client.RemoveObject(ctx, sess.Bucket.BucketName, key, minio.RemoveObjectOptions{ + VersionID: opts.VersionID, + GovernanceBypass: opts.BypassGovernance, + }) + if err != nil { + return nil, sess.humanize(err) + } + res.Deleted = true + rmEmit(res, opts, human, out) + return []objectstorage.DeleteResult{res}, nil +} + +// rmEmit prints one result in human mode, honouring --quiet and +// --only-show-errors (both silence successes only, like aws). Failures +// always go to stderr; structured mode prints nothing here. +func rmEmit(res objectstorage.DeleteResult, opts rmOptions, human bool, out io.Writer) { + if res.Error != "" { + if human { + fmt.Fprintln(os.Stderr, res.HumanLine()) + } + return + } + if !human || opts.Quiet || opts.OnlyShowErrors { + return + } + fmt.Fprintln(out, res.HumanLine()) +} + +// rmPlan enumerates the objects under prefix that pass the filters. +func rmPlan(ctx context.Context, sess *Session, prefix string, opts rmOptions) ([]minio.ObjectInfo, int64, error) { + var ( + objects []minio.ObjectInfo + total int64 + ) + for info := range sess.Client.ListObjects(ctx, sess.Bucket.BucketName, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + WithVersions: opts.Versions, + }) { + if info.Err != nil { + return nil, 0, sess.humanize(info.Err) + } + if info.Key == "" { + continue + } + // Filters see the key relative to the prefix, like aws. + rel := strings.TrimPrefix(info.Key, prefix) + if !opts.Filters.Include(rel) { + continue + } + objects = append(objects, info) + total += info.Size + } + return objects, total, nil +} + +// rmRecursive is `rm --recursive`: lock checks, enumeration, --max-delete, +// confirmation, batched deletion and the partial-failure summary. +func rmRecursive(ctx context.Context, sess *Session, prefix string, opts rmOptions, human bool, out io.Writer, confirm func(string) error) ([]objectstorage.DeleteResult, error) { + if err := rmCheckObjectLock(sess.Bucket, opts.BypassGovernance); err != nil { + return nil, err + } + label := rmBucketLabel(sess.Bucket) + target := "s3://" + label + if prefix != "" { + target += "/" + prefix + } + + objects, total, err := rmPlan(ctx, sess, prefix, opts) + if err != nil { + return nil, err + } + what := "objects" + if opts.Versions { + what = "object versions" + } + if opts.MaxDelete > 0 && len(objects) > opts.MaxDelete { + return nil, exitcode.Errorf(exitcode.Refused, "%d %s match under %s but --max-delete is %d; refusing to delete anything", len(objects), what, target, opts.MaxDelete) + } + + if opts.DryRun { + rows := make([]objectstorage.DeleteResult, 0, len(objects)) + for _, o := range objects { + res := objectstorage.DeleteResult{Bucket: label, Key: o.Key, VersionID: o.VersionID, DryRun: true} + rmEmit(res, opts, human, out) + rows = append(rows, res) + } + objectstorage.Hintf("would delete %d %s (%s)", len(objects), what, objectstorage.HumanSize(total)) + return rows, nil + } + if len(objects) == 0 { + return nil, nil + } + + // target already ends with "/" for a prefix. + question := fmt.Sprintf("Delete %d %s under %s?", len(objects), what, target) + if prefix == "" { + question = fmt.Sprintf("Delete all %d %s in %s (%s)?", len(objects), what, target, rmBucketDetails(sess.Bucket)) + } + if err := confirm(question); err != nil { + return nil, err + } + + var ( + rows []objectstorage.DeleteResult + deleted, failed int + stopped bool + ) + for start := 0; start < len(objects) && !stopped; start += rmBatchSize { + end := start + rmBatchSize + if end > len(objects) { + end = len(objects) + } + results := rmDeleteBatch(ctx, sess, objects[start:end], opts.BypassGovernance, label) + for _, res := range results { + rmEmit(res, opts, human, out) + rows = append(rows, res) + if res.Error == "" { + deleted++ + continue + } + failed++ + if rmIsRetentionError(res.Error) { + stopped = true + } + } + if stopped { + remaining := len(objects) - end + if remaining > 0 { + objectstorage.Warnf("stopped after a retention error; %d %s were not attempted", remaining, what) + } + } + if ctx.Err() != nil { + return rows, objectstorage.Humanize(ctx.Err(), sess.Bucket, &sess.Cred) + } + } + if failed > 0 { + return rows, exitcode.Errorf(exitcode.Partial, "%d deleted, %d failed", deleted, failed) + } + return rows, nil +} + +// rmBucketDetails renders the identity shown when the whole bucket is at +// stake: id, backend name and project. +func rmBucketDetails(b *objectstorage.Bucket) string { + parts := []string{} + if b.ID != "" { + parts = append(parts, b.ID) + } + if b.BucketName != "" && b.BucketName != b.Name { + parts = append(parts, "backend name "+b.BucketName) + } + if p := b.ProjectRef(); p != "" { + parts = append(parts, "project "+p) + } + if len(parts) == 0 { + return b.BucketName + } + return strings.Join(parts, ", ") +} + +// rmDeleteBatch deletes up to rmBatchSize objects with one multi-delete call +// and returns a result per object, in input order. A request-level failure +// (the whole POST rejected) marks every object of the batch as failed. +func rmDeleteBatch(ctx context.Context, sess *Session, batch []minio.ObjectInfo, bypass bool, label string) []objectstorage.DeleteResult { + objectsCh := make(chan minio.ObjectInfo, len(batch)) + for _, o := range batch { + objectsCh <- o + } + close(objectsCh) + + failed := map[string]error{} + var batchErr error + for e := range sess.Client.RemoveObjects(ctx, sess.Bucket.BucketName, objectsCh, minio.RemoveObjectsOptions{GovernanceBypass: bypass}) { + if e.ObjectName == "" { + batchErr = e.Err + continue + } + failed[rmObjectID(e.ObjectName, e.VersionID)] = e.Err + } + + results := make([]objectstorage.DeleteResult, 0, len(batch)) + for _, o := range batch { + res := objectstorage.DeleteResult{Bucket: label, Key: o.Key, VersionID: o.VersionID} + err, ok := failed[rmObjectID(o.Key, o.VersionID)] + if !ok { + err, ok = failed[rmObjectID(o.Key, "")] + } + switch { + case ok: + res.Error = rmErrorText(err) + case batchErr != nil: + res.Error = rmErrorText(batchErr) + default: + res.Deleted = true + } + results = append(results, res) + } + return results +} + +func rmObjectID(key, versionID string) string { return key + "\x00" + versionID } + +// rmErrorText renders a per-object failure compactly (code: message). +func rmErrorText(err error) string { + if err == nil { + return "unknown error" + } + if resp := minio.ToErrorResponse(err); resp.Code != "" { + if resp.Message != "" { + return resp.Code + ": " + resp.Message + } + return resp.Code + } + return objectstorage.Redact(err.Error()) +} + +// rmIsRetentionError reports whether a per-object failure text denotes an +// object lock / retention refusal, after which the command stops. +func rmIsRetentionError(text string) bool { + t := strings.ToLower(text) + return strings.Contains(t, "objectlocked") || + strings.Contains(t, "retention") || + strings.Contains(t, "object lock") || + strings.Contains(t, "governance") +} + +// Compile-time check that DeleteResult renders through the shared renderer. +var _ renderer.ResponseData = objectstorage.DeleteResult{} diff --git a/cmd/s3/rm_test.go b/cmd/s3/rm_test.go new file mode 100644 index 0000000..dc9d029 --- /dev/null +++ b/cmd/s3/rm_test.go @@ -0,0 +1,662 @@ +package s3 + +import ( + "bytes" + "context" + "io" + "net/http" + "os" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" +) + +// rmTestSession opens a Session against the fake S3 server. +func rmTestSession(t *testing.T, srv *s3test.Server, bucketName string) *Session { + t.Helper() + b := &objectstorage.Bucket{ID: "bkt_1", Name: "b", BucketName: bucketName, Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: "us-east-1"} + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +func rmYes(string) error { return nil } + +func rmFilters(t *testing.T, rules ...string) *objectstorage.Filters { + t.Helper() + f := &objectstorage.Filters{} + for i := 0; i+1 < len(rules); i += 2 { + if err := f.Add(rules[i] == "include", rules[i+1]); err != nil { + t.Fatal(err) + } + } + return f +} + +func TestRmParseArgs(t *testing.T) { + cases := []struct { + name string + arg string + opts rmOptions + wantKey string + wantCode int + wantMsg string + }{ + {name: "single object", arg: "s3://logs/tmp/old.log", wantKey: "tmp/old.log"}, + {name: "single without scheme", arg: "logs/tmp/old.log", wantKey: "tmp/old.log"}, + {name: "single trailing slash needs recursive", arg: "s3://logs/tmp/", wantCode: exitcode.Usage, wantMsg: "--recursive"}, + {name: "single bucket only", arg: "s3://logs", wantCode: exitcode.Usage, wantMsg: "names a bucket"}, + {name: "recursive normalizes prefix", arg: "s3://logs/tmp", opts: rmOptions{Recursive: true}, wantKey: "tmp/"}, + {name: "recursive keeps trailing slash", arg: "s3://logs/tmp/", opts: rmOptions{Recursive: true}, wantKey: "tmp/"}, + {name: "recursive whole bucket needs all", arg: "s3://logs", opts: rmOptions{Recursive: true}, wantCode: exitcode.Usage, wantMsg: "refusing to delete every object in s3://logs without --all"}, + {name: "recursive slash only needs all", arg: "s3://logs/", opts: rmOptions{Recursive: true}, wantCode: exitcode.Usage, wantMsg: "without --all"}, + {name: "recursive double slash needs all", arg: "s3://logs//", opts: rmOptions{Recursive: true}, wantCode: exitcode.Usage, wantMsg: "without --all"}, + {name: "recursive whole bucket with all", arg: "s3://logs", opts: rmOptions{Recursive: true, All: true}, wantKey: ""}, + {name: "all without recursive", arg: "s3://logs/k", opts: rmOptions{All: true}, wantCode: exitcode.Usage, wantMsg: "--all"}, + {name: "versions without recursive", arg: "s3://logs/k", opts: rmOptions{Versions: true}, wantCode: exitcode.Usage, wantMsg: "--versions"}, + {name: "version-id with recursive", arg: "s3://logs/k/", opts: rmOptions{Recursive: true, VersionID: "v1"}, wantCode: exitcode.Usage, wantMsg: "--version-id"}, + {name: "negative max-delete", arg: "s3://logs/k", opts: rmOptions{MaxDelete: -1}, wantCode: exitcode.Usage, wantMsg: "--max-delete"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ref, err := rmParseArgs(tc.arg, tc.opts) + if tc.wantCode != 0 { + if err == nil { + t.Fatalf("expected error, got ref %+v", ref) + } + if exitcode.Of(err) != tc.wantCode { + t.Errorf("exit code = %d, want %d (%v)", exitcode.Of(err), tc.wantCode, err) + } + if !strings.Contains(err.Error(), tc.wantMsg) { + t.Errorf("error %q does not mention %q", err, tc.wantMsg) + } + return + } + if err != nil { + t.Fatal(err) + } + if ref.Key != tc.wantKey { + t.Errorf("key = %q, want %q", ref.Key, tc.wantKey) + } + }) + } +} + +func TestRmSingleObjectNoHeadAndIdempotent(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/old.log", []byte("x"), "text/plain") + srv.AddObject("logs-7f3a", "tmp/keep.log", []byte("y"), "text/plain") + sess := rmTestSession(t, srv, "logs-7f3a") + + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/old.log", Remote: true}, rmOptions{}, true, &out, rmYes) + if err != nil { + t.Fatal(err) + } + if got := out.String(); got != "delete: s3://b/tmp/old.log\n" { + t.Errorf("stdout = %q", got) + } + if len(rows) != 1 || !rows[0].Deleted || rows[0].Key != "tmp/old.log" { + t.Errorf("rows = %+v", rows) + } + if srv.Object("logs-7f3a", "tmp/old.log") != nil { + t.Error("object still present") + } + if srv.Object("logs-7f3a", "tmp/keep.log") == nil { + t.Error("sibling object was deleted") + } + for _, r := range srv.Requests() { + if r.Method == http.MethodHead || r.Method == http.MethodGet { + t.Errorf("unexpected %s %s: single delete must not HEAD or list", r.Method, r.Path) + } + if r.Method == http.MethodDelete && r.Path != "/logs-7f3a/tmp/old.log" { + t.Errorf("unexpected delete path %s", r.Path) + } + } + + // Missing key: still exit 0 and still reported as deleted. + srv.ResetRequests() + out.Reset() + rows, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/missing.log", Remote: true}, rmOptions{}, true, &out, rmYes) + if err != nil { + t.Fatalf("delete of a missing key must succeed: %v", err) + } + if len(rows) != 1 || out.String() != "delete: s3://b/tmp/missing.log\n" { + t.Errorf("rows=%+v out=%q", rows, out.String()) + } +} + +func TestRmSingleVersionIDAndQuiet(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("vers-7f3a").Versioned = true + // Each put on a versioned bucket gets the next version id (v1, v2…). + srv.AddObject("vers-7f3a", "k", []byte("1"), "") + current := srv.AddObject("vers-7f3a", "k", []byte("22"), "") + if current.VersionID != "v2" { + t.Fatalf("fake server version id = %q, want v2", current.VersionID) + } + sess := rmTestSession(t, srv, "vers-7f3a") + + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "k", Remote: true}, rmOptions{VersionID: "v2", Quiet: true}, true, &out, rmYes) + if err != nil { + t.Fatal(err) + } + if out.Len() != 0 { + t.Errorf("--quiet printed %q", out.String()) + } + if len(rows) != 1 || rows[0].VersionID != "v2" { + t.Errorf("rows = %+v", rows) + } + var sawVersion bool + for _, r := range srv.WriteRequests() { + if r.Method == http.MethodDelete && r.Query.Get("versionId") == "v2" { + sawVersion = true + } + } + if !sawVersion { + t.Error("DELETE did not carry versionId=v2") + } + if o := srv.Object("vers-7f3a", "k"); o == nil || string(o.Data) != "1" || o.VersionID != "v1" { + t.Errorf("previous version (v1) should be current now, got %+v", o) + } +} + +func TestRmRecursiveWithFiltersAndCount(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a.log", []byte("aaa"), "") + srv.AddObject("logs-7f3a", "tmp/b.log", []byte("bb"), "") + srv.AddObject("logs-7f3a", "tmp/notes.txt", []byte("n"), "") + srv.AddObject("logs-7f3a", "tmp/sub/c.log", []byte("c"), "") + srv.AddObject("logs-7f3a", "tmp2/d.log", []byte("d"), "") + srv.AddObject("logs-7f3a", "other.log", []byte("o"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + var out bytes.Buffer + var asked string + confirm := func(q string) error { asked = q; return nil } + opts := rmOptions{Recursive: true, Filters: rmFilters(t, "exclude", "*", "include", "*.log")} + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, opts, true, &out, confirm) + if err != nil { + t.Fatal(err) + } + if asked != "Delete 3 objects under s3://b/tmp/?" { + t.Errorf("confirmation = %q", asked) + } + if len(rows) != 3 { + t.Fatalf("rows = %+v", rows) + } + want := "delete: s3://b/tmp/a.log\ndelete: s3://b/tmp/b.log\ndelete: s3://b/tmp/sub/c.log\n" + if out.String() != want { + t.Errorf("stdout = %q, want %q", out.String(), want) + } + if keys := srv.Keys("logs-7f3a"); strings.Join(keys, ",") != "other.log,tmp/notes.txt,tmp2/d.log" { + t.Errorf("remaining keys = %v", keys) + } + // One multi-delete POST for the batch, no per-object DELETEs. + var posts, deletes int + for _, r := range srv.WriteRequests() { + switch r.Method { + case http.MethodPost: + posts++ + if !r.Query.Has("delete") { + t.Errorf("POST without ?delete: %s", r.Path) + } + case http.MethodDelete: + deletes++ + } + } + if posts != 1 || deletes != 0 { + t.Errorf("posts=%d deletes=%d, want one multi-delete", posts, deletes) + } +} + +func TestRmRecursiveStructuredRowsAndOnlyShowErrors(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a.log", []byte("a"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, OnlyShowErrors: true}, true, &out, rmYes) + if err != nil { + t.Fatal(err) + } + if out.Len() != 0 { + t.Errorf("--only-show-errors printed %q", out.String()) + } + if len(rows) != 1 || !rows[0].Deleted || rows[0].Bucket != "b" || rows[0].Key != "tmp/a.log" { + t.Errorf("rows = %+v", rows) + } + // Structured mode never writes to out either. + srv.AddObject("logs-7f3a", "tmp/b.log", []byte("b"), "") + out.Reset() + rows, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, false, &out, rmYes) + if err != nil || out.Len() != 0 || len(rows) != 1 { + t.Errorf("structured: err=%v out=%q rows=%+v", err, out.String(), rows) + } +} + +func TestRmRecursiveEmptyPrefixIsNoop(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "keep.log", []byte("k"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + called := false + confirm := func(string) error { called = true; return nil } + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "nothing/", Remote: true}, rmOptions{Recursive: true}, true, &out, confirm) + if err != nil || len(rows) != 0 || out.Len() != 0 { + t.Errorf("err=%v rows=%+v out=%q", err, rows, out.String()) + } + if called { + t.Error("no confirmation should be asked when nothing matches") + } + if len(srv.WriteRequests()) != 0 { + t.Error("no writes expected") + } +} + +func TestRmRecursiveMaxDeleteRefusesWithoutWrites(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + for _, k := range []string{"tmp/a", "tmp/b", "tmp/c"} { + srv.AddObject("logs-7f3a", k, []byte("x"), "") + } + sess := rmTestSession(t, srv, "logs-7f3a") + + called := false + confirm := func(string) error { called = true; return nil } + var out bytes.Buffer + _, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, MaxDelete: 2}, true, &out, confirm) + if err == nil || exitcode.Of(err) != exitcode.Refused { + t.Fatalf("expected exit %d, got %v", exitcode.Refused, err) + } + if !strings.Contains(err.Error(), "3 objects match") || !strings.Contains(err.Error(), "--max-delete is 2") { + t.Errorf("message = %q", err) + } + if called { + t.Error("must refuse before asking for confirmation") + } + if n := len(srv.WriteRequests()); n != 0 { + t.Errorf("%d write requests, want 0", n) + } + if len(srv.Keys("logs-7f3a")) != 3 { + t.Error("objects were deleted") + } + + // Exactly at the limit is allowed. + _, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, MaxDelete: 3}, true, &out, rmYes) + if err != nil { + t.Fatalf("max-delete equal to the count must pass: %v", err) + } + if len(srv.Keys("logs-7f3a")) != 0 { + t.Error("objects not deleted") + } +} + +func TestRmRecursiveRefusedConfirmation(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a", []byte("x"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + refuse := func(q string) error { return exitcode.Errorf(exitcode.Refused, "cancelled") } + var out bytes.Buffer + _, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, true, &out, refuse) + if exitcode.Of(err) != exitcode.Refused { + t.Fatalf("expected exit 7, got %v", err) + } + if len(srv.WriteRequests()) != 0 || srv.Object("logs-7f3a", "tmp/a") == nil { + t.Error("declined confirmation must not delete") + } +} + +func TestRmDryRunZeroWrites(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a.log", []byte("aaaa"), "") + srv.AddObject("logs-7f3a", "tmp/b.txt", []byte("bb"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + called := false + confirm := func(string) error { called = true; return nil } + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, DryRun: true}, true, &out, confirm) + if err != nil { + t.Fatal(err) + } + want := "(dryrun) delete: s3://b/tmp/a.log\n(dryrun) delete: s3://b/tmp/b.txt\n" + if out.String() != want { + t.Errorf("stdout = %q, want %q", out.String(), want) + } + if len(rows) != 2 || !rows[0].DryRun || rows[0].Deleted { + t.Errorf("rows = %+v", rows) + } + if called { + t.Error("dry-run must not prompt") + } + if n := len(srv.WriteRequests()); n != 0 { + t.Errorf("%d write requests during dry-run", n) + } + if len(srv.Keys("logs-7f3a")) != 2 { + t.Error("dry-run deleted objects") + } + + // Single-object dry-run: no request at all. + srv.ResetRequests() + out.Reset() + rows, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/a.log", Remote: true}, rmOptions{DryRun: true}, true, &out, confirm) + if err != nil || len(rows) != 1 || out.String() != "(dryrun) delete: s3://b/tmp/a.log\n" { + t.Errorf("single dry-run: err=%v rows=%+v out=%q", err, rows, out.String()) + } + if len(srv.Requests()) != 0 { + t.Error("single-object dry-run must not call the backend") + } +} + +func TestRmRecursiveVersions(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("vers-7f3a").Versioned = true + srv.AddObject("vers-7f3a", "tmp/a", []byte("1"), "") + srv.AddObject("vers-7f3a", "tmp/a", []byte("22"), "") + srv.AddObject("vers-7f3a", "tmp/b", []byte("3"), "") + sess := rmTestSession(t, srv, "vers-7f3a") + + // Without --versions only the current objects are targeted. + var out bytes.Buffer + var asked string + confirm := func(q string) error { asked = q; return nil } + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, DryRun: true}, true, &out, confirm) + if err != nil || len(rows) != 2 { + t.Fatalf("plain plan: err=%v rows=%+v", err, rows) + } + + // With --versions every version is enumerated and deleted. + out.Reset() + rows, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, Versions: true}, true, &out, confirm) + if err != nil { + t.Fatal(err) + } + if asked != "Delete 3 object versions under s3://b/tmp/?" { + t.Errorf("confirmation = %q", asked) + } + if len(rows) != 3 { + t.Fatalf("rows = %+v", rows) + } + perKey := map[string]int{} + for _, r := range rows { + if !r.Deleted || r.VersionID == "" { + t.Errorf("row %+v should be a deleted version", r) + } + perKey[r.Key]++ + } + if perKey["tmp/a"] != 2 || perKey["tmp/b"] != 1 { + t.Errorf("versions per key = %v, want tmp/a:2 tmp/b:1", perKey) + } + if !strings.Contains(out.String(), "delete: s3://b/tmp/a (version v1)") { + t.Errorf("stdout = %q", out.String()) + } + if len(srv.Keys("vers-7f3a")) != 0 { + t.Errorf("remaining keys %v", srv.Keys("vers-7f3a")) + } + var listedVersions bool + for _, r := range srv.Requests() { + if r.Method == http.MethodGet && r.Query.Has("versions") { + listedVersions = true + } + } + if !listedVersions { + t.Error("expected a ListObjectVersions call") + } +} + +func TestRmWholeBucketWithAll(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "a", []byte("1"), "") + srv.AddObject("logs-7f3a", "dir/b", []byte("2"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + sess.Bucket.ProjectSlug = "my-project" + + var asked string + confirm := func(q string) error { asked = q; return nil } + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "", Remote: true}, rmOptions{Recursive: true, All: true}, true, &out, confirm) + if err != nil || len(rows) != 2 { + t.Fatalf("err=%v rows=%+v", err, rows) + } + for _, want := range []string{"Delete all 2 objects in s3://b", "bkt_1", "backend name logs-7f3a", "project my-project"} { + if !strings.Contains(asked, want) { + t.Errorf("confirmation %q lacks %q", asked, want) + } + } + if len(srv.Keys("logs-7f3a")) != 0 { + t.Error("bucket not emptied") + } +} + +func TestRmObjectLockRefusals(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("lock-7f3a") + srv.AddObject("lock-7f3a", "tmp/a", []byte("1"), "") + sess := rmTestSession(t, srv, "lock-7f3a") + sess.Bucket.Locking = true + sess.Bucket.RetentionMode = "COMPLIANCE" + + var out bytes.Buffer + _, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, true, &out, rmYes) + if exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "COMPLIANCE") { + t.Errorf("compliance: %v", err) + } + sess.Bucket.RetentionMode = "GOVERNANCE" + _, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, true, &out, rmYes) + if exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "--bypass-governance-retention") { + t.Errorf("governance: %v", err) + } + if len(srv.Requests()) != 0 { + t.Error("lock refusals must happen before any request") + } + // Bypass: proceeds and sends the bypass header. + _, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, BypassGovernance: true}, true, &out, rmYes) + if err != nil { + t.Fatal(err) + } + var sawBypass bool + for _, r := range srv.WriteRequests() { + if strings.EqualFold(r.Header.Get("X-Amz-Bypass-Governance-Retention"), "true") { + sawBypass = true + } + } + if !sawBypass { + t.Error("multi-delete did not carry x-amz-bypass-governance-retention") + } +} + +func TestRmRecursivePartialFailure(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a", []byte("1"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + var out bytes.Buffer + // Listing succeeds; the multi-delete POST is denied. + confirm := func(string) error { + srv.FailNext = &s3test.ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: 403} + return nil + } + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, true, &out, confirm) + if exitcode.Of(err) != exitcode.Partial { + t.Fatalf("expected exit %d, got %v", exitcode.Partial, err) + } + if !strings.Contains(err.Error(), "0 deleted, 1 failed") { + t.Errorf("summary = %q", err) + } + if len(rows) != 1 || rows[0].Deleted || !strings.Contains(rows[0].Error, "AccessDenied") { + t.Errorf("rows = %+v", rows) + } + if out.Len() != 0 { + t.Errorf("failures must not go to stdout: %q", out.String()) + } +} + +// rmCaptureOutput runs fn with os.Stdout and os.Stderr redirected to pipes and +// returns what was written to each. Command-level tests need it because the +// rm/presign RunE and objectstorage.PrintError write straight to the process +// streams (and cobra echoes errors to os.Stderr unless silenced). +func rmCaptureOutput(t *testing.T, fn func()) (stdout, stderr string) { + t.Helper() + redirect := func(target **os.File) func() string { + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := *target + *target = w + done := make(chan string, 1) + go func() { + b, _ := io.ReadAll(r) + done <- string(b) + }() + return func() string { + _ = w.Close() + *target = old + return <-done + } + } + finishOut := redirect(&os.Stdout) + finishErr := redirect(&os.Stderr) + fn() + // Restore stderr first so a failing assertion below is visible. + stderr = finishErr() + stdout = finishOut() + return stdout, stderr +} + +// TestRmCommandRecursiveWithoutAllIsUsageError drives the cobra command end to +// end: `rm s3://bucket --recursive` without --all must exit 2 through the +// newCmd wrapper and print the refusal exactly once (RunE prints it, cobra is +// silenced). It fails before any bucket lookup, so no API or S3 call is made. +func TestRmCommandRecursiveWithoutAllIsUsageError(t *testing.T) { + for _, target := range []string{"s3://logs", "s3://logs/"} { + t.Run(target, func(t *testing.T) { + cmd := NewRmCmd() + Finalize(cmd) // production installs this in build_s3.go + cmd.SetArgs([]string{target, "--recursive"}) + var err error + stdout, stderr := rmCaptureOutput(t, func() { err = cmd.Execute() }) + if exitcode.Of(err) != exitcode.Usage { + t.Fatalf("exit code = %d, want %d (%v)", exitcode.Of(err), exitcode.Usage, err) + } + if n := strings.Count(stderr, "refusing to delete every object in s3://logs without --all"); n != 1 { + t.Errorf("refusal printed %d times, want exactly once:\n%s", n, stderr) + } + if strings.Contains(stderr, "Usage:") { + t.Errorf("usage must not be echoed on errors:\n%s", stderr) + } + if stdout != "" { + t.Errorf("nothing should reach stdout, got %q", stdout) + } + }) + } +} + +// TestRmCommandSingleMissingKeyExitsZero runs a plain `rm` of a key that does +// not exist through the s3 group in --endpoint-url mode against the fake S3 +// server: S3 deletes are idempotent, so the command exits 0 and reports the +// key as deleted. +func TestRmCommandSingleMissingKeyExitsZero(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + t.Setenv(objectstorage.EnvAccessKeyID, "AK") + t.Setenv(objectstorage.EnvSecretAccessKey, "SK") + + group := NewGroupCmd() + group.AddCommand(NewRmCmd()) + Finalize(group) // production installs this in build_s3.go + group.SetArgs([]string{"rm", "s3://logs-7f3a/tmp/missing.log", "--endpoint-url", srv.URL()}) + var err error + stdout, stderr := rmCaptureOutput(t, func() { err = group.Execute() }) + if err != nil { + t.Fatalf("rm of a missing key must exit 0, got %v\nstderr: %s", err, stderr) + } + if stdout != "delete: s3://logs-7f3a/tmp/missing.log\n" { + t.Errorf("stdout = %q", stdout) + } + if strings.Contains(stderr, "Error") { + t.Errorf("no error expected on stderr:\n%s", stderr) + } + var deletes int + for _, r := range srv.Requests() { + if r.Method == http.MethodDelete { + deletes++ + } + if r.Method == http.MethodHead || r.Method == http.MethodGet { + t.Errorf("unexpected %s %s: single delete must not HEAD or list", r.Method, r.Path) + } + } + if deletes != 1 { + t.Errorf("%d DELETE requests, want 1", deletes) + } +} + +// TestRmCommandRejectsRegionFlag guards the aws-compat shim: --region is an +// unsupported flag on object commands and must fail with exit 2 and the +// pointer to --signing-region, before any network call. +func TestRmCommandRejectsRegionFlag(t *testing.T) { + cmd := NewRmCmd() + Finalize(cmd) + cmd.SetArgs([]string{"s3://logs/tmp/old.log", "--region", "us-east-1"}) + var err error + _, stderr := rmCaptureOutput(t, func() { err = cmd.Execute() }) + if exitcode.Of(err) != exitcode.Usage { + t.Fatalf("exit code = %d, want %d (%v)", exitcode.Of(err), exitcode.Usage, err) + } + if !strings.Contains(stderr, "--region is not supported") || !strings.Contains(stderr, "--signing-region") { + t.Errorf("stderr lacks the directed --region explanation:\n%s", stderr) + } +} + +// TestRmBucketOnlyPointsAtRb covers the migration path of the legacy +// `storage-objects rm `, which deleted a bucket: that alias now lands +// on `s3 rm`, so the usage error names both destinations. +func TestRmBucketOnlyPointsAtRb(t *testing.T) { + _, err := rmParseArgs("bkt_abc123", rmOptions{}) + if err == nil { + t.Fatal("expected a usage error for a bucket-only target") + } + if code := exitcode.Of(err); code != exitcode.Usage { + t.Errorf("exit code = %d, want %d", code, exitcode.Usage) + } + for _, want := range []string{"names a bucket, not an object", "--recursive --all", "lsh s3 rb s3://bkt_abc123"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q is missing %q", err, want) + } + } + // A prefix keeps the shorter --recursive hint. + if _, err := rmParseArgs("s3://logs/tmp/", rmOptions{}); err == nil || strings.Contains(err.Error(), "lsh s3 rb") { + t.Errorf("a prefix must keep the --recursive hint, got %v", err) + } +} diff --git a/cmd/s3/s3.go b/cmd/s3/s3.go new file mode 100644 index 0000000..ca58dcd --- /dev/null +++ b/cmd/s3/s3.go @@ -0,0 +1,478 @@ +// Package s3 implements `lsh s3`, the object storage command group. It mixes +// control-plane calls to the Latitude API (buckets, access keys, lifecycle, +// metrics, usage) with data-plane calls straight to the bucket's S3 endpoint +// (ls, cp, rm, stat, presign), using the verbs and flags of `aws s3` so +// existing habits and scripts carry over. +package s3 + +import ( + "context" + "fmt" + "os" + "regexp" + "strings" + "sync" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cli" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// Persistent flag names shared by the data-plane commands. +const ( + flagAccessKey = "access-key" + flagEndpointURL = "endpoint-url" + flagSigningRegion = "signing-region" + flagS3Region = "s3-region" // aws-flavoured alias of --signing-region + flagAddressing = "addressing-style" + flagDryRunAWS = "dryrun" // aws spelling of --dry-run + flagProject = "project" + flagYes = "yes" + flagSite = "site" +) + +// NewGroupCmd builds the `s3` group. Subcommands are attached by +// cmd/build_s3.go so each lives in its own file. +func NewGroupCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "s3", + Aliases: []string{"buckets", "storage-objects"}, + // Typing these at the root gets a "did you mean s3" hint. + SuggestFor: []string{"s3api", "upload", "download", "configure", "bucket", "objects", "object-storage"}, + GroupID: cli.StorageGroupID, + Short: "Object storage: buckets, objects, access keys and lifecycle", + Long: `Manage S3-compatible object storage. + +Bucket management, access keys, lifecycle rules and metrics go through the +Latitude API; object operations (ls, cp, rm, stat, presign) talk to the +bucket's S3 endpoint directly. Buckets are addressed as s3://[/], +where is the display name, the bkt_ ID or the backend bucket name. +Endpoint, signing region and path-style addressing are resolved from the API, +so nothing has to be configured by hand. + +Getting started: + lsh s3 mb s3://backups --region DAL --project my-project + lsh s3 cp ./dump.sql s3://backups/2026/09/ + lsh s3 ls s3://backups/2026/09/ + +Exit codes for scripts: 'lsh help exit-codes'.`, + Example: ` lsh s3 ls + lsh s3 ls s3://backups/2026/ --recursive --human-readable --summarize + lsh s3 cp ./dump.sql s3://backups/2026/09/ + lsh s3 cp s3://backups/2026/09/dump.sql ./restore/ + lsh s3 rm s3://backups/tmp/ --recursive --dryrun + lsh s3 presign s3://backups/report.pdf --expires-in 15m + lsh s3 access-keys create --bucket backups --name ci-deploy + lsh s3 lifecycle create s3://logs --prefix tmp/ --expiration-days 7`, + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + if v, _ := cmd.Flags().GetBool(flagDryRunAWS); v { + lsh.DryRun = true + } + return nil + }, + } + + pf := cmd.PersistentFlags() + pf.String(flagAccessKey, "", "use the saved access key with this name instead of the automatic selection") + pf.String(flagEndpointURL, "", "address the bucket on this S3 endpoint without the Latitude API (bucket is the backend name; credentials only from the environment)") + pf.String(flagSigningRegion, "", "override the SigV4 signing region derived from the endpoint") + pf.String(flagS3Region, "", "alias of --signing-region") + pf.String(flagAddressing, objectstorage.AddressingPath, "S3 addressing style: path or virtual") + pf.Bool(flagDryRunAWS, false, "alias of --dry-run") + _ = pf.MarkHidden(flagS3Region) + _ = pf.MarkHidden(flagAddressing) + _ = pf.MarkHidden(flagDryRunAWS) + + return cmd +} + +// newCmd applies the conventions every s3 subcommand shares: usage is never +// echoed on errors. Error printing/exit-code handling is installed by +// Finalize once the whole tree is built (commands may assign RunE/PreRunE +// after construction). +func newCmd(c *cobra.Command) *cobra.Command { + c.SilenceUsage = true + return c +} + +// Finalize walks the group's command tree and installs the shared error +// contract on every command: +// - errors raised by cobra itself (bad flag, wrong argument count) are +// printed once and exit 2; +// - errors returned by PreRunE/RunE are printed once through printErr (exit +// code preserved) and cobra is told not to echo them again. +// +// build_s3.go calls it after attaching the subcommands. +func Finalize(root *cobra.Command) { + root.SetFlagErrorFunc(func(cmd *cobra.Command, err error) error { + cmd.SilenceErrors = true + return printErr(exitcode.New(exitcode.Usage, err)) + }) + var walk func(c *cobra.Command) + walk = func(c *cobra.Command) { + c.SilenceUsage = true + // Mark the subtree so shared validation (cli.MakeRootCmd's pre-run) + // can apply these exit codes here without changing the older groups. + if c.Annotations == nil { + c.Annotations = map[string]string{} + } + c.Annotations[exitcode.OptInAnnotation] = "true" + if args := c.Args; args != nil { + c.Args = func(cmd *cobra.Command, a []string) error { + if err := args(cmd, a); err != nil { + cmd.SilenceErrors = true + return printErr(exitcode.New(exitcode.Usage, err)) + } + return nil + } + } + if pre := c.PreRunE; pre != nil { + c.PreRunE = func(cmd *cobra.Command, a []string) error { + return printOnce(cmd, pre(cmd, a)) + } + } + if run := c.RunE; run != nil { + c.RunE = func(cmd *cobra.Command, a []string) error { + return printOnce(cmd, run(cmd, a)) + } + } + for _, sub := range c.Commands() { + walk(sub) + } + } + walk(root) +} + +// printOnce prints err unless the command already did (objectstorage.PrintError +// marks it) and silences cobra so the message is not duplicated; the exit +// code travels with the returned error. +func printOnce(cmd *cobra.Command, err error) error { + if err == nil { + return nil + } + if !objectstorage.IsPrinted(err) { + err = printErr(err) + } + cmd.SilenceErrors = true + return err +} + +// addProjectFlag registers --project. optional=true marks it as a filter / +// disambiguator so the root pre-run never prompts for it. +func addProjectFlag(cmd *cobra.Command, optional bool, usage string) { + cmd.Flags().String(flagProject, "", usage) + if optional { + if cmd.Annotations == nil { + cmd.Annotations = map[string]string{} + } + cmd.Annotations[cli.ProjectOptionalAnnotation] = "true" + } +} + +// addBucketFilterFlags registers the flags that disambiguate a bucket name that +// resolves to several buckets: --storage-class (short -c) and --site. They are +// only useful when a display name is reused across classes or sites; the bkt_ +// ID always resolves without them. +func addBucketFilterFlags(cmd *cobra.Command) { + cmd.Flags().StringP(flagStorageClass, "c", "", "disambiguate a repeated bucket name by storage class (standard or high_performance)") + cmd.Flags().String(flagSite, "", "disambiguate a repeated bucket name by Latitude site (e.g. DAL, TYO4)") +} + +// addYesFlag registers --yes/-y. +func addYesFlag(cmd *cobra.Command) { + cmd.Flags().BoolP(flagYes, "y", false, "do not ask for confirmation") +} + +// projectFlag returns the --project value (or LSH_PROJECT when unset). +func projectFlag(cmd *cobra.Command) string { + if v, _ := cmd.Flags().GetString(flagProject); v != "" { + return v + } + return os.Getenv("LSH_PROJECT") +} + +// profileFlag returns the --profile override. +func profileFlag(cmd *cobra.Command) string { + v, _ := cmd.Flags().GetString("profile") + return v +} + +// endpointOverride returns --endpoint-url or LSH_S3_ENDPOINT_URL. +func endpointOverride(cmd *cobra.Command) string { + if v, _ := cmd.Flags().GetString(flagEndpointURL); v != "" { + return v + } + return os.Getenv(objectstorage.EnvEndpointURL) +} + +// signingRegionOverride returns --signing-region / --s3-region / env. +func signingRegionOverride(cmd *cobra.Command) string { + if v, _ := cmd.Flags().GetString(flagSigningRegion); v != "" { + return v + } + if v, _ := cmd.Flags().GetString(flagS3Region); v != "" { + return v + } + return os.Getenv(objectstorage.EnvSigningRegion) +} + +// dryRun reports whether --dry-run (or --dryrun) is active. +func dryRun() bool { return lsh.DryRun } + +// isHuman reports whether output goes to the human (table) format, in which +// case the aws-style plain lines are printed instead of the table renderer. +func isHuman() bool { return renderer.ResolveFormat() == renderer.FormatTable } + +// render prints structured results through the shared renderer without the +// interactive table (object listings can be huge and must stay pipeable). +func render(items []renderer.ResponseData) { renderer.RenderStatic(items) } + +// printErr prints a humanized error to stderr and returns it for RunE. +func printErr(err error) error { return objectstorage.PrintError(err) } + +// newResolver builds the bucket resolver for a command. +func newResolver(cmd *cobra.Command) *objectstorage.Resolver { + r := &objectstorage.Resolver{ + Project: projectFlag(cmd), + EndpointURL: endpointOverride(cmd), + SigningRegion: signingRegionOverride(cmd), + RetryOptions: []operations.Option{operations.WithRetries(lsh.RetryConfig())}, + HasFilterFlags: cmd.Flags().Lookup(flagSite) != nil, + } + if raw, _ := cmd.Flags().GetString(flagStorageClass); strings.TrimSpace(raw) != "" { + class, err := objectstorage.ParseStorageClass(raw) + if err != nil { + r.FilterErr = fmt.Errorf("--%s: %w", flagStorageClass, err) + } else { + r.ClassFilter = class + } + } + if site, _ := cmd.Flags().GetString(flagSite); strings.TrimSpace(site) != "" { + r.SiteFilter = strings.TrimSpace(site) + } + // Without a token the SDK would send an empty bearer and every API call + // would come back as 401; leave API nil so callers get the "not logged in" + // error (exit 4) with the LSH_S3_ENDPOINT_URL alternative instead. + if r.EndpointURL == "" && viper.GetString("Authorization") != "" { + r.API = apiClient() + } + if r.EndpointURL != "" { + endpointHintOnce.Do(func() { + objectstorage.Hintf("note: --endpoint-url set; bypassing the Latitude API and using credentials from the environment only") + }) + } + return r +} + +// endpointHintOnce makes the endpoint-override note appear once per run even +// when a command builds several resolvers (stat, remote-to-remote copies). +var endpointHintOnce sync.Once + +// apiClient returns the Latitude API client for object storage commands (the +// regular client plus the /storage response normalization the current SDK +// needs; see objectstorage.NewAPIClient). +func apiClient() *sdk.Latitudesh { return objectstorage.NewAPIClient() } + +// resolveBucket resolves a bucket token (name, bkt_ id or backend name). +func resolveBucket(ctx context.Context, cmd *cobra.Command, token string) (*objectstorage.Bucket, error) { + return newResolver(cmd).Resolve(ctx, token) +} + +// Session bundles everything a data-plane command needs for one bucket. +type Session struct { + Bucket *objectstorage.Bucket + Cred objectstorage.Credential + Client *minio.Client +} + +// openBucket resolves the bucket, selects a credential and builds the S3 +// client. write=true requests a credential with write permission. +func openBucket(ctx context.Context, cmd *cobra.Command, token string, write bool) (*Session, error) { + b, err := resolveBucket(ctx, cmd, token) + if err != nil { + return nil, err + } + // A high_performance key is bound to one site, but the SDK model drops the + // bucket's site, so without this the site check in keyMatchesBucket is a + // no-op on the data plane and a key from another site can be selected. + if b.StorageClass == objectstorage.ClassHighPerformance && b.Site == "" && !b.EndpointOverride { + if fillErr := newResolver(cmd).FillSite(ctx, b); fillErr != nil { + lsh.LogDebugf("[s3] could not determine the site of %s: %v", b.Display(), fillErr) + } + } + return openResolved(cmd, b, write) +} + +// openResolved is openBucket for an already-resolved bucket. +func openResolved(cmd *cobra.Command, b *objectstorage.Bucket, write bool) (*Session, error) { + if err := b.Validate(); err != nil { + return nil, err + } + accessKey, _ := cmd.Flags().GetString(flagAccessKey) + cred, err := objectstorage.ResolveCredential(b, objectstorage.CredentialOptions{ + AccessKeyName: accessKey, + ProfileOverride: profileFlag(cmd), + Write: write, + }) + if err != nil { + return nil, err + } + addressing, _ := cmd.Flags().GetString(flagAddressing) + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{ + Debug: lsh.Debug, + Addressing: addressing, + }) + if err != nil { + return nil, err + } + if lsh.Debug { + fmt.Fprintf(os.Stderr, "[s3] bucket=%s backend=%s endpoint=%s signing-region=%s addressing=%s credential=%s\n", + b.Display(), b.BucketName, b.Endpoint, b.SigningRegion, addressing, cred.Describe(b.ID)) + } + return &Session{Bucket: b, Cred: cred, Client: client}, nil +} + +// humanize maps an S3/API error for a session. +func (s *Session) humanize(err error) error { + if s == nil { + return objectstorage.Humanize(err, nil, nil) + } + return objectstorage.Humanize(err, s.Bucket, &s.Cred) +} + +// unsupportedAWSFlags registers flags that exist in `aws s3` but have no +// equivalent on Latitude, so scripts get an explanation instead of +// "unknown flag". Each entry maps the flag to the reason/alternative. +func unsupportedAWSFlags(cmd *cobra.Command, flags map[string]string) { + for name := range flags { + cmd.Flags().String(name, "", "not supported on Latitude object storage") + _ = cmd.Flags().MarkHidden(name) + } + // Bool-style flags must also parse without a value. + pre := cmd.PreRunE + cmd.PreRunE = func(c *cobra.Command, args []string) error { + for name, why := range flags { + if c.Flags().Changed(name) { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s is not supported on Latitude object storage: %s", name, why)) + } + } + if pre != nil { + return pre(c, args) + } + return nil + } +} + +// unsupportedAWSBoolFlags registers boolean aws flags that are rejected with +// an explanation when set. +func unsupportedAWSBoolFlags(cmd *cobra.Command, flags map[string]string) { + for name := range flags { + cmd.Flags().Bool(name, false, "not supported on Latitude object storage") + _ = cmd.Flags().MarkHidden(name) + } + pre := cmd.PreRunE + cmd.PreRunE = func(c *cobra.Command, args []string) error { + for name, why := range flags { + if c.Flags().Changed(name) { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s is not supported on Latitude object storage: %s", name, why)) + } + } + if pre != nil { + return pre(c, args) + } + return nil + } +} + +// joinNonEmpty joins the non-empty strings with sep. +func joinNonEmpty(sep string, parts ...string) string { + var out []string + for _, p := range parts { + if strings.TrimSpace(p) != "" { + out = append(out, p) + } + } + return strings.Join(out, sep) +} + +// rejectRegionFlag registers --region on object commands (ls, cp, mv, sync, +// rm, stat, presign) so aws users get a directed explanation instead of an +// "unknown flag" error: on Latitude --region is a site (mb, access-keys) and +// the SigV4 signing region comes from the bucket's endpoint. +func rejectRegionFlag(cmd *cobra.Command) { + unsupportedAWSFlags(cmd, map[string]string{ + "region": "on object commands the signing region is derived from the bucket's endpoint. To disambiguate a repeated bucket name by location use --site; to override the SigV4 signing region use --signing-region ", + }) +} + +// outputExplicit reports whether the user asked for a structured format on +// the command line (-o/--output or --json), as opposed to LSH_OUTPUT or the +// config file. Secrets are only embedded in structured output on explicit +// request. +func outputExplicit(cmd *cobra.Command) bool { + if viper.GetBool("output_explicit") { + return true + } + if cmd == nil { + return false + } + changed := cmd.Flags().Changed("json") + if f := cmd.Root().PersistentFlags().Lookup("json"); f != nil && f.Changed { + changed = true + } + return changed +} + +// awsRegionPattern matches AWS-style region names (us-east-1, sa-east-1…), +// which users coming from aws type by reflex where a Latitude site is expected. +var awsRegionPattern = regexp.MustCompile(`^[a-z]{2}(?:-[a-z]+)+-[0-9]+$`) + +// looksLikeAWSRegion reports whether s is an AWS region name. +func looksLikeAWSRegion(s string) bool { + return awsRegionPattern.MatchString(strings.ToLower(strings.TrimSpace(s))) +} + +// emptyCell is the placeholder for empty table cells across the group. +const emptyCell = "-" + +// orEmptyCell returns s, or the shared placeholder when empty. +func orEmptyCell(s string) string { + if strings.TrimSpace(s) == "" { + return emptyCell + } + return s +} + +// saveNewKey stores a freshly created access key in the active profile under +// name, appending -2, -3… when that name already holds a different key so a +// saved key is never overwritten silently. It returns the name actually used +// and the profile name. Every creation path (access-keys create, configure, +// mb) goes through it so the clash policy is the same everywhere; rotate +// replaces in place with objectstorage.SaveKey instead. +func saveNewKey(cmd *cobra.Command, name string, k config.StoredAccessKey) (string, string, error) { + _, profileName, profile, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return "", "", err + } + existing := profile.ObjectStorageKeys() + final := name + for i := 2; ; i++ { + cur, taken := existing[final] + if !taken || cur.AccessKeyID == k.AccessKeyID { + break + } + final = fmt.Sprintf("%s-%d", name, i) + } + if _, err := objectstorage.SaveKey(profileFlag(cmd), final, k); err != nil { + return final, profileName, err + } + return final, profileName, nil +} diff --git a/cmd/s3/stat.go b/cmd/s3/stat.go new file mode 100644 index 0000000..b549e09 --- /dev/null +++ b/cmd/s3/stat.go @@ -0,0 +1,289 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + cobra "github.com/spf13/cobra" +) + +const flagStatVersionID = "version-id" + +// NewStatCmd builds `lsh s3 stat s3://bucket[/key]`. +func NewStatCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "stat s3://bucket[/key]", + Aliases: []string{"get", "describe", "head"}, + SuggestFor: []string{"show", "info"}, + Short: "Show a bucket or object", + Long: `Show a bucket or an object. + +For a bucket (s3://bucket) the details come from the Latitude API: id, names, +endpoint, site, signing region, class, versioning, object lock and which saved +access keys cover it. For an object (s3://bucket/key) the CLI issues a HEAD +request to the S3 endpoint: size, ETag, content type, last modified, version +and user metadata.`, + Example: ` lsh s3 stat s3://backups + lsh s3 stat s3://backups/2026/09/dump.sql + lsh s3 stat s3://backups/2026/09/dump.sql --version-id 3HL4kqtJlcpXroDTDmJ + lsh s3 stat s3://backups -o json`, + Args: cobra.ExactArgs(1), + RunE: runStat, + }) + addProjectFlag(cmd, true, "project ID or slug to disambiguate the bucket name") + addBucketFilterFlags(cmd) + cmd.Flags().String(flagStatVersionID, "", "inspect this object version") + rejectRegionFlag(cmd) + return cmd +} + +func runStat(cmd *cobra.Command, args []string) error { + ref, err := objectstorage.ParseRemote(args[0]) + if err != nil { + return printErr(err) + } + versionID, _ := cmd.Flags().GetString(flagStatVersionID) + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + if ref.Key == "" { + if versionID != "" { + return printErr(objectstorage.ErrUsagef("--version-id only applies to objects (s3://bucket/key)")) + } + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + if err := newResolver(cmd).FillSite(ctx, b); err != nil && lsh.Debug { + fmt.Fprintf(os.Stderr, "[s3] could not fill site for %s: %v\n", b.Display(), err) + } + if !isHuman() { + render([]renderer.ResponseData{NewBucketRow(b)}) + return nil + } + writeBucketStat(os.Stdout, b, savedKeysCovering(cmd, b)) + return nil + } + + if strings.HasSuffix(ref.Key, "/") { + return printErr(objectstorage.ErrUsagef("key %q ends with '/'; stat inspects a single object (use 'lsh s3 ls %s' to list a prefix)", ref.Key, ref)) + } + sess, err := openBucket(ctx, cmd, ref.Bucket, false) + if err != nil { + return printErr(err) + } + obj, err := statObject(ctx, sess, ref.Key, versionID) + if err != nil { + return printErr(err) + } + if !isHuman() { + render([]renderer.ResponseData{obj}) + return nil + } + writeObjectStat(os.Stdout, obj) + return nil +} + +// statObject issues the HEAD request and converts the result. +func statObject(ctx context.Context, sess *Session, key, versionID string) (objectstorage.Object, error) { + info, err := sess.Client.StatObject(ctx, sess.Bucket.BucketName, key, minio.StatObjectOptions{VersionID: versionID}) + if err != nil { + return objectstorage.Object{}, sess.humanize(err) + } + // A HEAD response never says whether the version is the latest, so the + // listing-only IsLatest flag stays unset; the version ID is still copied + // whenever the backend returns one. + obj := objectstorage.ObjectFromInfo(info, false) + if obj.Key == "" { + obj.Key = key + } + obj.Metadata = statUserMetadata(info) + return obj, nil +} + +// statUserMetadata collects x-amz-meta-* values from a HEAD response. minio fills +// UserMetadata with the prefix stripped; the raw headers are consulted too for +// backends that only surface them there. +func statUserMetadata(info minio.ObjectInfo) map[string]string { + out := map[string]string{} + for k, v := range info.UserMetadata { + lk := strings.ToLower(strings.TrimPrefix(strings.ToLower(k), "x-amz-meta-")) + if lk == "" || v == "" { + continue + } + out[lk] = v + } + for k, v := range info.Metadata { + lk := strings.ToLower(k) + if !strings.HasPrefix(lk, "x-amz-meta-") || len(v) == 0 { + continue + } + name := strings.TrimPrefix(lk, "x-amz-meta-") + if _, dup := out[name]; !dup && name != "" { + out[name] = v[0] + } + } + if len(out) == 0 { + return nil + } + return out +} + +// kv is one "Label: value" line of the human output. +type statKV struct { + label string + value string +} + +// statWriteKV prints aligned "Label: value" lines. +func statWriteKV(w io.Writer, pairs []statKV) { + width := 0 + for _, p := range pairs { + if len(p.label) > width { + width = len(p.label) + } + } + for _, p := range pairs { + fmt.Fprintf(w, "%-*s %s\n", width+1, p.label+":", p.value) + } +} + +// writeBucketStat renders the bucket details for humans. +func writeBucketStat(w io.Writer, b *objectstorage.Bucket, keys []statCoveringKey) { + pairs := []statKV{} + if !b.EndpointOverride { + pairs = append(pairs, + statKV{"ID", b.ID}, + statKV{"Name", b.Name}, + ) + } + pairs = append(pairs, statKV{"Bucket name (backend)", b.BucketName}) + if !b.EndpointOverride { + project := b.ProjectRef() + if b.ProjectName != "" && b.ProjectName != project { + project = fmt.Sprintf("%s (%s)", project, b.ProjectName) + } + pairs = append(pairs, + statKV{"Project", orEmptyCell(project)}, + statKV{"Class", orEmptyCell(b.StorageClass)}, + statKV{"Site", orEmptyCell(firstNonEmptyStr(b.Site, b.City))}, + ) + } + pairs = append(pairs, + statKV{"Endpoint", orEmptyCell(b.Endpoint)}, + statKV{"Signing region", orEmptyCell(b.SigningRegion)}, + ) + if !b.EndpointOverride { + created := "" + if b.CreatedAt != nil { + created = objectstorage.FormatTime(*b.CreatedAt) + } + pairs = append(pairs, + statKV{"Versioning", yesNo(b.Versioning)}, + statKV{"Locking", lockingLabel(b)}, + statKV{"Source", orEmptyCell(b.Source)}, + statKV{"Created", orEmptyCell(created)}, + ) + } + statWriteKV(w, pairs) + if b.EndpointOverride { + return + } + fmt.Fprintln(w, "Access keys covering this bucket:") + if len(keys) == 0 { + fmt.Fprintln(w, " (none saved)") + return + } + for _, k := range keys { + fmt.Fprintf(w, " %s (%s)\n", k.Name, k.Permission) + } +} + +// writeObjectStat renders the object details for humans. +func writeObjectStat(w io.Writer, o objectstorage.Object) { + pairs := []statKV{ + {"Key", o.Key}, + {"Size", fmt.Sprintf("%d (%s)", o.Size, objectstorage.HumanSize(o.Size))}, + {"Last modified", orEmptyCell(objectstorage.FormatTime(o.LastModified))}, + {"ETag", orEmptyCell(o.ETag)}, + {"Content type", orEmptyCell(o.ContentType)}, + } + if o.VersionID != "" { + pairs = append(pairs, statKV{"Version ID", o.VersionID}) + } + if o.StorageClass != "" { + pairs = append(pairs, statKV{"Storage class", o.StorageClass}) + } + if o.IsDeleteMarker { + pairs = append(pairs, statKV{"Delete marker", "yes"}) + } + if len(o.Metadata) > 0 { + names := make([]string, 0, len(o.Metadata)) + for k := range o.Metadata { + names = append(names, k) + } + sort.Strings(names) + parts := make([]string, 0, len(names)) + for _, k := range names { + parts = append(parts, k+"="+o.Metadata[k]) + } + pairs = append(pairs, statKV{"Metadata", strings.Join(parts, " ")}) + } + statWriteKV(w, pairs) +} + +// statCoveringKey is a saved key that grants access to the bucket. +type statCoveringKey struct { + Name string + Permission string +} + +// savedKeysCovering lists the saved keys of the active profile that cover b, +// sorted by name. Profile problems are not errors for stat: the list is +// simply empty. +func savedKeysCovering(cmd *cobra.Command, b *objectstorage.Bucket) []statCoveringKey { + _, _, profile, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return nil + } + return statCoveringKeys(profile.ObjectStorageKeys(), b) +} + +// statCoveringKeys applies the class/site/project compatibility rules and +// StoredAccessKey.Covers to the saved keys. +func statCoveringKeys(keys map[string]config.StoredAccessKey, b *objectstorage.Bucket) []statCoveringKey { + var out []statCoveringKey + for name, k := range keys { + if k.Scope == config.ScopeUnknown { + continue + } + if k.StorageClass != "" && b.StorageClass != "" && k.StorageClass != b.StorageClass { + continue + } + if k.Site != "" && b.Site != "" && !strings.EqualFold(k.Site, b.Site) { + continue + } + if k.ProjectID != "" && b.ProjectID != "" && k.ProjectID != b.ProjectID { + continue + } + if !k.Covers(b.ID, false) { + continue + } + perm := k.Permission(b.ID) + if k.Scope == config.ScopeFullAccess { + perm = "fullaccess" + } + out = append(out, statCoveringKey{Name: name, Permission: perm}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} diff --git a/cmd/s3/stat_test.go b/cmd/s3/stat_test.go new file mode 100644 index 0000000..238e7e5 --- /dev/null +++ b/cmd/s3/stat_test.go @@ -0,0 +1,284 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" + "github.com/minio/minio-go/v7" +) + +func TestStatObject_ReturnsMetadata(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + obj := srv.AddObject("backups-7f3a", "2026/09/dump.sql", []byte("select 1;"), "application/sql") + obj.Metadata = map[string]string{"owner": "ops", "env": "prod"} + sess := newRbStatSession(t, srv, "backups-7f3a") + + got, err := statObject(context.Background(), sess, "2026/09/dump.sql", "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Key != "2026/09/dump.sql" { + t.Errorf("key = %q", got.Key) + } + if got.Size != int64(len("select 1;")) { + t.Errorf("size = %d", got.Size) + } + if got.ContentType != "application/sql" { + t.Errorf("content type = %q", got.ContentType) + } + if got.ETag == "" || strings.Contains(got.ETag, `"`) { + t.Errorf("etag should be set and unquoted, got %q", got.ETag) + } + if got.LastModified.IsZero() { + t.Errorf("last modified should be set") + } + if got.Metadata["owner"] != "ops" || got.Metadata["env"] != "prod" { + t.Errorf("metadata = %v", got.Metadata) + } + if got.VersionID != "" { + t.Errorf("unversioned object should have no version id, got %q", got.VersionID) + } + if got.IsLatest != nil { + t.Errorf("a HEAD response carries no is_latest flag, got %v", *got.IsLatest) + } + + // Only a HEAD reaches the backend. + reqs := srv.Requests() + if len(reqs) != 1 || reqs[0].Method != http.MethodHead { + t.Errorf("expected a single HEAD, got %+v", reqs) + } + + var out bytes.Buffer + writeObjectStat(&out, got) + text := out.String() + for _, want := range []string{"Key:", "2026/09/dump.sql", "Size:", "9 (9 Bytes)", "Content type:", "application/sql", "Metadata:", "env=prod owner=ops"} { + if !strings.Contains(text, want) { + t.Errorf("output lacks %q:\n%s", want, text) + } + } +} + +func TestStatObject_VersionID(t *testing.T) { + srv := s3test.New() + defer srv.Close() + bkt := srv.CreateBucket("backups-7f3a") + bkt.Versioned = true + // Pin distinct version IDs (the fake server would reuse v1 for both). + srv.AddObject("backups-7f3a", "a.txt", []byte("one"), "text/plain").VersionID = "v1" + srv.AddObject("backups-7f3a", "a.txt", []byte("three"), "text/plain").VersionID = "v2" + sess := newRbStatSession(t, srv, "backups-7f3a") + + got, err := statObject(context.Background(), sess, "a.txt", "v1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Size != 3 || got.VersionID != "v1" { + t.Errorf("got size=%d version=%q, want the old version", got.Size, got.VersionID) + } + // HEAD never reports whether a version is the latest (only listings do), so + // stat must not fabricate is_latest even when --version-id was given. + if got.IsLatest != nil { + t.Errorf("is_latest must stay unset for stat, got %v", *got.IsLatest) + } + reqs := srv.Requests() + if len(reqs) != 1 || reqs[0].Query.Get("versionId") != "v1" { + t.Errorf("HEAD should carry versionId=v1, got %+v", reqs) + } +} + +func TestStatObject_NotFoundIsExit3(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + sess := newRbStatSession(t, srv, "backups-7f3a") + + _, err := statObject(context.Background(), sess, "missing.txt", "") + if err == nil { + t.Fatalf("expected an error") + } + if exitcode.Of(err) != exitcode.NotFound { + t.Errorf("exit code = %d, want %d; err=%v", exitcode.Of(err), exitcode.NotFound, err) + } + if !strings.Contains(err.Error(), "not found") { + t.Errorf("message %q should say not found", err.Error()) + } +} + +func TestStatObject_AccessDeniedIsExit5(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + srv.DenyAll = true + sess := newRbStatSession(t, srv, "backups-7f3a") + + _, err := statObject(context.Background(), sess, "a.txt", "") + if err == nil || exitcode.Of(err) != exitcode.Permission { + t.Fatalf("expected exit 5, got %v (code %d)", err, exitcode.Of(err)) + } + if strings.Contains(err.Error(), "SK") { + t.Errorf("error must not include the secret: %q", err.Error()) + } +} + +func TestStatUserMetadata(t *testing.T) { + info := minio.ObjectInfo{ + UserMetadata: map[string]string{"Owner": "ops", "X-Amz-Meta-Env": "prod"}, + Metadata: http.Header{"X-Amz-Meta-Team": {"storage"}, "Content-Type": {"text/plain"}, "X-Amz-Meta-Owner": {"ignored"}}, + } + got := statUserMetadata(info) + want := map[string]string{"owner": "ops", "env": "prod", "team": "storage"} + if len(got) != len(want) { + t.Fatalf("metadata = %v, want %v", got, want) + } + for k, v := range want { + if got[k] != v { + t.Errorf("metadata[%q] = %q, want %q", k, got[k], v) + } + } + if statUserMetadata(minio.ObjectInfo{}) != nil { + t.Errorf("no metadata should yield nil") + } +} + +func TestWriteBucketStat(t *testing.T) { + created := time.Date(2026, 9, 7, 15, 12, 1, 0, time.UTC) + b := &objectstorage.Bucket{ + ID: "bkt_1Gjbang9n0L2w", + Name: "backups", + BucketName: "backups-7f3a", + Endpoint: "https://s3.us-central-1.storage.sh", + SigningRegion: "us-central-1", + StorageClass: "standard", + Site: "DAL", + ProjectID: "proj_1", + ProjectSlug: "my-project", + Versioning: true, + Locking: true, + RetentionMode: "GOVERNANCE", + RetentionDays: 30, + Source: "default", + CreatedAt: &created, + } + var out bytes.Buffer + writeBucketStat(&out, b, []statCoveringKey{{Name: "lsh-me-standard", Permission: "fullaccess"}, {Name: "ci", Permission: "readonly"}}) + text := out.String() + for _, want := range []string{ + "ID:", "bkt_1Gjbang9n0L2w", + "Name:", "backups", + "Bucket name (backend):", "backups-7f3a", + "Project:", "my-project", + "Class:", "standard", + "Site:", "DAL", + "Endpoint:", "https://s3.us-central-1.storage.sh", + "Signing region:", "us-central-1", + "Versioning:", "yes", + "Locking:", "GOVERNANCE (30d)", + "Source:", "default", + "Created:", + "Access keys covering this bucket:", + " lsh-me-standard (fullaccess)", + " ci (readonly)", + } { + if !strings.Contains(text, want) { + t.Errorf("output lacks %q:\n%s", want, text) + } + } + // Labels are aligned: every value starts in the same column. + col := -1 + for _, line := range strings.Split(strings.TrimSpace(text), "\n") { + if !strings.Contains(line, ":") || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "Access keys") { + continue + } + i := strings.Index(line, ": ") + rest := line[i+1:] + valueCol := i + 1 + (len(rest) - len(strings.TrimLeft(rest, " "))) + if col == -1 { + col = valueCol + } else if valueCol != col { + t.Errorf("value column %d differs from %d in %q", valueCol, col, line) + } + } + + out.Reset() + writeBucketStat(&out, b, nil) + if !strings.Contains(out.String(), "(none saved)") { + t.Errorf("empty key list should print (none saved):\n%s", out.String()) + } +} + +func TestWriteBucketStat_EndpointOverride(t *testing.T) { + b := &objectstorage.Bucket{BucketName: "raw-bucket", Endpoint: "https://objects.tyo4.storage.sh", SigningRegion: "tyo4", EndpointOverride: true} + var out bytes.Buffer + writeBucketStat(&out, b, nil) + text := out.String() + if strings.Contains(text, "ID:") || strings.Contains(text, "Access keys") { + t.Errorf("API-only fields must be skipped in endpoint-override mode:\n%s", text) + } + if !strings.Contains(text, "raw-bucket") || !strings.Contains(text, "tyo4") { + t.Errorf("backend name and signing region should be shown:\n%s", text) + } +} + +func TestStatCoveringKeys(t *testing.T) { + b := &objectstorage.Bucket{ID: "bkt_1", StorageClass: "standard", Site: "DAL", ProjectID: "proj_1"} + keys := map[string]config.StoredAccessKey{ + "full": {Scope: config.ScopeFullAccess, StorageClass: "standard", ProjectID: "proj_1"}, + "limited-rw": {Scope: config.ScopeLimitedAccess, StorageClass: "standard", Buckets: map[string]string{"bkt_1": "rw"}}, + "limited-ro": {Scope: config.ScopeLimitedAccess, StorageClass: "standard", Buckets: map[string]string{"bkt_1": "readonly"}}, + "other-bkt": {Scope: config.ScopeLimitedAccess, StorageClass: "standard", Buckets: map[string]string{"bkt_2": "rw"}}, + "other-class": {Scope: config.ScopeFullAccess, StorageClass: "high_performance", Site: "TYO4"}, + "other-proj": {Scope: config.ScopeFullAccess, StorageClass: "standard", ProjectID: "proj_2"}, + "unknown": {Scope: config.ScopeUnknown}, + } + got := statCoveringKeys(keys, b) + if len(got) != 3 { + t.Fatalf("covering keys = %+v, want full, limited-ro, limited-rw", got) + } + if got[0].Name != "full" || got[0].Permission != "fullaccess" { + t.Errorf("got[0] = %+v", got[0]) + } + if got[1].Name != "limited-ro" || got[1].Permission != "readonly" { + t.Errorf("got[1] = %+v", got[1]) + } + if got[2].Name != "limited-rw" || got[2].Permission != "rw" { + t.Errorf("got[2] = %+v", got[2]) + } +} + +// TestBucketRowStructuredWithoutAPIPayload covers endpoint-override mode: there +// is no API document to marshal, so the row used to render as "{}" while the +// human output showed the bucket name, endpoint and signing region. +func TestBucketRowStructuredWithoutAPIPayload(t *testing.T) { + b := &objectstorage.Bucket{ + Name: "backups-7f3a", BucketName: "backups-7f3a", + Endpoint: "https://s3.us-central-1.storage.sh", SigningRegion: "us-central-1", + EndpointOverride: true, + } + raw, err := json.Marshal(NewBucketRow(b)) + if err != nil { + t.Fatal(err) + } + var doc map[string]string + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatalf("unmarshal %s: %v", raw, err) + } + for k, want := range map[string]string{ + "name": "backups-7f3a", "bucket_name": "backups-7f3a", + "endpoint": "https://s3.us-central-1.storage.sh", "signing_region": "us-central-1", + } { + if doc[k] != want { + t.Errorf("%s = %q, want %q (in %s)", k, doc[k], want, raw) + } + } +} diff --git a/cmd/s3/sync.go b/cmd/s3/sync.go new file mode 100644 index 0000000..ce9d464 --- /dev/null +++ b/cmd/s3/sync.go @@ -0,0 +1,237 @@ +package s3 + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "time" + + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/spf13/cobra" +) + +// syncOptions are the flags specific to sync. +type syncOptions struct { + // Delete removes destination entries that are absent from the source. + Delete bool + // SizeOnly compares only sizes. + SizeOnly bool + // ExactTimestamps makes same-sized downloads skip only when the + // timestamps match exactly. + ExactTimestamps bool +} + +// NewSyncCmd builds `lsh s3 sync `. +func NewSyncCmd() *cobra.Command { + var f transferFlags + var so syncOptions + cmd := newCmd(&cobra.Command{ + Use: "sync ", + Short: "Synchronize directories and prefixes", + Long: `Recursively copy new and changed files between a local directory and a +prefix, or between two prefixes on the same endpoint. + +An entry is transferred when it is missing at the destination, its size +differs, or the source is newer than the destination. --size-only ignores +timestamps; --exact-timestamps (S3 -> local) skips same-sized files only when +the timestamps match exactly. ETags are never compared. --delete removes +destination entries that no longer exist at the source; --exclude/--include +apply to paths relative to the source (and to the destination for --delete). +Deletions run after the copies and are skipped entirely if any copy failed, so +a partial run never leaves the destination without both versions; re-run the +sync to apply them. They also never leave the destination tree: a candidate +reached through a directory symlink is reported and kept. + +Output lines use upload:/download:/copy:/delete:. --dry-run (or --dryrun) +prints the plan without writing or deleting anything.`, + Example: ` lsh s3 sync ./site s3://www + lsh s3 sync s3://backups/2026/ ./backups/2026/ + lsh s3 sync ./logs s3://logs/host-1/ --exclude "*" --include "*.log" --delete --dryrun + lsh s3 sync s3://backups/ s3://archive/backups/ --size-only`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return runSyncCommand(cmd, args, &f, so) + }, + }) + addTransferFlags(cmd, &f, false) + cmd.Flags().BoolVar(&so.Delete, "delete", false, "delete destination entries that are absent from the source (skipped when a copy fails)") + cmd.Flags().BoolVar(&so.SizeOnly, "size-only", false, "compare by size only, ignoring timestamps") + cmd.Flags().BoolVar(&so.ExactTimestamps, "exact-timestamps", false, "S3 -> local: skip same-sized files only when the timestamps match exactly") + return cmd +} + +// runSyncCommand is the RunE body of sync. +func runSyncCommand(cmd *cobra.Command, args []string, f *transferFlags, so syncOptions) error { + srcRef, dstRef, err := classifyOperands(args[0], args[1], false, true) + if err != nil { + return printErr(err) + } + opts, err := buildTransferOptions(cmd, f) + if err != nil { + return printErr(err) + } + opts.Recursive = true + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + src, dst, err := openEndpoints(ctx, cmd, srcRef, dstRef, false) + if err != nil { + return printErr(err) + } + plan, err := buildSyncPlan(ctx, src, dst, opts, so) + if err != nil { + return printErr(err) + } + results, err := runTransfers(ctx, plan, opts, os.Stdout, os.Stderr) + return finishTransfer(plan, opts, results, err) +} + +// buildSyncPlan enumerates both sides and decides what to transfer and, with +// --delete, what to remove. Only read-only calls are made. +func buildSyncPlan(ctx context.Context, src, dst endpoint, opts *transferOptions, so syncOptions) (*transferPlan, error) { + plan := &transferPlan{Src: src, Dst: dst} + if err := checkCrossEndpoint(src, dst); err != nil { + return nil, err + } + if src.local() && dst.local() { + return nil, objectstorage.ErrUsagef("both operands are local paths; at least one must be an s3:// location") + } + + srcEntries, err := enumerate(ctx, src, opts.FollowSymlinks) + if err != nil { + return nil, err + } + dstEntries, err := enumerateDestination(ctx, dst, opts.FollowSymlinks) + if err != nil { + return nil, err + } + dstByRel := make(map[string]entry, len(dstEntries)) + for _, e := range dstEntries { + dstByRel[e.Rel] = e + } + + op := opFor(src, dst, false) + download := src.remote() && dst.local() + dstPrefix := "" + if dst.remote() { + dstPrefix = objectstorage.NormalizePrefix(dst.Ref.Key) + } + inSource := make(map[string]bool, len(srcEntries)) + for _, e := range srcEntries { + if !opts.Filters.Include(e.Rel) { + continue + } + inSource[e.Rel] = true + d, exists := dstByRel[e.Rel] + if !syncNeedsTransfer(e, d, exists, so, download) { + continue + } + item := transferItem{Op: op, Size: e.Size, ModTime: e.ModTime} + if src.remote() { + item.Src, item.SrcKey = src.remoteURI(e.Key), e.Key + } else { + item.Src, item.SrcPath = displayLocal(e.Path), e.Path + } + if dst.remote() { + key := dstPrefix + e.Rel + if sameObject(src, dst, e.Key, key) { + continue + } + item.Dst, item.DstKey = dst.remoteURI(key), key + } else { + path, display, err := localRelDestination(dst.Ref.Raw, e.Rel) + if err != nil { + return nil, err + } + item.Dst, item.DstPath = display, path + } + plan.Items = append(plan.Items, item) + } + + if so.Delete { + // A local destination may contain directory symlinks, which the + // enumeration follows by default. Deleting through one of them would + // remove files outside the tree the user pointed at, so a candidate + // whose parent directory resolves outside the destination root is + // skipped with a warning. Removing a link that itself lives inside the + // root stays allowed: only the link goes away, not its target. + root := "" + if dst.local() { + if resolved, err := filepath.EvalSymlinks(dst.Ref.Raw); err == nil { + root = resolved + } + } + for _, d := range dstEntries { + if inSource[d.Rel] || !opts.Filters.Include(d.Rel) { + continue + } + if root != "" && !deleteInsideRoot(root, d.Path) { + objectstorage.Warnf("not deleting %s: it resolves outside %s through a symbolic link; pass --no-follow-symlinks to leave links out of the sync", displayLocal(d.Path), displayLocal(dst.Ref.Raw)) + continue + } + del := deleteItem{Path: d.Path, Key: d.Key} + if dst.remote() { + del.Display = dst.remoteURI(d.Key) + } else { + del.Display = displayLocal(d.Path) + } + plan.Deletes = append(plan.Deletes, del) + } + } + return plan, nil +} + +// deleteInsideRoot reports whether removing path only affects the destination +// tree: its parent directory, with every symbolic link resolved, must be root +// or below it. root is expected to be resolved already. +func deleteInsideRoot(root, path string) bool { + parent, err := filepath.EvalSymlinks(filepath.Dir(path)) + if err != nil { + return false + } + rel, err := filepath.Rel(root, parent) + if err != nil { + return false + } + return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))) +} + +// enumerateDestination lists the destination; a local directory that does +// not exist yet is simply empty. +func enumerateDestination(ctx context.Context, dst endpoint, followSymlinks bool) ([]entry, error) { + if dst.remote() { + return enumerate(ctx, dst, followSymlinks) + } + st, err := os.Stat(dst.Ref.Raw) + switch { + case errors.Is(err, os.ErrNotExist): + return nil, nil + case err != nil: + return nil, localErr(err) + case !st.IsDir(): + return nil, objectstorage.ErrUsagef("%s is not a directory", dst.Ref.Raw) + } + return enumerate(ctx, dst, followSymlinks) +} + +// syncNeedsTransfer applies the comparison rules. Timestamps are compared at +// second precision because S3 does not keep sub-second modification times. +func syncNeedsTransfer(src, dst entry, exists bool, so syncOptions, download bool) bool { + if !exists { + return true + } + if src.Size != dst.Size { + return true + } + if so.SizeOnly { + return false + } + s, d := src.ModTime.Truncate(time.Second), dst.ModTime.Truncate(time.Second) + if download && so.ExactTimestamps { + return !s.Equal(d) + } + return s.After(d) +} diff --git a/cmd/s3/transfer.go b/cmd/s3/transfer.go new file mode 100644 index 0000000..afc383c --- /dev/null +++ b/cmd/s3/transfer.go @@ -0,0 +1,1405 @@ +package s3 + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" + "golang.org/x/term" +) + +// Shared machinery for cp, mv and sync: operand classification, the aws +// destination rules, local/remote enumeration, the transfer loop and its +// output. Nothing in this file depends on cobra except the flag registration +// helpers at the bottom, so the core can be tested against s3test. + +// Transfer operation names (also the aws output prefixes). +const ( + opUpload = "upload" + opDownload = "download" + opCopy = "copy" + opMove = "move" + opDelete = "delete" +) + +// stdinPartSize is the smallest multipart part size used for stdin uploads +// (minio's default). S3 allows 10000 parts, so it covers streams up to +// stdinMaxPlainSize; --expected-size grows the parts beyond that. +const stdinPartSize = 16 * 1024 * 1024 + +// stdinMaxParts is the S3 limit on parts per multipart upload. +const stdinMaxParts = 10000 + +// stdinMaxPlainSize is the largest stream that fits in 10000 parts of +// stdinPartSize (160 GiB) without --expected-size. +const stdinMaxPlainSize = int64(stdinPartSize) * stdinMaxParts + +// partialSuffix marks a download in progress; the file is renamed on success. +const partialSuffix = ".lsh-partial" + +// transferOptions carries every knob the transfer core understands. The +// cobra layer fills it from flags; tests build it directly. +type transferOptions struct { + Recursive bool + Filters *objectstorage.Filters + + ContentType string + CacheControl string + ContentEncoding string + ContentDisposition string + ContentLanguage string + Expires time.Time + Metadata map[string]string + NoGuessMime bool + + NoOverwrite bool + ExpectedSize int64 + FollowSymlinks bool + + // Move deletes the source after a successful copy (mv). + Move bool + // DryRun plans and prints without writing anything. + DryRun bool + + // Human prints aws-style lines to Out; otherwise results are only + // collected for the structured renderer. + Human bool + Quiet bool + OnlyShowErrors bool + // ShowProgress enables the single-line progress meter on Hints. + ShowProgress bool + + // Stdin/Stdout back the "-" operands (defaults: os.Stdin/os.Stdout). + Stdin io.Reader + Stdout io.Writer +} + +func (o *transferOptions) showLines() bool { + return o.Human && !o.Quiet && !o.OnlyShowErrors +} + +func (o *transferOptions) stdin() io.Reader { + if o.Stdin != nil { + return o.Stdin + } + return os.Stdin +} + +func (o *transferOptions) stdout() io.Writer { + if o.Stdout != nil { + return o.Stdout + } + return os.Stdout +} + +// endpoint is one side of a transfer: a parsed operand plus, for remote +// references, the open bucket session. +type endpoint struct { + Ref objectstorage.Ref + Sess *Session +} + +func (e endpoint) remote() bool { return e.Ref.Remote } +func (e endpoint) stdio() bool { return e.Ref.Stdio } +func (e endpoint) local() bool { return !e.Ref.Remote && !e.Ref.Stdio } + +// remoteURI renders s3:/// using the bucket token as typed. +func (e endpoint) remoteURI(key string) string { return e.Ref.WithKey(key).String() } + +// transferItem is one planned operation. +type transferItem struct { + Op string + // Display strings for output lines. + Src, Dst string + // SrcPath/DstPath are local paths; SrcKey/DstKey are object keys. The + // unused pair is empty. Stdio sides have both empty. + SrcPath, SrcKey string + DstPath, DstKey string + // Size is known for local sources and listed/stat'ed remote sources. + Size int64 + ModTime time.Time +} + +// transferPlan is the outcome of planning: the two endpoints and the items. +type transferPlan struct { + Src, Dst endpoint + Items []transferItem + // Deletes are destination entries removed by `sync --delete`. + Deletes []deleteItem +} + +// deleteItem is one `sync --delete` removal. +type deleteItem struct { + Display string + Path string // local + Key string // remote +} + +// entry is a file or object discovered while enumerating a side, keyed by +// its path relative to the source directory / prefix (forward slashes). +type entry struct { + Rel string + Path string // local absolute-ish path (as joined from the operand) + Key string // remote key + Size int64 + ModTime time.Time +} + +// --------------------------------------------------------------------------- +// Operand validation + +// classifyOperands validates the source/destination combination shared by +// cp, mv and sync and returns the parsed refs. +func classifyOperands(srcArg, dstArg string, move, sync bool) (objectstorage.Ref, objectstorage.Ref, error) { + src, err := objectstorage.ParseTransferArg(srcArg) + if err != nil { + return src, objectstorage.Ref{}, err + } + dst, err := objectstorage.ParseTransferArg(dstArg) + if err != nil { + return src, dst, err + } + switch { + case !src.Remote && !dst.Remote: + if src.Stdio || dst.Stdio { + return src, dst, objectstorage.ErrUsagef("one of and must be an s3:// location") + } + return src, dst, objectstorage.ErrUsagef("both operands are local paths; at least one must be an s3:// location (use cp(1) for local copies)") + case src.Stdio && sync, dst.Stdio && sync: + return src, dst, objectstorage.ErrUsagef("sync does not support '-' (stdin/stdout)") + case src.Stdio && move, dst.Stdio && move: + return src, dst, objectstorage.ErrUsagef("mv does not support '-' (stdin/stdout); use cp") + } + return src, dst, nil +} + +// sameEndpoint reports whether two buckets live on the same S3 endpoint. +func sameEndpoint(a, b *objectstorage.Bucket) bool { + norm := func(s string) string { return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(s), "/")) } + return norm(a.Endpoint) == norm(b.Endpoint) +} + +// checkCrossEndpoint rejects server-side copies between different endpoints. +func checkCrossEndpoint(src, dst endpoint) error { + if !src.remote() || !dst.remote() { + return nil + } + if sameEndpoint(src.Sess.Bucket, dst.Sess.Bucket) { + return nil + } + return objectstorage.ErrUsagef("buckets %s (%s) and %s (%s) are on different endpoints; server-side copy is only possible within one endpoint — download and re-upload, or use rclone with 'lsh s3 configure export --format rclone'", + src.Sess.Bucket.Display(), src.Sess.Bucket.Endpoint, dst.Sess.Bucket.Display(), dst.Sess.Bucket.Endpoint) +} + +// needsStreamedCopy reports whether a failed server-side copy should be retried +// by streaming the object. Only an authorization failure between two buckets +// that resolved different credentials qualifies: any other error (missing +// object, quota, backend outage) would fail the same way twice. +func needsStreamedCopy(src, dst endpoint, err error) bool { + if !src.remote() || !dst.remote() { + return false + } + if minio.ToErrorResponse(err).Code != "AccessDenied" { + return false + } + return src.Sess.Cred.AccessKeyID != dst.Sess.Cred.AccessKeyID +} + +// streamCopy copies one object through the client: GET with the source session, +// PUT with the destination one. It is the fallback for buckets a single +// credential cannot span, so cp/mv/sync keep working instead of reporting an +// access denial the user cannot act on. +func streamCopy(ctx context.Context, src, dst endpoint, item transferItem, opts *transferOptions, res *objectstorage.TransferResult, hints io.Writer, label string) error { + obj, err := src.Sess.Client.GetObject(ctx, src.Sess.Bucket.BucketName, item.SrcKey, minio.GetObjectOptions{}) + if err != nil { + return err + } + defer obj.Close() + stat, err := obj.Stat() + if err != nil { + // The read is the source's; blaming the destination (as the shared + // humanizer would) points at the wrong bucket and the wrong key. + return src.Sess.humanize(err) + } + meter := meterFor(opts, hints, label, stat.Size) + defer meter.finish() + put := streamedPutOptions(stat, opts, progressReader(meter)) + contentType := put.ContentType + info, err := dst.Sess.Client.PutObject(ctx, dst.Sess.Bucket.BucketName, item.DstKey, obj, stat.Size, put) + if err != nil { + return dst.Sess.humanize(err) + } + res.ETag, res.ContentType = strings.Trim(info.ETag, `"`), contentType + if info.Size > 0 { + res.Size = info.Size + } + return nil +} + +// streamedPutOptions makes a streamed copy keep what a server-side copy would +// have kept: the source's user metadata and its caching/content headers. The +// explicit flags still win — putOptions applies them first and this only fills +// what they left empty — so a cross-credential copy does not silently rewrite +// an object's metadata. +func streamedPutOptions(stat minio.ObjectInfo, opts *transferOptions, progress io.Reader) minio.PutObjectOptions { + put := putOptions(firstNonEmptyStr(opts.ContentType, stat.ContentType), opts, progress) + if len(put.UserMetadata) == 0 { + put.UserMetadata = userMetadataOf(stat.Metadata) + } + if put.CacheControl == "" { + put.CacheControl = stat.Metadata.Get("Cache-Control") + } + if put.ContentEncoding == "" { + put.ContentEncoding = firstNonEmptyStr(stat.ContentEncoding, stat.Metadata.Get("Content-Encoding")) + } + if put.ContentDisposition == "" { + put.ContentDisposition = stat.Metadata.Get("Content-Disposition") + } + if put.ContentLanguage == "" { + put.ContentLanguage = stat.Metadata.Get("Content-Language") + } + if put.Expires.IsZero() { + put.Expires = stat.Expires + } + return put +} + +// userMetadataOf extracts the x-amz-meta-* response headers, stripped of the +// prefix. ObjectInfo.UserMetadata only carries them against MinIO servers, so +// the raw headers are the portable source. +func userMetadataOf(h http.Header) map[string]string { + out := map[string]string{} + for key, values := range h { + lower := strings.ToLower(key) + if name := strings.TrimPrefix(lower, "x-amz-meta-"); name != lower && len(values) > 0 { + out[name] = values[0] + } + } + if len(out) == 0 { + return nil + } + return out +} + +// sameObject reports whether both endpoints name the same object. +func sameObject(src, dst endpoint, srcKey, dstKey string) bool { + if !src.remote() || !dst.remote() { + return false + } + return sameEndpoint(src.Sess.Bucket, dst.Sess.Bucket) && + src.Sess.Bucket.BucketName == dst.Sess.Bucket.BucketName && srcKey == dstKey +} + +// --------------------------------------------------------------------------- +// Planning (cp / mv) + +// buildPlan enumerates the source and applies the aws destination rules. It +// only performs read-only calls (stat, list), so it is safe under --dry-run. +func buildPlan(ctx context.Context, src, dst endpoint, opts *transferOptions) (*transferPlan, error) { + plan := &transferPlan{Src: src, Dst: dst} + op := opFor(src, dst, opts.Move) + + if err := checkCrossEndpoint(src, dst); err != nil { + return nil, err + } + + switch { + case src.stdio(): + if opts.Recursive { + return nil, objectstorage.ErrUsagef("--recursive cannot be combined with stdin ('-')") + } + if dst.Ref.IsDir() { + return nil, objectstorage.ErrUsagef("%s names a prefix; uploading from stdin needs a full object key (s3:///)", dst.Ref) + } + plan.Items = append(plan.Items, transferItem{Op: op, Src: "-", Dst: dst.remoteURI(dst.Ref.Key), DstKey: dst.Ref.Key, Size: opts.ExpectedSize}) + return plan, nil + + case dst.stdio(): + if opts.Recursive { + return nil, objectstorage.ErrUsagef("--recursive cannot be combined with stdout ('-')") + } + if src.Ref.Key == "" || strings.HasSuffix(src.Ref.Key, "/") { + return nil, objectstorage.ErrUsagef("%s names a prefix, not an object; streaming to stdout needs a single object key", src.Ref) + } + info, err := src.Sess.Client.StatObject(ctx, src.Sess.Bucket.BucketName, src.Ref.Key, minio.StatObjectOptions{}) + if err != nil { + return nil, src.Sess.humanize(err) + } + plan.Items = append(plan.Items, transferItem{Op: op, Src: src.remoteURI(src.Ref.Key), Dst: "-", SrcKey: src.Ref.Key, Size: info.Size, ModTime: info.LastModified}) + return plan, nil + + case opts.Recursive: + return buildRecursivePlan(ctx, plan, op, opts) + } + + // Single object / file. + var item transferItem + item.Op = op + switch { + case src.local(): + st, err := os.Stat(src.Ref.Raw) + if err != nil { + return nil, localErr(err) + } + if st.IsDir() { + return nil, objectstorage.ErrUsagef("%s is a directory; use --recursive to copy its contents", src.Ref.Raw) + } + name := filepath.Base(src.Ref.Raw) + key := dst.Ref.Key + if dst.Ref.IsDir() { + key = objectstorage.JoinKey(dst.Ref.Key, name) + } + item.Src, item.SrcPath = displayLocal(src.Ref.Raw), src.Ref.Raw + item.Dst, item.DstKey = dst.remoteURI(key), key + item.Size, item.ModTime = st.Size(), st.ModTime() + if !opts.Filters.Include(name) { + return plan, nil + } + default: // remote source + if src.Ref.Key == "" || strings.HasSuffix(src.Ref.Key, "/") { + return nil, objectstorage.ErrUsagef("%s names a prefix, not an object; use --recursive to copy everything under it", src.Ref) + } + info, err := src.Sess.Client.StatObject(ctx, src.Sess.Bucket.BucketName, src.Ref.Key, minio.StatObjectOptions{}) + if err != nil { + return nil, src.Sess.humanize(err) + } + name := objectstorage.BaseName(src.Ref.Key) + item.Src, item.SrcKey = src.remoteURI(src.Ref.Key), src.Ref.Key + item.Size, item.ModTime = info.Size, info.LastModified + if dst.remote() { + key := dst.Ref.Key + if dst.Ref.IsDir() { + key = objectstorage.JoinKey(dst.Ref.Key, name) + } + if sameObject(src, dst, src.Ref.Key, key) { + return nil, objectstorage.ErrUsagef("source and destination are the same object (%s)", item.Src) + } + item.Dst, item.DstKey = dst.remoteURI(key), key + } else { + path, display, err := localDestination(dst.Ref.Raw, name) + if err != nil { + return nil, err + } + item.Dst, item.DstPath = display, path + } + if !opts.Filters.Include(name) { + return plan, nil + } + } + plan.Items = append(plan.Items, item) + return plan, nil +} + +// buildRecursivePlan handles --recursive: both sides are directories / +// prefixes and every relative path is mirrored. +func buildRecursivePlan(ctx context.Context, plan *transferPlan, op string, opts *transferOptions) (*transferPlan, error) { + src, dst := plan.Src, plan.Dst + entries, err := enumerate(ctx, src, opts.FollowSymlinks) + if err != nil { + return nil, err + } + dstPrefix := "" + if dst.remote() { + dstPrefix = objectstorage.NormalizePrefix(dst.Ref.Key) + } + for _, e := range entries { + if !opts.Filters.Include(e.Rel) { + continue + } + item := transferItem{Op: op, Size: e.Size, ModTime: e.ModTime} + if src.remote() { + item.Src, item.SrcKey = src.remoteURI(e.Key), e.Key + } else { + item.Src, item.SrcPath = displayLocal(e.Path), e.Path + } + if dst.remote() { + key := dstPrefix + e.Rel + if sameObject(src, dst, e.Key, key) { + continue + } + item.Dst, item.DstKey = dst.remoteURI(key), key + } else { + path, display, err := localRelDestination(dst.Ref.Raw, e.Rel) + if err != nil { + return nil, err + } + item.Dst, item.DstPath = display, path + } + plan.Items = append(plan.Items, item) + } + return plan, nil +} + +// opFor picks the output verb for a direction. +func opFor(src, dst endpoint, move bool) string { + if move { + return opMove + } + switch { + case src.remote() && dst.remote(): + return opCopy + case src.remote(): + return opDownload + default: + return opUpload + } +} + +// --------------------------------------------------------------------------- +// Enumeration + +// enumerate lists a directory (local) or a prefix (remote) recursively and +// returns entries keyed by relative path, sorted. +func enumerate(ctx context.Context, e endpoint, followSymlinks bool) ([]entry, error) { + var out []entry + var err error + if e.remote() { + out, err = listRemote(ctx, e.Sess, objectstorage.NormalizePrefix(e.Ref.Key)) + } else { + out, err = walkLocal(e.Ref.Raw, followSymlinks) + } + if err != nil { + return nil, err + } + sort.Slice(out, func(i, j int) bool { return out[i].Rel < out[j].Rel }) + return out, nil +} + +// listRemote lists every object under prefix. Directory markers (keys ending +// in "/") are skipped like aws does. +func listRemote(ctx context.Context, sess *Session, prefix string) ([]entry, error) { + var out []entry + for info := range sess.Client.ListObjects(ctx, sess.Bucket.BucketName, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if info.Err != nil { + return nil, sess.humanize(info.Err) + } + if strings.HasSuffix(info.Key, "/") || len(info.Key) <= len(prefix) { + continue + } + out = append(out, entry{Rel: info.Key[len(prefix):], Key: info.Key, Size: info.Size, ModTime: info.LastModified}) + } + return out, nil +} + +// walkLocal lists the regular files under root. Symlinks are followed (with +// loop protection) when follow is true and ignored otherwise. +func walkLocal(root string, follow bool) ([]entry, error) { + st, err := os.Stat(root) + if err != nil { + return nil, localErr(err) + } + if !st.IsDir() { + return nil, objectstorage.ErrUsagef("%s is not a directory", root) + } + visited := map[string]bool{} + if real, err := filepath.EvalSymlinks(root); err == nil { + visited[real] = true + } + var out []entry + var walk func(dir, relPrefix string) error + walk = func(dir, relPrefix string) error { + des, err := os.ReadDir(dir) + if err != nil { + return localErr(err) + } + for _, d := range des { + path := filepath.Join(dir, d.Name()) + rel := relPrefix + d.Name() + info, err := d.Info() + if err != nil { + return localErr(err) + } + if info.Mode()&os.ModeSymlink != 0 { + if !follow { + continue + } + info, err = os.Stat(path) + if err != nil { + objectstorage.Warnf("skipping %s: %v", path, err) + continue + } + if info.IsDir() { + real, err := filepath.EvalSymlinks(path) + if err != nil || visited[real] { + continue + } + visited[real] = true + } + } + switch { + case info.IsDir(): + if err := walk(path, rel+"/"); err != nil { + return err + } + case info.Mode().IsRegular(): + out = append(out, entry{Rel: rel, Path: path, Size: info.Size(), ModTime: info.ModTime()}) + } + } + return nil + } + if err := walk(root, ""); err != nil { + return nil, err + } + return out, nil +} + +// --------------------------------------------------------------------------- +// Local path helpers + +// displayLocal renders a local path the way aws prints it (os.path.relpath): +// relative to the working directory when possible, with a leading "./" only +// when the result has no directory component ("./dump.sql", "sub/b.txt", +// "../other/x"). Every local side of an output line (sources, single and +// recursive destinations, sync deletes) goes through it. +func displayLocal(path string) string { + if cwd, err := os.Getwd(); err == nil { + abs := path + if !filepath.IsAbs(abs) { + abs = filepath.Join(cwd, path) + } + // Rel fails across Windows volumes; the absolute path is shown then. + if rel, err := filepath.Rel(cwd, abs); err == nil { + path = rel + } + } + if path == "." || path == ".." || strings.ContainsRune(path, filepath.Separator) || filepath.VolumeName(path) != "" { + return path + } + return "." + string(filepath.Separator) + path +} + +// endsWithSeparator reports whether a path ends with a path separator. +func endsWithSeparator(p string) bool { + return strings.HasSuffix(p, "/") || strings.HasSuffix(p, string(filepath.Separator)) +} + +// escapeErr is the exit-7 error for a key that would be written outside the +// destination directory. +func escapeErr(key string) error { + return exitcode.Errorf(exitcode.Refused, "refusing to write %q: the key escapes the destination directory", key) +} + +// safeLocalName validates the file name appended to a destination directory +// for a single download: after FromSlash+Clean it must be a plain name (no +// separator, not "." or "..", no volume name), otherwise the key would land +// outside the directory. +func safeLocalName(name string) (string, error) { + clean := filepath.Clean(filepath.FromSlash(name)) + if clean == "." || clean == ".." || filepath.IsAbs(clean) || strings.ContainsRune(clean, filepath.Separator) || filepath.VolumeName(clean) != "" { + return "", escapeErr(name) + } + return clean, nil +} + +// localDestination applies the aws rule for a single download: an existing +// directory or a trailing separator means "put inside", anything else +// is the literal file path. It returns the path to write and the display +// form; a name that would escape the directory is refused (exit 7). +func localDestination(dst, name string) (path, display string, err error) { + inside := endsWithSeparator(dst) + if !inside { + if st, statErr := os.Stat(dst); statErr == nil && st.IsDir() { + inside = true + } + } + if !inside { + return dst, displayLocal(dst), nil + } + clean, err := safeLocalName(name) + if err != nil { + return "", "", err + } + path = filepath.Join(dst, clean) + return path, displayLocal(path), nil +} + +// localRelDestination joins a relative key path under the destination +// directory for recursive downloads, refusing paths that escape it. +func localRelDestination(dstDir, rel string) (path, display string, err error) { + clean := filepath.Clean(filepath.FromSlash(rel)) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) || filepath.VolumeName(clean) != "" { + return "", "", escapeErr(rel) + } + path = filepath.Join(dstDir, clean) + return path, displayLocal(path), nil +} + +// localErr wraps a filesystem error with an exit code. +func localErr(err error) error { + switch { + case errors.Is(err, os.ErrNotExist): + return exitcode.New(exitcode.NotFound, err) + case errors.Is(err, os.ErrPermission): + return exitcode.New(exitcode.Permission, err) + } + return exitcode.New(exitcode.Generic, err) +} + +// --------------------------------------------------------------------------- +// Content type and headers + +// guessContentType implements the cp rules: explicit flag, else extension, +// else sniffing the first bytes, else application/octet-stream. +// --no-guess-mime-type yields binary/octet-stream (like aws). +func guessContentType(path string, opts *transferOptions) string { + if opts.ContentType != "" { + return opts.ContentType + } + if opts.NoGuessMime { + return "binary/octet-stream" + } + if ext := filepath.Ext(path); ext != "" { + if ct := mime.TypeByExtension(ext); ct != "" { + return ct + } + } + f, err := os.Open(path) + if err != nil { + return "application/octet-stream" + } + defer f.Close() + buf := make([]byte, 512) + n, _ := io.ReadFull(f, buf) + if n == 0 { + return "application/octet-stream" + } + return http.DetectContentType(buf[:n]) +} + +// putOptions builds the minio options shared by uploads. +func putOptions(contentType string, opts *transferOptions, progress io.Reader) minio.PutObjectOptions { + return minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: opts.Metadata, + CacheControl: opts.CacheControl, + ContentEncoding: opts.ContentEncoding, + ContentDisposition: opts.ContentDisposition, + ContentLanguage: opts.ContentLanguage, + Expires: opts.Expires, + Progress: progress, + } +} + +// parseMetadata parses the aws shorthand `k=v,k2=v2` (repeatable, merged). +func parseMetadata(values []string) (map[string]string, error) { + if len(values) == 0 { + return nil, nil + } + out := map[string]string{} + for _, v := range values { + for _, pair := range strings.Split(v, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + k, val, ok := strings.Cut(pair, "=") + k = strings.TrimSpace(k) + if !ok || k == "" { + return nil, objectstorage.ErrUsagef("invalid --metadata %q: expected key=value[,key2=value2]", pair) + } + out[k] = strings.TrimSpace(val) + } + } + return out, nil +} + +// parseExpires accepts RFC3339 or YYYY-MM-DD. +func parseExpires(s string) (time.Time, error) { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{}, nil + } + for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, s); err == nil { + return t, nil + } + } + return time.Time{}, objectstorage.ErrUsagef("invalid --expires %q: use RFC3339 (2026-09-07T15:04:05Z) or YYYY-MM-DD", s) +} + +// --------------------------------------------------------------------------- +// Progress + +// progressMeter is a minio Progress reader that rewrites one stderr line +// every ~200ms. It also works as a counting writer wrapper for downloads. +type progressMeter struct { + w io.Writer + label string + total int64 + done atomic.Int64 + + mu sync.Mutex // guards last and rendered (minio uploads parts concurrently) + last time.Time + rendered bool +} + +func newProgressMeter(w io.Writer, label string, total int64) *progressMeter { + return &progressMeter{w: w, label: label, total: total} +} + +// Read implements minio's progress hook contract: every call reports len(b) +// bytes transferred. +func (p *progressMeter) Read(b []byte) (int, error) { + p.add(int64(len(b))) + return len(b), nil +} + +func (p *progressMeter) add(n int64) { + if p == nil { + return + } + done := p.done.Add(n) + now := time.Now() + p.mu.Lock() + defer p.mu.Unlock() + // total is 0 for a stdin upload: there is no final frame to force, so the + // throttle applies to every update. + if now.Sub(p.last) < 200*time.Millisecond && (p.total <= 0 || done < p.total) { + return + } + p.last = now + p.render(done) +} + +// render draws the line; the caller holds p.mu. +func (p *progressMeter) render(done int64) { + p.rendered = true + if p.total > 0 { + pct := float64(done) / float64(p.total) * 100 + if pct > 100 { + pct = 100 + } + fmt.Fprintf(p.w, "\r\033[K%s: %s / %s (%.0f%%)", p.label, objectstorage.HumanSize(done), objectstorage.HumanSize(p.total), pct) + return + } + fmt.Fprintf(p.w, "\r\033[K%s: %s", p.label, objectstorage.HumanSize(done)) +} + +// finish clears the progress line. +func (p *progressMeter) finish() { + if p == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + if p.rendered { + fmt.Fprint(p.w, "\r\033[K") + } +} + +// countingWriter feeds a progress meter while copying downloads. +type countingWriter struct { + w io.Writer + p *progressMeter +} + +func (c *countingWriter) Write(b []byte) (int, error) { + n, err := c.w.Write(b) + c.p.add(int64(n)) + return n, err +} + +// meterFor returns a progress meter when enabled, nil otherwise. A nil +// *progressMeter must never be passed as minio Progress (it is an interface). +func meterFor(opts *transferOptions, hints io.Writer, label string, total int64) *progressMeter { + if !opts.ShowProgress { + return nil + } + return newProgressMeter(hints, label, total) +} + +func progressReader(p *progressMeter) io.Reader { + if p == nil { + return nil + } + return p +} + +// --------------------------------------------------------------------------- +// Execution + +// runTransfers executes a plan. Human lines go to out, hints/progress/errors +// to hints. It returns the collected results and, when something failed, an +// exit-coded error (the item's own error for a single-item plan, exit 1 +// otherwise). Deletes from sync plans are executed after the copies, and only +// when every copy succeeded (see below). +func runTransfers(ctx context.Context, plan *transferPlan, opts *transferOptions, out, hints io.Writer) ([]objectstorage.TransferResult, error) { + // total is the planned size and never changes: it decides where a failure + // is printed (emit) and whether the caller prints the item's own error. + // attempted excludes deletions that were skipped, so the failure summary + // counts only what ran. + total := len(plan.Items) + len(plan.Deletes) + attempted := total + results := make([]objectstorage.TransferResult, 0, total) + failures := 0 + var firstErr error + + emit := func(res objectstorage.TransferResult) { + results = append(results, res) + if res.Error != "" { + // Failures always go to stderr, even with --quiet. A single-item + // plan returns its error instead, so it is printed once. + if total > 1 { + fmt.Fprintln(hints, humanLine(res)) + } + return + } + if !opts.showLines() { + return + } + if plan.Dst.stdio() { + // stdout carries the object itself, so lines are suppressed — + // except under --dry-run, where nothing is streamed and the plan + // would otherwise be invisible. + if opts.DryRun { + fmt.Fprintln(hints, humanLine(res)) + } + return + } + fmt.Fprintln(out, humanLine(res)) + } + + for _, item := range plan.Items { + if err := ctx.Err(); err != nil { + return results, exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + res := objectstorage.TransferResult{Op: item.Op, Source: item.Src, Destination: item.Dst, Size: item.Size, DryRun: opts.DryRun} + skip, err := shouldSkipExisting(ctx, plan, item, opts, hints) + if err == nil && skip { + continue + } + if err == nil && !opts.DryRun { + err = transferOne(ctx, plan, item, opts, &res, hints) + } + if err != nil { + if ctx.Err() != nil { + return results, exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + err = humanizeFor(plan, err) + res.Error = err.Error() + failures++ + if firstErr == nil { + firstErr = err + } + } + emit(res) + } + + // `sync --delete` only plans deletions for keys the source does not have, + // so applying them after a failed copy would leave the destination with + // neither the object that failed to transfer nor the one that was about to + // be pruned. Skip them instead: re-running the sync applies them once the + // transfers succeed, and nothing has been lost in the meantime. + deletes := plan.Deletes + if failures > 0 && len(deletes) > 0 { + fmt.Fprintf(hints, "warning: %d planned deletion(s) not applied because %d transfer(s) failed; the destination is not in sync — re-run once the transfers succeed\n", len(deletes), failures) + attempted -= len(deletes) + deletes = nil + } + + for _, del := range deletes { + if err := ctx.Err(); err != nil { + return results, exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + res := objectstorage.TransferResult{Op: opDelete, Destination: del.Display, DryRun: opts.DryRun} + if !opts.DryRun { + if err := deleteOne(ctx, plan.Dst, del); err != nil { + if ctx.Err() != nil { + return results, exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + err = humanizeFor(plan, err) + res.Error = err.Error() + failures++ + if firstErr == nil { + firstErr = err + } + } + } + emit(res) + } + + if failures == 0 { + return results, nil + } + if total == 1 { + return results, firstErr + } + return results, exitcode.Errorf(exitcode.Generic, "%d of %d transfers failed", failures, attempted) +} + +// humanLine renders a result as an aws-style line; deletes have no source. +func humanLine(res objectstorage.TransferResult) string { + if res.Op != opDelete { + return res.HumanLine() + } + prefix := "" + if res.DryRun { + prefix = "(dryrun) " + } + if res.Error != "" { + return fmt.Sprintf("%sdelete failed: %s: %s", prefix, res.Destination, res.Error) + } + return fmt.Sprintf("%sdelete: %s", prefix, res.Destination) +} + +// humanizeFor maps an error through the most relevant session. +func humanizeFor(plan *transferPlan, err error) error { + var already *exitcode.Error + if errors.As(err, &already) { + return err + } + if plan.Dst.remote() { + return plan.Dst.Sess.humanize(err) + } + if plan.Src.remote() { + return plan.Src.Sess.humanize(err) + } + return objectstorage.Humanize(err, nil, nil) +} + +// shouldSkipExisting implements --no-overwrite: an existing destination is +// left alone and reported on hints. +func shouldSkipExisting(ctx context.Context, plan *transferPlan, item transferItem, opts *transferOptions, hints io.Writer) (bool, error) { + if !opts.NoOverwrite || plan.Dst.stdio() { + return false, nil + } + exists := false + if plan.Dst.remote() { + _, err := plan.Dst.Sess.Client.StatObject(ctx, plan.Dst.Sess.Bucket.BucketName, item.DstKey, minio.StatObjectOptions{}) + switch { + case err == nil: + exists = true + case minio.ToErrorResponse(err).StatusCode == http.StatusNotFound: + default: + return false, err + } + } else { + _, err := os.Stat(item.DstPath) + switch { + case err == nil: + exists = true + case errors.Is(err, os.ErrNotExist): + default: + return false, err + } + } + if exists { + fmt.Fprintf(hints, "skip: %s already exists (--no-overwrite)\n", item.Dst) + } + return exists, nil +} + +// transferOne performs a single planned item and fills the result. +func transferOne(ctx context.Context, plan *transferPlan, item transferItem, opts *transferOptions, res *objectstorage.TransferResult, hints io.Writer) error { + src, dst := plan.Src, plan.Dst + label := fmt.Sprintf("%s -> %s", item.Src, item.Dst) + switch { + case src.stdio(): + return uploadStdin(ctx, dst, item, opts, res, hints) + case dst.stdio(): + return download(ctx, src, item, opts, res, opts.stdout(), nil) + case src.local() && dst.remote(): + meter := meterFor(opts, hints, label, item.Size) + defer meter.finish() + ct := guessContentType(item.SrcPath, opts) + info, err := dst.Sess.Client.FPutObject(ctx, dst.Sess.Bucket.BucketName, item.DstKey, item.SrcPath, putOptions(ct, opts, progressReader(meter))) + if err != nil { + return err + } + res.ETag, res.ContentType = strings.Trim(info.ETag, `"`), ct + if info.Size > 0 { + res.Size = info.Size + } + if opts.Move { + return os.Remove(item.SrcPath) + } + return nil + case src.remote() && dst.local(): + meter := meterFor(opts, hints, label, item.Size) + defer meter.finish() + if err := download(ctx, src, item, opts, res, nil, meter); err != nil { + return err + } + if opts.Move { + return src.Sess.Client.RemoveObject(ctx, src.Sess.Bucket.BucketName, item.SrcKey, minio.RemoveObjectOptions{}) + } + return nil + default: // remote -> remote + dstOpts := minio.CopyDestOptions{Bucket: dst.Sess.Bucket.BucketName, Object: item.DstKey} + if len(opts.Metadata) > 0 || opts.ContentType != "" || opts.CacheControl != "" || opts.ContentEncoding != "" || opts.ContentDisposition != "" || opts.ContentLanguage != "" || !opts.Expires.IsZero() { + dstOpts.ReplaceMetadata = true + dstOpts.UserMetadata = opts.Metadata + dstOpts.ContentType = opts.ContentType + dstOpts.CacheControl = opts.CacheControl + dstOpts.ContentEncoding = opts.ContentEncoding + dstOpts.ContentDisposition = opts.ContentDisposition + dstOpts.ContentLanguage = opts.ContentLanguage + dstOpts.Expires = opts.Expires + } + info, err := dst.Sess.Client.CopyObject(ctx, dstOpts, minio.CopySrcOptions{Bucket: src.Sess.Bucket.BucketName, Object: item.SrcKey}) + switch { + case err == nil: + res.ETag = strings.Trim(info.ETag, `"`) + case needsStreamedCopy(src, dst, err): + // A CopyObject carries a single identity: it is signed with the + // destination credential and the backend reads the source as that + // same identity. When the two buckets resolved different keys the + // destination key cannot read the source, so the object is streamed + // through this machine instead — read with the source credential, + // written with the destination one. + fmt.Fprintf(hints, "%s and %s resolved different access keys, so the copy streams through this machine instead of the backend\n", src.Sess.Bucket.Display(), dst.Sess.Bucket.Display()) + if streamErr := streamCopy(ctx, src, dst, item, opts, res, hints, label); streamErr != nil { + return streamErr + } + default: + return err + } + if opts.Move { + return src.Sess.Client.RemoveObject(ctx, src.Sess.Bucket.BucketName, item.SrcKey, minio.RemoveObjectOptions{}) + } + return nil + } +} + +// stdinPartSizeFor picks the multipart part size for a stdin upload. Without +// --expected-size it is stdinPartSize; with it, the size minio would use for +// an object of that length (so streams above 160 GiB still fit in 10000 +// parts), never below stdinPartSize. The value only shapes the parts: the +// upload itself is always length-unknown and succeeds whatever the real size. +func stdinPartSizeFor(expected int64) uint64 { + if expected <= 0 { + return stdinPartSize + } + _, partSize, _, err := minio.OptimalPartInfo(expected, 0) + if err != nil || partSize < stdinPartSize { + return stdinPartSize + } + return uint64(partSize) +} + +// uploadStdin streams stdin as a multipart upload of unknown length. minio +// reads one part at a time into a buffer of the part size, so memory use is +// one part; --expected-size only sizes the parts and the progress total. +func uploadStdin(ctx context.Context, dst endpoint, item transferItem, opts *transferOptions, res *objectstorage.TransferResult, hints io.Writer) error { + rd := bufio.NewReaderSize(opts.stdin(), 512) + ct := opts.ContentType + switch { + case ct != "": + case opts.NoGuessMime: + ct = "binary/octet-stream" + default: + ct = "application/octet-stream" + if ext := filepath.Ext(item.DstKey); ext != "" { + if byExt := mime.TypeByExtension(ext); byExt != "" { + ct = byExt + } + } + if ct == "application/octet-stream" { + if head, _ := rd.Peek(512); len(head) > 0 { + ct = http.DetectContentType(head) + } + } + } + partSize := stdinPartSizeFor(opts.ExpectedSize) + if opts.ExpectedSize <= 0 { + fmt.Fprintf(hints, "note: streaming from stdin in %s parts (streams up to %s; pass --expected-size for larger ones)\n", objectstorage.HumanSize(stdinPartSize), objectstorage.HumanSize(stdinMaxPlainSize)) + } + meter := meterFor(opts, hints, "- -> "+item.Dst, opts.ExpectedSize) + defer meter.finish() + po := putOptions(ct, opts, progressReader(meter)) + po.PartSize = partSize + // Size -1: the real length is whatever stdin delivers. Passing the + // expected size here would make minio fail on a shorter stream and + // truncate a longer one. + info, err := dst.Sess.Client.PutObject(ctx, dst.Sess.Bucket.BucketName, item.DstKey, rd, -1, po) + if err != nil { + return err + } + res.ETag, res.ContentType, res.Size = strings.Trim(info.ETag, `"`), ct, info.Size + return nil +} + +// download fetches one object either into w (stdout) or into item.DstPath +// through a temporary "..lsh-partial" file renamed on success. The +// local file's mtime is set to the object's LastModified so sync can compare. +func download(ctx context.Context, src endpoint, item transferItem, opts *transferOptions, res *objectstorage.TransferResult, w io.Writer, meter *progressMeter) error { + obj, err := src.Sess.Client.GetObject(ctx, src.Sess.Bucket.BucketName, item.SrcKey, minio.GetObjectOptions{}) + if err != nil { + return err + } + defer obj.Close() + info, err := obj.Stat() + if err != nil { + return err + } + res.Size, res.ETag, res.ContentType = info.Size, strings.Trim(info.ETag, `"`), info.ContentType + + var body io.Reader = obj + if w != nil { + if meter != nil { + _, err = io.Copy(&countingWriter{w: w, p: meter}, body) + } else { + _, err = io.Copy(w, body) + } + return err + } + + dir := filepath.Dir(item.DstPath) + if err := os.MkdirAll(dir, 0o755); err != nil { + return localErr(err) + } + tmp := filepath.Join(dir, "."+filepath.Base(item.DstPath)+partialSuffix) + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return localErr(err) + } + var dst io.Writer = f + if meter != nil { + dst = &countingWriter{w: f, p: meter} + } + if _, err := io.Copy(dst, body); err != nil { + f.Close() + os.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return localErr(err) + } + if err := os.Rename(tmp, item.DstPath); err != nil { + os.Remove(tmp) + return localErr(err) + } + if !info.LastModified.IsZero() { + _ = os.Chtimes(item.DstPath, info.LastModified, info.LastModified) + } + return nil +} + +// deleteOne removes a destination entry (sync --delete). +func deleteOne(ctx context.Context, dst endpoint, del deleteItem) error { + if dst.remote() { + return dst.Sess.Client.RemoveObject(ctx, dst.Sess.Bucket.BucketName, del.Key, minio.RemoveObjectOptions{}) + } + if err := os.Remove(del.Path); err != nil && !errors.Is(err, os.ErrNotExist) { + return localErr(err) + } + return nil +} + +// --------------------------------------------------------------------------- +// Cobra glue shared by cp, mv and sync + +// transferFlags holds the raw flag values before conversion. +type transferFlags struct { + recursive bool + filters objectstorage.Filters + contentType string + cacheControl string + contentEncoding string + contentDisposition string + contentLanguage string + expires string + metadata []string + noGuessMime bool + noOverwrite bool + expectedSize int64 + quiet bool + onlyShowErrors bool + noProgress bool + followSymlinks bool + noFollowSymlinks bool +} + +// addTransferFlags registers the flags shared by cp, mv and sync. recursive +// controls whether --recursive/-r is offered (sync is always recursive). +func addTransferFlags(cmd *cobra.Command, f *transferFlags, recursive bool) { + fs := cmd.Flags() + if recursive { + fs.BoolVarP(&f.recursive, "recursive", "r", false, "copy every file/object under the source directory or prefix, mirroring relative paths") + } + objectstorage.FilterFlags(fs, &f.filters) + fs.StringVar(&f.contentType, "content-type", "", "Content-Type to store (default: guessed from the extension, then the content)") + fs.StringVar(&f.cacheControl, "cache-control", "", "Cache-Control header to store with the object") + fs.StringVar(&f.contentEncoding, "content-encoding", "", "Content-Encoding header to store with the object") + fs.StringVar(&f.contentDisposition, "content-disposition", "", "Content-Disposition header to store with the object") + fs.StringVar(&f.contentLanguage, "content-language", "", "Content-Language header to store with the object") + fs.StringVar(&f.expires, "expires", "", "Expires header (RFC3339 or YYYY-MM-DD)") + fs.StringArrayVar(&f.metadata, "metadata", nil, "user metadata as key=value[,key2=value2] (repeatable, merged)") + fs.BoolVar(&f.noGuessMime, "no-guess-mime-type", false, "store binary/octet-stream instead of guessing the Content-Type") + fs.BoolVar(&f.noOverwrite, "no-overwrite", false, "skip destinations that already exist") + fs.Int64Var(&f.expectedSize, "expected-size", 0, "approximate size in bytes of a stdin upload; only sizes the multipart parts (needed above 160 GiB) and the progress total, the upload succeeds whatever the real length") + fs.BoolVarP(&f.quiet, "quiet", "q", false, "do not print the per-object lines") + fs.BoolVar(&f.onlyShowErrors, "only-show-errors", false, "print only errors and warnings") + fs.BoolVar(&f.noProgress, "no-progress", false, "do not show the progress meter") + fs.BoolVar(&f.followSymlinks, "follow-symlinks", true, "follow symbolic links when enumerating local directories") + fs.BoolVar(&f.noFollowSymlinks, "no-follow-symlinks", false, "ignore symbolic links when enumerating local directories") + addProjectFlag(cmd, true, "project that owns the bucket(s) (disambiguates names shared across projects)") + + // `aws s3 cp ... --region x` gets the directed explanation (exit 2). + rejectRegionFlag(cmd) + unsupportedAWSFlags(cmd, map[string]string{ + "acl": "buckets are private; grant access with scoped access keys (lsh s3 access-keys create)", + "grants": "buckets are private; grant access with scoped access keys (lsh s3 access-keys create)", + "sse": "server-side encryption is managed by the platform", + "sse-c": "customer-provided encryption keys are not supported", + "sse-c-key": "customer-provided encryption keys are not supported", + "sse-kms-key-id": "KMS is not available on Latitude object storage", + "sse-c-copy-source": "customer-provided encryption keys are not supported", + "sse-c-copy-source-key": "customer-provided encryption keys are not supported", + "metadata-directive": "server-side copies keep the source metadata; pass --metadata/--content-type to replace it", + "copy-props": "server-side copies always keep the source properties", + "checksum-algorithm": "the CLI uses Content-MD5; additional checksums are rejected by S3-compatible backends", + "checksum-mode": "checksum validation on download is not available", + "request-payer": "there is no requester-pays billing on Latitude", + "storage-class": "storage class is a bucket attribute on Latitude (standard|high_performance); set it with lsh s3 mb --storage-class", + }) + if sse := fs.Lookup("sse"); sse != nil { + sse.NoOptDefVal = "AES256" + } +} + +// buildTransferOptions converts flags into transferOptions. +func buildTransferOptions(cmd *cobra.Command, f *transferFlags) (*transferOptions, error) { + meta, err := parseMetadata(f.metadata) + if err != nil { + return nil, err + } + expires, err := parseExpires(f.expires) + if err != nil { + return nil, err + } + if f.expectedSize < 0 { + return nil, objectstorage.ErrUsagef("--expected-size must be a positive number of bytes") + } + follow := f.followSymlinks + if f.noFollowSymlinks { + follow = false + } + human := isHuman() + return &transferOptions{ + Recursive: f.recursive, + Filters: &f.filters, + ContentType: f.contentType, + CacheControl: f.cacheControl, + ContentEncoding: f.contentEncoding, + ContentDisposition: f.contentDisposition, + ContentLanguage: f.contentLanguage, + Expires: expires, + Metadata: meta, + NoGuessMime: f.noGuessMime, + NoOverwrite: f.noOverwrite, + ExpectedSize: f.expectedSize, + FollowSymlinks: follow, + DryRun: dryRun(), + Human: human, + Quiet: f.quiet, + OnlyShowErrors: f.onlyShowErrors, + ShowProgress: human && !f.quiet && !f.onlyShowErrors && !f.noProgress && stderrIsTerminal(), + }, nil +} + +// stderrIsTerminal reports whether progress can be drawn. +func stderrIsTerminal() bool { return term.IsTerminal(int(os.Stderr.Fd())) } + +// openEndpoints resolves the remote side(s) of a transfer. The destination +// always needs write permission; the source needs it only for mv. +func openEndpoints(ctx context.Context, cmd *cobra.Command, src, dst objectstorage.Ref, move bool) (endpoint, endpoint, error) { + s, d := endpoint{Ref: src}, endpoint{Ref: dst} + var err error + if src.Remote { + if s.Sess, err = openBucket(ctx, cmd, src.Bucket, move); err != nil { + return s, d, err + } + } + if !dst.Remote { + return s, d, nil + } + switch { + case s.Sess != nil && src.Bucket == dst.Bucket && move: + // Same bucket and the source session already has write permission. + d.Sess = s.Sess + case s.Sess != nil && src.Bucket == dst.Bucket: + // Same bucket: skip the second API lookup but select a write credential. + d.Sess, err = openResolved(cmd, s.Sess.Bucket, true) + default: + d.Sess, err = openBucket(ctx, cmd, dst.Bucket, true) + } + return s, d, err +} + +// finishTransfer renders structured output and returns the command error. +func finishTransfer(plan *transferPlan, opts *transferOptions, results []objectstorage.TransferResult, err error) error { + if !opts.Human && !plan.Dst.stdio() { + render(objectstorage.AsResponseData(results)) + } + if err != nil { + return printErr(err) + } + return nil +} + +// runCopyCommand is the RunE body shared by cp and mv. +func runCopyCommand(cmd *cobra.Command, args []string, f *transferFlags, move bool) error { + srcRef, dstRef, err := classifyOperands(args[0], args[1], move, false) + if err != nil { + return printErr(err) + } + opts, err := buildTransferOptions(cmd, f) + if err != nil { + return printErr(err) + } + opts.Move = move + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + src, dst, err := openEndpoints(ctx, cmd, srcRef, dstRef, move) + if err != nil { + return printErr(err) + } + plan, err := buildPlan(ctx, src, dst, opts) + if err != nil { + return printErr(err) + } + if len(plan.Items) == 0 && opts.showLines() && !dst.stdio() { + objectstorage.Hintf("nothing to %s", verbFor(move)) + } + results, err := runTransfers(ctx, plan, opts, os.Stdout, os.Stderr) + return finishTransfer(plan, opts, results, err) +} + +func verbFor(move bool) string { + if move { + return "move" + } + return "copy" +} + +// destinationRulesHelp documents the aws fileformat rules in --help. +const destinationRulesHelp = `Destination rules: + s3:// destination ending in "/" or naming only the bucket -> the source file name is appended + s3:// destination without a trailing "/" -> the key is used literally + (cp dump.sql s3://b/2026/09 creates the key "2026/09"; the output shows it) + local destination that is an existing directory or ends in the path separator + -> the file name is appended (directories are created) + any other local destination -> literal file path + --recursive forces directory semantics on both sides and mirrors paths relative to the source. + +"-" reads the source from stdin or writes the destination to stdout (nothing else is printed on stdout). +Both operands remote -> server-side copy, only within the same endpoint.` diff --git a/cmd/s3/transfer_test.go b/cmd/s3/transfer_test.go new file mode 100644 index 0000000..becb6e7 --- /dev/null +++ b/cmd/s3/transfer_test.go @@ -0,0 +1,1194 @@ +package s3 + +import ( + "bytes" + "context" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" +) + +// newTestSession builds a Session against the fake server for bucket name. +func newTestSession(t *testing.T, srv *s3test.Server, name string) *Session { + t.Helper() + b := &objectstorage.Bucket{ID: "bkt_1", Name: "b", BucketName: name, Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: "us-east-1"} + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +func remoteEP(sess *Session, key string) endpoint { + raw := "s3://" + sess.Bucket.Name + if key != "" { + raw += "/" + key + } + return endpoint{Ref: objectstorage.Ref{Raw: raw, Bucket: sess.Bucket.Name, Key: key, Remote: true, HadScheme: true}, Sess: sess} +} + +func localEP(path string) endpoint { return endpoint{Ref: objectstorage.Ref{Raw: path}} } + +func stdioEP() endpoint { return endpoint{Ref: objectstorage.Ref{Raw: "-", Stdio: true}} } + +func baseOpts() *transferOptions { + return &transferOptions{Human: true, FollowSymlinks: true, Filters: &objectstorage.Filters{}} +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// run plans and executes a cp/mv between two endpoints. +func run(t *testing.T, src, dst endpoint, opts *transferOptions) ([]objectstorage.TransferResult, string, string, error) { + t.Helper() + ctx := context.Background() + plan, err := buildPlan(ctx, src, dst, opts) + if err != nil { + return nil, "", "", err + } + var out, hints bytes.Buffer + results, err := runTransfers(ctx, plan, opts, &out, &hints) + return results, out.String(), hints.String(), err +} + +func TestUploadDownloadRoundTrip(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + + // Local paths are displayed relative to the working directory (aws + // behaviour), so the test runs inside the temp dir with relative operands. + dir := t.TempDir() + t.Chdir(dir) + file := "report.json" + writeFile(t, file, `{"ok":true}`) + + opts := baseOpts() + opts.Metadata = map[string]string{"owner": "ops", "team": "infra"} + opts.CacheControl = "max-age=60" + opts.ContentLanguage = "en" + results, out, _, err := run(t, localEP(file), remoteEP(sess, "2026/09/"), opts) + if err != nil { + t.Fatalf("upload: %v", err) + } + if want := "upload: ./report.json to s3://b/2026/09/report.json\n"; out != want { + t.Errorf("stdout = %q, want %q", out, want) + } + if len(results) != 1 || results[0].Op != "upload" || results[0].ContentType != "application/json" { + t.Errorf("results = %+v", results) + } + obj := srv.Object("bucket-a", "2026/09/report.json") + if obj == nil { + t.Fatalf("object not stored; keys=%v", srv.Keys("bucket-a")) + } + if string(obj.Data) != `{"ok":true}` { + t.Errorf("data = %q", obj.Data) + } + if obj.ContentType != "application/json" { + t.Errorf("content type = %q, want application/json (guessed from extension)", obj.ContentType) + } + if obj.Metadata["owner"] != "ops" || obj.Metadata["team"] != "infra" { + t.Errorf("metadata = %v", obj.Metadata) + } + if obj.Headers["Cache-Control"] != "max-age=60" || obj.Headers["Content-Language"] != "en" { + t.Errorf("headers = %v", obj.Headers) + } + if srv.HasChecksumHeaders() { + t.Error("upload must not send checksum trailers") + } + + // Download into a directory (trailing separator appends the base name); + // the destination is shown relative to cwd without "./" because it has a + // directory component. + dest := "restore" + string(filepath.Separator) + results, out, _, err = run(t, remoteEP(sess, "2026/09/report.json"), localEP(dest), baseOpts()) + if err != nil { + t.Fatalf("download: %v", err) + } + if want := "download: s3://b/2026/09/report.json to " + filepath.Join("restore", "report.json") + "\n"; out != want { + t.Errorf("stdout = %q, want %q", out, want) + } + got, err := os.ReadFile(filepath.Join(dir, "restore", "report.json")) + if err != nil || string(got) != `{"ok":true}` { + t.Fatalf("downloaded content = %q, err = %v", got, err) + } + if results[0].Size != int64(len(`{"ok":true}`)) || results[0].ContentType != "application/json" { + t.Errorf("download result = %+v", results[0]) + } + if _, err := os.Stat(filepath.Join(dir, "restore", ".report.json"+partialSuffix)); !os.IsNotExist(err) { + t.Error("partial file must be renamed away") + } +} + +func TestContentTypeSniffAndNoGuess(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + + // No extension: sniffed from content. + html := filepath.Join(dir, "page") + writeFile(t, html, "hi") + if _, _, _, err := run(t, localEP(html), remoteEP(sess, ""), baseOpts()); err != nil { + t.Fatal(err) + } + if ct := srv.Object("bucket-a", "page").ContentType; !strings.HasPrefix(ct, "text/html") { + t.Errorf("sniffed content type = %q", ct) + } + + // --no-guess-mime-type stores binary/octet-stream. + opts := baseOpts() + opts.NoGuessMime = true + if _, _, _, err := run(t, localEP(html), remoteEP(sess, "raw"), opts); err != nil { + t.Fatal(err) + } + if ct := srv.Object("bucket-a", "raw").ContentType; ct != "binary/octet-stream" { + t.Errorf("no-guess content type = %q", ct) + } + + // Explicit flag wins. + opts = baseOpts() + opts.ContentType = "application/x-custom" + if _, _, _, err := run(t, localEP(html), remoteEP(sess, "custom"), opts); err != nil { + t.Fatal(err) + } + if ct := srv.Object("bucket-a", "custom").ContentType; ct != "application/x-custom" { + t.Errorf("explicit content type = %q", ct) + } +} + +// The destination rules from the aws user guide (and the spec table). +func TestDestinationRules(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "dir/file.txt", []byte("remote"), "text/plain") + sess := newTestSession(t, srv, "bucket-a") + + // Run inside the temp dir: local display strings are relative to cwd. + t.Chdir(t.TempDir()) + file := "file.txt" + writeFile(t, file, "local") + existing := "existing" + if err := os.Mkdir(existing, 0o755); err != nil { + t.Fatal(err) + } + tree := "tree" + writeFile(t, filepath.Join(tree, "a.txt"), "a") + writeFile(t, filepath.Join(tree, "sub", "b.txt"), "b") + + cases := []struct { + name string + src, dst endpoint + recursive bool + wantKeys []string // remote destination keys + wantPaths []string // local destination paths + wantDst []string // display strings + }{ + {"file to bucket root", localEP(file), remoteEP(sess, ""), false, []string{"file.txt"}, nil, []string{"s3://b/file.txt"}}, + {"file to prefix with slash", localEP(file), remoteEP(sess, "2026/09/"), false, []string{"2026/09/file.txt"}, nil, []string{"s3://b/2026/09/file.txt"}}, + {"file to literal key", localEP(file), remoteEP(sess, "2026/09"), false, []string{"2026/09"}, nil, []string{"s3://b/2026/09"}}, + {"file to renamed key", localEP(file), remoteEP(sess, "renamed.bin"), false, []string{"renamed.bin"}, nil, []string{"s3://b/renamed.bin"}}, + // Destinations with a directory component are shown without "./", + // a bare file name with it (aws relpath rules). + {"object to existing dir", remoteEP(sess, "dir/file.txt"), localEP(existing), false, nil, []string{filepath.Join(existing, "file.txt")}, []string{filepath.Join(existing, "file.txt")}}, + {"object to new dir with slash", remoteEP(sess, "dir/file.txt"), localEP("new/"), false, nil, []string{filepath.Join("new", "file.txt")}, []string{filepath.Join("new", "file.txt")}}, + {"object to literal file", remoteEP(sess, "dir/file.txt"), localEP("copy.txt"), false, nil, []string{"copy.txt"}, []string{"." + string(filepath.Separator) + "copy.txt"}}, + {"recursive dir to prefix without slash", localEP(tree), remoteEP(sess, "pre"), true, []string{"pre/a.txt", "pre/sub/b.txt"}, nil, []string{"s3://b/pre/a.txt", "s3://b/pre/sub/b.txt"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + opts := baseOpts() + opts.Recursive = tc.recursive + plan, err := buildPlan(context.Background(), tc.src, tc.dst, opts) + if err != nil { + t.Fatalf("buildPlan: %v", err) + } + var keys, paths, dsts []string + for _, it := range plan.Items { + if it.DstKey != "" { + keys = append(keys, it.DstKey) + } + if it.DstPath != "" { + paths = append(paths, it.DstPath) + } + dsts = append(dsts, it.Dst) + } + if strings.Join(keys, ",") != strings.Join(tc.wantKeys, ",") { + t.Errorf("keys = %v, want %v", keys, tc.wantKeys) + } + if strings.Join(paths, ",") != strings.Join(tc.wantPaths, ",") { + t.Errorf("paths = %v, want %v", paths, tc.wantPaths) + } + if strings.Join(dsts, ",") != strings.Join(tc.wantDst, ",") { + t.Errorf("display = %v, want %v", dsts, tc.wantDst) + } + }) + } +} + +func TestSingleObjectUsageErrors(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + + // Directory without --recursive. + if _, err := buildPlan(context.Background(), localEP(dir), remoteEP(sess, ""), baseOpts()); exitcode.Of(err) != exitcode.Usage { + t.Errorf("dir without --recursive: err = %v", err) + } + // Prefix source without --recursive. + if _, err := buildPlan(context.Background(), remoteEP(sess, "pre/"), localEP(dir), baseOpts()); exitcode.Of(err) != exitcode.Usage { + t.Errorf("prefix without --recursive: err = %v", err) + } + // Missing object -> not found. + if _, err := buildPlan(context.Background(), remoteEP(sess, "missing.txt"), localEP(dir), baseOpts()); exitcode.Of(err) != exitcode.NotFound { + t.Errorf("missing object: err = %v", err) + } + // Missing local file -> not found. + if _, err := buildPlan(context.Background(), localEP(filepath.Join(dir, "nope")), remoteEP(sess, ""), baseOpts()); exitcode.Of(err) != exitcode.NotFound { + t.Errorf("missing file: err = %v", err) + } + // Recursive download refuses keys escaping the destination. + srv.AddObject("bucket-a", "pre/../../etc/passwd", []byte("x"), "") + opts := baseOpts() + opts.Recursive = true + if _, err := buildPlan(context.Background(), remoteEP(sess, "pre/"), localEP(dir), opts); exitcode.Of(err) != exitcode.Refused { + t.Errorf("escaping key: err = %v", err) + } + // Single-object download into a directory applies the same guard to the + // appended name (exit 7); a literal file destination never uses the name. + for _, name := range []string{"..", ".", "sub/x", "../x"} { + if _, _, err := localDestination(dir, name); exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "escapes the destination directory") { + t.Errorf("localDestination(dir, %q): err = %v", name, err) + } + } + if _, _, err := localDestination(filepath.Join(dir, "literal.txt"), ".."); err != nil { + t.Errorf("literal destination must not check the name: %v", err) + } + srv.AddObject("bucket-a", "pre/..", []byte("x"), "") + if _, err := buildPlan(context.Background(), remoteEP(sess, "pre/.."), localEP(dir), baseOpts()); exitcode.Of(err) != exitcode.Refused { + t.Errorf("single download of an escaping key: err = %v", err) + } +} + +func TestClassifyOperands(t *testing.T) { + if _, _, err := classifyOperands("./a", "./b", false, false); exitcode.Of(err) != exitcode.Usage { + t.Errorf("local->local: %v", err) + } + if _, _, err := classifyOperands("-", "./b", false, false); exitcode.Of(err) != exitcode.Usage { + t.Errorf("stdin->local: %v", err) + } + if _, _, err := classifyOperands("-", "s3://b/k", true, false); exitcode.Of(err) != exitcode.Usage { + t.Errorf("mv from stdin: %v", err) + } + if _, _, err := classifyOperands("./a", "s3://b/", false, true); err != nil { + t.Errorf("sync local->remote: %v", err) + } + src, dst, err := classifyOperands("s3://b/k", "-", false, false) + if err != nil || !src.Remote || !dst.Stdio { + t.Errorf("remote->stdout: %v %+v %+v", err, src, dst) + } +} + +func TestNoOverwrite(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + file := filepath.Join(dir, "f.txt") + writeFile(t, file, "v1") + + opts := baseOpts() + opts.NoOverwrite = true + if _, _, _, err := run(t, localEP(file), remoteEP(sess, ""), opts); err != nil { + t.Fatal(err) + } + writeFile(t, file, "v2") + srv.ResetRequests() + results, out, hints, err := run(t, localEP(file), remoteEP(sess, ""), opts) + if err != nil { + t.Fatal(err) + } + if len(results) != 0 || out != "" { + t.Errorf("skipped transfer must produce no result/line: %+v %q", results, out) + } + if !strings.Contains(hints, "skip: s3://b/f.txt already exists") { + t.Errorf("hints = %q", hints) + } + if n := len(srv.WriteRequests()); n != 0 { + t.Errorf("no-overwrite issued %d write requests", n) + } + if string(srv.Object("bucket-a", "f.txt").Data) != "v1" { + t.Error("object was overwritten") + } + + // Local destination is honoured too. + dest := filepath.Join(dir, "out.txt") + writeFile(t, dest, "keep") + if _, _, hints, err = run(t, remoteEP(sess, "f.txt"), localEP(dest), opts); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(dest); string(got) != "keep" || !strings.Contains(hints, "skip: ") { + t.Errorf("local file overwritten: %q hints=%q", got, hints) + } +} + +func TestDryRunMakesNoWrites(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "pre/x.txt", []byte("x"), "") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + t.Chdir(dir) // display strings are relative to cwd + writeFile(t, filepath.Join(dir, "a.txt"), "a") + writeFile(t, filepath.Join(dir, "sub", "b.txt"), "b") + + opts := baseOpts() + opts.Recursive, opts.DryRun = true, true + results, out, _, err := run(t, localEP("."), remoteEP(sess, "pre/"), opts) + if err != nil { + t.Fatal(err) + } + want := "(dryrun) upload: ./a.txt to s3://b/pre/a.txt\n" + + "(dryrun) upload: " + filepath.Join("sub", "b.txt") + " to s3://b/pre/sub/b.txt\n" + if out != want { + t.Errorf("stdout = %q, want %q", out, want) + } + if len(results) != 2 || !results[0].DryRun { + t.Errorf("results = %+v", results) + } + // Dry-run download and mv too. + opts.Move = true + if _, out, _, err = run(t, remoteEP(sess, "pre/"), localEP(filepath.Join(dir, "down")), opts); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(out, "(dryrun) move: s3://b/pre/x.txt to ") { + t.Errorf("stdout = %q", out) + } + if srv.Object("bucket-a", "pre/x.txt") == nil { + t.Error("dry-run mv deleted the source") + } + if _, err := os.Stat(filepath.Join(dir, "down")); !os.IsNotExist(err) { + t.Error("dry-run created the destination directory") + } + if w := srv.WriteRequests(); len(w) != 0 { + t.Errorf("dry-run issued %d write requests: %+v", len(w), w[0].Method+" "+w[0].Path) + } +} + +func TestRecursiveUploadWithFilters(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "a.log"), "a") + writeFile(t, filepath.Join(dir, "b.txt"), "b") + writeFile(t, filepath.Join(dir, "sub", "c.log"), "c") + writeFile(t, filepath.Join(dir, "sub", "d.tmp"), "d") + + opts := baseOpts() + opts.Recursive = true + _ = opts.Filters.Add(false, "*") + _ = opts.Filters.Add(true, "*.log") + results, out, _, err := run(t, localEP(dir), remoteEP(sess, "logs/"), opts) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(srv.Keys("bucket-a"), ","); got != "logs/a.log,logs/sub/c.log" { + t.Errorf("keys = %s", got) + } + if len(results) != 2 || strings.Count(out, "upload: ") != 2 { + t.Errorf("results = %d, out = %q", len(results), out) + } +} + +func TestQuietSuppressesLines(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + file := filepath.Join(t.TempDir(), "q.txt") + writeFile(t, file, "q") + opts := baseOpts() + opts.Quiet = true + results, out, _, err := run(t, localEP(file), remoteEP(sess, ""), opts) + if err != nil || out != "" || len(results) != 1 { + t.Errorf("quiet: err=%v out=%q results=%d", err, out, len(results)) + } +} + +func TestStdoutDestinationWritesOnlyPayload(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + payload := strings.Repeat("payload-", 1000) + srv.AddObject("bucket-a", "big.bin", []byte(payload), "application/octet-stream") + sess := newTestSession(t, srv, "bucket-a") + + var stdout bytes.Buffer + opts := baseOpts() + opts.Stdout = &stdout + opts.ShowProgress = true // must go to hints, never stdout + results, out, _, err := run(t, remoteEP(sess, "big.bin"), stdioEP(), opts) + if err != nil { + t.Fatal(err) + } + if stdout.String() != payload { + t.Errorf("stdout payload mismatch (len %d)", stdout.Len()) + } + if out != "" { + t.Errorf("no lines may be printed on stdout when the destination is '-': %q", out) + } + if len(results) != 1 || results[0].Destination != "-" { + t.Errorf("results = %+v", results) + } +} + +func TestStdinUpload(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + + opts := baseOpts() + opts.Stdin = strings.NewReader("hello from stdin\n") + results, out, hints, err := run(t, stdioEP(), remoteEP(sess, "in/notes.txt"), opts) + if err != nil { + t.Fatal(err) + } + obj := srv.Object("bucket-a", "in/notes.txt") + if obj == nil || string(obj.Data) != "hello from stdin\n" { + t.Fatalf("object = %+v", obj) + } + if !strings.HasPrefix(obj.ContentType, "text/plain") { + t.Errorf("content type = %q", obj.ContentType) + } + if out != "upload: - to s3://b/in/notes.txt\n" || len(results) != 1 { + t.Errorf("out = %q results=%+v", out, results) + } + if !strings.Contains(hints, "streaming from stdin") { + t.Errorf("expected memory note on stderr, got %q", hints) + } + // A prefix destination is a usage error for stdin. + if _, err := buildPlan(context.Background(), stdioEP(), remoteEP(sess, "in/"), baseOpts()); exitcode.Of(err) != exitcode.Usage { + t.Errorf("stdin to prefix: %v", err) + } +} + +// --expected-size is a hint for part sizing and the progress total only: the +// upload must store the whole stream whether it is shorter or longer than the +// flag (minio would otherwise fail on a short stream and truncate a long one). +func TestStdinUploadExpectedSizeIsOnlyAHint(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + payload := strings.Repeat("0123456789", 4) // 40 bytes + + for _, expected := range []int64{5, 1 << 30} { + opts := baseOpts() + opts.Stdin = strings.NewReader(payload) + opts.ExpectedSize = expected + key := "in/hint.bin" + results, _, hints, err := run(t, stdioEP(), remoteEP(sess, key), opts) + if err != nil { + t.Fatalf("expected-size %d: %v", expected, err) + } + obj := srv.Object("bucket-a", key) + if obj == nil || string(obj.Data) != payload { + t.Fatalf("expected-size %d: stored %q, want the full %d-byte payload", expected, obj.Data, len(payload)) + } + if len(results) != 1 || results[0].Size != int64(len(payload)) { + t.Errorf("expected-size %d: result size = %+v, want %d", expected, results, len(payload)) + } + if strings.Contains(hints, "streaming from stdin") { + t.Errorf("expected-size %d: the part-size note must only appear without --expected-size: %q", expected, hints) + } + } + + // Part sizing: minio's default up to 160 GiB, larger multiples beyond. + if got := stdinPartSizeFor(0); got != stdinPartSize { + t.Errorf("stdinPartSizeFor(0) = %d", got) + } + if got := stdinPartSizeFor(5); got != stdinPartSize { + t.Errorf("stdinPartSizeFor(5) = %d", got) + } + if got := stdinPartSizeFor(stdinMaxPlainSize); got != stdinPartSize { + t.Errorf("stdinPartSizeFor(160 GiB) = %d", got) + } + if got := stdinPartSizeFor(1 << 40); got <= stdinPartSize || got%stdinPartSize != 0 || got*stdinMaxParts < 1<<40 { + t.Errorf("stdinPartSizeFor(1 TiB) = %d, want a multiple of 16 MiB that fits 10000 parts", got) + } +} + +func TestMvRemovesSource(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + t.Chdir(t.TempDir()) // display strings are relative to cwd + file := "m.txt" + writeFile(t, file, "move me") + + opts := baseOpts() + opts.Move = true + _, out, _, err := run(t, localEP(file), remoteEP(sess, "moved/"), opts) + if err != nil { + t.Fatal(err) + } + if out != "move: ./m.txt to s3://b/moved/m.txt\n" { + t.Errorf("out = %q", out) + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Error("local source must be removed after a successful upload") + } + if srv.Object("bucket-a", "moved/m.txt") == nil { + t.Fatal("object missing") + } + + // Remote -> remote move within the bucket (server-side copy + delete). + _, out, _, err = run(t, remoteEP(sess, "moved/m.txt"), remoteEP(sess, "archive/"), opts) + if err != nil { + t.Fatal(err) + } + if out != "move: s3://b/moved/m.txt to s3://b/archive/m.txt\n" { + t.Errorf("out = %q", out) + } + if srv.Object("bucket-a", "moved/m.txt") != nil || srv.Object("bucket-a", "archive/m.txt") == nil { + t.Errorf("keys after move = %v", srv.Keys("bucket-a")) + } + if string(srv.Object("bucket-a", "archive/m.txt").Data) != "move me" { + t.Error("copied data mismatch") + } + + // Same object is refused. + if _, err := buildPlan(context.Background(), remoteEP(sess, "archive/m.txt"), remoteEP(sess, "archive/"), opts); exitcode.Of(err) != exitcode.Usage { + t.Errorf("same object: %v", err) + } +} + +func TestMvKeepsSourceWhenCopyFails(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + file := filepath.Join(t.TempDir(), "keep.txt") + writeFile(t, file, "keep") + + opts := baseOpts() + opts.Move = true + srv.FailNext = &s3test.ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: http.StatusForbidden} + results, _, hints, err := run(t, localEP(file), remoteEP(sess, "x/"), opts) + if exitcode.Of(err) != exitcode.Permission { + t.Errorf("err = %v", err) + } + if _, statErr := os.Stat(file); statErr != nil { + t.Error("source must survive a failed copy") + } + // Single-item plans return the error (printed once by the command) and + // do not duplicate it as a failure line. + if strings.Contains(hints, "move failed: ") || len(results) != 1 || results[0].Error == "" { + t.Errorf("hints = %q results = %+v", hints, results) + } +} + +func TestCrossEndpointCopyRefused(t *testing.T) { + srvA, srvB := s3test.New(), s3test.New() + defer srvA.Close() + defer srvB.Close() + srvA.CreateBucket("bucket-a") + srvB.CreateBucket("bucket-b") + a, b := newTestSession(t, srvA, "bucket-a"), newTestSession(t, srvB, "bucket-b") + _, err := buildPlan(context.Background(), remoteEP(a, "k"), remoteEP(b, "k"), baseOpts()) + if exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "configure export --format rclone") { + t.Errorf("err = %v", err) + } +} + +func TestRecursiveFailuresContinue(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "1.txt"), "1") + writeFile(t, filepath.Join(dir, "2.txt"), "2") + + opts := baseOpts() + opts.Recursive = true + srv.FailNext = &s3test.ErrorResponse{Code: "InternalError", Message: "boom", Status: 500} + results, out, hints, err := run(t, localEP(dir), remoteEP(sess, ""), opts) + if exitcode.Of(err) != exitcode.Generic || !strings.Contains(err.Error(), "1 of 2 transfers failed") { + t.Errorf("err = %v", err) + } + if len(results) != 2 || results[0].Error == "" || results[1].Error != "" { + t.Errorf("results = %+v", results) + } + if !strings.Contains(hints, "upload failed: ") || strings.Count(out, "upload: ") != 1 { + t.Errorf("out=%q hints=%q", out, hints) + } +} + +func TestSyncPlan(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "new.txt"), "new") + writeFile(t, filepath.Join(dir, "same.txt"), "same") + writeFile(t, filepath.Join(dir, "bigger.txt"), "bigger now") + writeFile(t, filepath.Join(dir, "skip.tmp"), "tmp") + old := time.Now().Add(-2 * time.Hour) + for _, n := range []string{"same.txt", "bigger.txt"} { + if err := os.Chtimes(filepath.Join(dir, n), old, old); err != nil { + t.Fatal(err) + } + } + srv.AddObject("bucket-a", "site/same.txt", []byte("same"), "") + srv.AddObject("bucket-a", "site/bigger.txt", []byte("small"), "") + srv.AddObject("bucket-a", "site/stale.txt", []byte("stale"), "") + srv.AddObject("bucket-a", "site/keep.tmp", []byte("keep"), "") + + opts := baseOpts() + opts.Recursive = true + _ = opts.Filters.Add(false, "*.tmp") + plan, err := buildSyncPlan(context.Background(), localEP(dir), remoteEP(sess, "site"), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + var keys []string + for _, it := range plan.Items { + keys = append(keys, it.DstKey) + } + if got := strings.Join(keys, ","); got != "site/bigger.txt,site/new.txt" { + t.Errorf("planned uploads = %s", got) + } + if len(plan.Deletes) != 1 || plan.Deletes[0].Key != "site/stale.txt" { + t.Errorf("planned deletes = %+v (filters must protect keep.tmp)", plan.Deletes) + } + + var out, hints bytes.Buffer + results, err := runTransfers(context.Background(), plan, opts, &out, &hints) + if err != nil { + t.Fatalf("run: %v (%s)", err, hints.String()) + } + if got := strings.Join(srv.Keys("bucket-a"), ","); got != "site/bigger.txt,site/keep.tmp,site/new.txt,site/same.txt" { + t.Errorf("keys after sync = %s", got) + } + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(lines) != 3 || !strings.HasPrefix(lines[0], "upload: ") || lines[2] != "delete: s3://b/site/stale.txt" { + t.Errorf("lines = %q", lines) + } + if len(results) != 3 || results[2].Op != "delete" { + t.Errorf("results = %+v", results) + } + + // Second run: nothing to do. + plan, err = buildSyncPlan(context.Background(), localEP(dir), remoteEP(sess, "site"), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + if len(plan.Items) != 0 || len(plan.Deletes) != 0 { + t.Errorf("second sync must be a no-op: items=%+v deletes=%+v", plan.Items, plan.Deletes) + } + + // Size-only ignores a newer source of the same size. + writeFile(t, filepath.Join(dir, "same.txt"), "SAME") + future := time.Now().Add(time.Hour) + _ = os.Chtimes(filepath.Join(dir, "same.txt"), future, future) + plan, _ = buildSyncPlan(context.Background(), localEP(dir), remoteEP(sess, "site"), opts, syncOptions{SizeOnly: true}) + if len(plan.Items) != 0 { + t.Errorf("--size-only planned %+v", plan.Items) + } + plan, _ = buildSyncPlan(context.Background(), localEP(dir), remoteEP(sess, "site"), opts, syncOptions{}) + if len(plan.Items) != 1 || plan.Items[0].DstKey != "site/same.txt" { + t.Errorf("newer source must be uploaded: %+v", plan.Items) + } +} + +func TestSyncDownloadDryRunAndDelete(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "data/a.txt", []byte("a"), "") + srv.AddObject("bucket-a", "data/sub/b.txt", []byte("b"), "") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + t.Chdir(dir) // display strings are relative to cwd + writeFile(t, filepath.Join(dir, "extra.txt"), "extra") + + opts := baseOpts() + opts.Recursive, opts.DryRun = true, true + plan, err := buildSyncPlan(context.Background(), remoteEP(sess, "data/"), localEP("."), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + var out, hints bytes.Buffer + if _, err := runTransfers(context.Background(), plan, opts, &out, &hints); err != nil { + t.Fatal(err) + } + want := "(dryrun) download: s3://b/data/a.txt to ./a.txt\n" + + "(dryrun) download: s3://b/data/sub/b.txt to " + filepath.Join("sub", "b.txt") + "\n" + + "(dryrun) delete: ./extra.txt\n" + if out.String() != want { + t.Errorf("out = %q, want %q", out.String(), want) + } + if _, err := os.Stat(filepath.Join(dir, "extra.txt")); err != nil { + t.Error("dry-run deleted a local file") + } + + opts.DryRun = false + out.Reset() + if _, err := runTransfers(context.Background(), plan, opts, &out, &hints); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "sub", "b.txt")); string(got) != "b" { + t.Errorf("downloaded b.txt = %q", got) + } + if _, err := os.Stat(filepath.Join(dir, "extra.txt")); !os.IsNotExist(err) { + t.Error("--delete must remove local extras") + } + // Downloaded files carry the object's mtime so the next sync is a no-op. + plan, err = buildSyncPlan(context.Background(), remoteEP(sess, "data/"), localEP("."), opts, syncOptions{Delete: true, ExactTimestamps: true}) + if err != nil { + t.Fatal(err) + } + if len(plan.Items) != 0 || len(plan.Deletes) != 0 { + t.Errorf("re-sync must be a no-op: %+v %+v", plan.Items, plan.Deletes) + } +} + +func TestParseMetadataAndExpires(t *testing.T) { + m, err := parseMetadata([]string{"a=1,b=2", "c=x=y"}) + if err != nil || m["a"] != "1" || m["b"] != "2" || m["c"] != "x=y" { + t.Errorf("metadata = %v err=%v", m, err) + } + if _, err := parseMetadata([]string{"novalue"}); exitcode.Of(err) != exitcode.Usage { + t.Errorf("invalid metadata: %v", err) + } + if ts, err := parseExpires("2026-09-07"); err != nil || ts.Year() != 2026 || ts.Day() != 7 { + t.Errorf("date expires = %v %v", ts, err) + } + if _, err := parseExpires("2026-09-07T10:00:00Z"); err != nil { + t.Errorf("rfc3339 expires: %v", err) + } + if _, err := parseExpires("tomorrow"); exitcode.Of(err) != exitcode.Usage { + t.Errorf("bad expires: %v", err) + } +} + +// displayLocal follows aws (os.path.relpath from cwd): "./" only for a bare +// file name, directory components as-is, absolute paths made relative. +func TestDisplayLocal(t *testing.T) { + t.Chdir(t.TempDir()) + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + sep := string(filepath.Separator) + cases := map[string]string{ + "dump.sql": "." + sep + "dump.sql", + "." + sep + "dump.sql": "." + sep + "dump.sql", + filepath.Join("sub", "b.txt"): filepath.Join("sub", "b.txt"), + filepath.Join("..", "x"): filepath.Join("..", "x"), + filepath.Join(cwd, "dump.sql"): "." + sep + "dump.sql", + filepath.Join(cwd, "sub", "x"): filepath.Join("sub", "x"), + filepath.Join(filepath.Dir(cwd), "o"): filepath.Join("..", "o"), + ".": ".", + } + for in, want := range cases { + if got := displayLocal(in); got != want { + t.Errorf("displayLocal(%q) = %q, want %q", in, got, want) + } + } +} + +func TestCommandsRegisterFlags(t *testing.T) { + cp, mv, sync := NewCpCmd(), NewMvCmd(), NewSyncCmd() + for _, name := range []string{"recursive", "exclude", "include", "content-type", "metadata", "no-overwrite", "expected-size", "quiet", "only-show-errors", "no-progress", "follow-symlinks", "no-follow-symlinks", "project", "storage-class", "acl", "sse", "region"} { + if cp.Flags().Lookup(name) == nil { + t.Errorf("cp lacks --%s", name) + } + if mv.Flags().Lookup(name) == nil { + t.Errorf("mv lacks --%s", name) + } + } + for _, name := range []string{"delete", "size-only", "exact-timestamps", "exclude", "include", "project", "region"} { + if sync.Flags().Lookup(name) == nil { + t.Errorf("sync lacks --%s", name) + } + } + // `aws s3 cp ... --region x` gets the directed explanation (exit 2) on + // every transfer command. + for _, c := range []*cobra.Command{mv, sync} { + if err := c.Flags().Set("region", "us-east-1"); err != nil { + t.Fatal(err) + } + err := c.PreRunE(c, nil) + if exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--signing-region") { + t.Errorf("%s --region: %v", c.Name(), err) + } + } + if sync.Flags().Lookup("recursive") != nil { + t.Error("sync is always recursive and must not expose --recursive") + } + if !strings.Contains(cp.Long, "2026/09") || !strings.Contains(mv.Long, "2026/09") { + t.Error("destination rules must be documented in the help text") + } + // --storage-class is rejected with the Latitude explanation. + if err := cp.Flags().Set("storage-class", "STANDARD_IA"); err != nil { + t.Fatal(err) + } + err := cp.PreRunE(cp, nil) + if exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "lsh s3 mb --storage-class") { + t.Errorf("--storage-class: %v", err) + } +} + +func TestMultipartUploadWithProgress(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + t.Chdir(t.TempDir()) // display strings are relative to cwd + file := "big.bin" + data := bytes.Repeat([]byte{0xAB}, 17<<20) // above minio's 16 MiB part size + if err := os.WriteFile(file, data, 0o644); err != nil { + t.Fatal(err) + } + + opts := baseOpts() + opts.ShowProgress = true + results, out, hints, err := run(t, localEP(file), remoteEP(sess, "big.bin"), opts) + if err != nil { + t.Fatal(err) + } + obj := srv.Object("bucket-a", "big.bin") + if obj == nil || len(obj.Data) != len(data) || !bytes.Equal(obj.Data, data) { + t.Fatalf("multipart object mismatch (len %d)", len(obj.Data)) + } + if results[0].Size != int64(len(data)) || out != "upload: ./big.bin to s3://b/big.bin\n" { + t.Errorf("results = %+v out = %q", results, out) + } + if !strings.Contains(hints, "%)") || !strings.HasSuffix(hints, "\r\033[K") { + t.Errorf("progress must be drawn and cleared on hints: %q", hints) + } + if srv.HasChecksumHeaders() { + t.Error("multipart upload must not send checksum trailers") + } +} + +// TestSyncDeleteSkippedAfterFailure locks the sync safety rule: when a +// transfer fails, the planned deletions are not applied, so the destination is +// never left without both the object that failed and the one being pruned. +func TestSyncDeleteSkippedAfterFailure(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "data/a.txt", []byte("a"), "") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + t.Chdir(dir) // display strings are relative to cwd + writeFile(t, filepath.Join(dir, "extra.txt"), "extra") + // A directory where the download must write its file fails the transfer + // without involving the fake backend. + if err := os.MkdirAll(filepath.Join(dir, "a.txt"), 0o755); err != nil { + t.Fatal(err) + } + + opts := baseOpts() + opts.Recursive = true + plan, err := buildSyncPlan(context.Background(), remoteEP(sess, "data/"), localEP("."), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + if len(plan.Items) != 1 { + t.Fatalf("planned items = %+v, want the one download", plan.Items) + } + var planned []string + for _, d := range plan.Deletes { + planned = append(planned, d.Path) + } + if len(plan.Deletes) == 0 { + t.Fatalf("expected extra.txt to be planned for deletion, got %v", planned) + } + + var out, hints bytes.Buffer + results, err := runTransfers(context.Background(), plan, opts, &out, &hints) + if err == nil { + t.Fatal("expected the failed download to be reported") + } + if _, statErr := os.Stat(filepath.Join(dir, "extra.txt")); statErr != nil { + t.Errorf("a failed sync must not apply its deletions: %v", statErr) + } + if !strings.Contains(hints.String(), "not applied because") { + t.Errorf("hints = %q, want the skipped-deletions warning", hints.String()) + } + // Skipping the deletions must not turn the run into a single-item plan: + // the failure was already reported on hints, so returning the item's own + // error would make the command wrapper print it a second time. + if !strings.Contains(err.Error(), "transfers failed") { + t.Errorf("err = %v, want the summary error (the failure is already on stderr)", err) + } + if n := strings.Count(hints.String(), "download failed:"); n != 1 { + t.Errorf("the failure must be reported once, found %d times:\n%s", n, hints.String()) + } + for _, r := range results { + if r.Op == opDelete { + t.Errorf("delete was executed after a failure: %+v", r) + } + } +} + +// TestSyncDeleteStaysInsideDestination locks the containment rule for +// `sync --delete` on a local destination: the enumeration follows directory +// symlinks, so a deletion candidate reached through one resolves outside the +// tree the user pointed at and must not be removed. +func TestSyncDeleteStaysInsideDestination(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "data/keep.txt", []byte("keep"), "") + sess := newTestSession(t, srv, "bucket-a") + + outside := t.TempDir() + writeFile(t, filepath.Join(outside, "secret.txt"), "do not delete me") + dst := t.TempDir() + if err := os.Symlink(outside, filepath.Join(dst, "link")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + // A genuine destination-only file is the positive control. + writeFile(t, filepath.Join(dst, "stale.txt"), "stale") + + opts := baseOpts() + opts.Recursive = true + if !opts.FollowSymlinks { + t.Fatal("this test needs symlink following enabled") + } + plan, err := buildSyncPlan(context.Background(), remoteEP(sess, "data/"), localEP(dst), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + var planned []string + for _, d := range plan.Deletes { + planned = append(planned, d.Path) + if strings.Contains(d.Path, "link") || strings.HasPrefix(d.Path, outside) { + t.Errorf("planned a deletion outside the destination: %s", d.Path) + } + } + if len(plan.Deletes) != 1 || !strings.HasSuffix(plan.Deletes[0].Path, "stale.txt") { + t.Fatalf("planned deletes = %v, want only stale.txt", planned) + } + + var out, hints bytes.Buffer + if _, err := runTransfers(context.Background(), plan, opts, &out, &hints); err != nil { + t.Fatalf("run: %v (%s)", err, hints.String()) + } + if _, err := os.Stat(filepath.Join(outside, "secret.txt")); err != nil { + t.Errorf("a file outside the destination was deleted: %v", err) + } + if _, err := os.Stat(filepath.Join(dst, "stale.txt")); !os.IsNotExist(err) { + t.Errorf("the destination-only file should have been deleted: %v", err) + } +} + +func TestDeleteInsideRoot(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sub"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "link")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + resolved, err := filepath.EvalSymlinks(root) + if err != nil { + t.Fatal(err) + } + cases := []struct { + path string + want bool + }{ + {filepath.Join(root, "a.txt"), true}, + {filepath.Join(root, "sub", "a.txt"), true}, + // The link itself lives in the root: removing it only drops the link. + {filepath.Join(root, "link"), true}, + {filepath.Join(root, "link", "a.txt"), false}, + {filepath.Join(outside, "a.txt"), false}, + // A parent that cannot be resolved is never deleted through. + {filepath.Join(root, "missing", "a.txt"), false}, + } + for _, c := range cases { + if got := deleteInsideRoot(resolved, c.path); got != c.want { + t.Errorf("deleteInsideRoot(%s) = %v, want %v", c.path, got, c.want) + } + } +} + +// TestProgressThrottleWithUnknownTotal covers the throttle fix: with total 0 +// (a stdin upload) the "final frame" condition was always false, so every +// transport chunk redrew the stderr line. +func TestProgressThrottleWithUnknownTotal(t *testing.T) { + var buf bytes.Buffer + p := newProgressMeter(&buf, "upload", 0) + for i := 0; i < 50; i++ { + p.add(1024) + } + if n := strings.Count(buf.String(), "\r"); n > 1 { + t.Errorf("redrew the line %d times for 50 chunks; the throttle must apply when the total is unknown:\n%q", n, buf.String()) + } + // A known total still forces the last frame. + buf.Reset() + q := newProgressMeter(&buf, "upload", 2048) + q.add(1024) + q.add(1024) + if !strings.Contains(buf.String(), "upload") { + t.Errorf("a completed transfer must render its final frame, got %q", buf.String()) + } +} + +// TestDryRunToStdoutTargetReportsOnStderr covers `cp s3://b/key - --dryrun`: +// stdout carries the object, so the plan line has to go to stderr instead of +// being swallowed entirely. +func TestDryRunToStdoutTargetReportsOnStderr(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "readme.txt", []byte("hi"), "text/plain") + sess := newTestSession(t, srv, "bucket-a") + + opts := baseOpts() + opts.DryRun = true + _, out, hints, err := run(t, remoteEP(sess, "readme.txt"), stdioEP(), opts) + if err != nil { + t.Fatalf("dry run: %v", err) + } + if out != "" { + t.Errorf("stdout must stay empty for a '-' destination, got %q", out) + } + if !strings.Contains(hints, "(dryrun)") || !strings.Contains(hints, "readme.txt") { + t.Errorf("the plan must be reported on stderr, got %q", hints) + } +} + +// newTestSessionAs builds a session for one bucket with a named credential, so +// a test can hold two sessions with different access keys on one endpoint. +func newTestSessionAs(t *testing.T, srv *s3test.Server, display, backend, keyID string) *Session { + t.Helper() + b := &objectstorage.Bucket{ID: "bkt_" + backend, Name: display, BucketName: backend, Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: "us-east-1"} + cred := objectstorage.NewCredential(keyID, "SK", "profile") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +// TestCopyStreamsWhenCredentialsDiffer covers the cross-key fallback: a +// server-side CopyObject carries one identity, so when the two buckets resolved +// different keys the destination key cannot read the source. The object is +// streamed through the client instead of failing with an access denial. +func TestCopyStreamsWhenCredentialsDiffer(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("source-bucket") + srv.CreateBucket("target-bucket") + source := srv.AddObject("source-bucket", "data.txt", []byte("payload"), "text/plain") + source.Metadata = map[string]string{"owner": "ops"} + source.Headers = map[string]string{"Cache-Control": "max-age=60"} + + srcSess := newTestSessionAs(t, srv, "logs", "source-bucket", "AK1") + dstSess := newTestSessionAs(t, srv, "backups", "target-bucket", "AK2") + opts := baseOpts() + src, dst := remoteEP(srcSess, "data.txt"), remoteEP(dstSess, "data.txt") + plan, err := buildPlan(context.Background(), src, dst, opts) + if err != nil { + t.Fatal(err) + } + // Refuse the server-side copy only, as the backend would when the + // destination key has no read permission on the source bucket. Planning + // already happened, so the failure lands on CopyObject. + srv.FailNext = &s3test.ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: 403} + + var outBuf, hintsBuf bytes.Buffer + results, err := runTransfers(context.Background(), plan, opts, &outBuf, &hintsBuf) + hints := hintsBuf.String() + if err != nil { + t.Fatalf("copy across credentials: %v (%s)", err, hints) + } + obj := srv.Object("target-bucket", "data.txt") + if obj == nil { + t.Fatalf("the destination object was not written; keys=%v", srv.Keys("target-bucket")) + } + if string(obj.Data) != "payload" { + t.Errorf("destination object = %q, want the streamed payload", obj.Data) + } + // A streamed copy must keep what a server-side copy would have kept. + if obj.ContentType != "text/plain" { + t.Errorf("content type = %q, want the source's", obj.ContentType) + } + if obj.Metadata["owner"] != "ops" { + t.Errorf("user metadata = %v, want the source's owner=ops", obj.Metadata) + } + if obj.Headers["Cache-Control"] != "max-age=60" { + t.Errorf("headers = %v, want the source's Cache-Control", obj.Headers) + } + if !strings.Contains(hints, "different access keys") { + t.Errorf("the fallback must say why it streams:\n%s", hints) + } + if len(results) != 1 || results[0].Error != "" { + t.Errorf("results = %+v, want one successful copy", results) + } +} + +// TestNeedsStreamedCopy keeps the fallback narrow: only an authorization +// failure between two different credentials qualifies. +func TestNeedsStreamedCopy(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("alpha-bucket") + srv.CreateBucket("beta-bucket") + src := remoteEP(newTestSessionAs(t, srv, "src", "alpha-bucket", "AK1"), "k") + dstOther := remoteEP(newTestSessionAs(t, srv, "dst", "beta-bucket", "AK2"), "k") + dstSame := remoteEP(newTestSessionAs(t, srv, "dst", "beta-bucket", "AK1"), "k") + denied := minio.ErrorResponse{Code: "AccessDenied", StatusCode: 403} + missing := minio.ErrorResponse{Code: "NoSuchKey", StatusCode: 404} + + if !needsStreamedCopy(src, dstOther, denied) { + t.Error("access denied across two credentials must stream") + } + if needsStreamedCopy(src, dstSame, denied) { + t.Error("the same credential cannot be fixed by streaming") + } + if needsStreamedCopy(src, dstOther, missing) { + t.Error("a missing object would fail the same way twice") + } + if needsStreamedCopy(localEP("./f"), dstOther, denied) { + t.Error("only remote -> remote copies use CopyObject") + } +} diff --git a/cmd/s3/usage.go b/cmd/s3/usage.go new file mode 100644 index 0000000..bd99761 --- /dev/null +++ b/cmd/s3/usage.go @@ -0,0 +1,489 @@ +package s3 + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/latitudesh-go-sdk/types" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/utils" + "github.com/spf13/cobra" +) + +const ( + flagSince = "since" + flagUntil = "until" + flagUsageBucket = "bucket" + flagGroupBy = "group-by" + optHumanReadable = "human-readable" + + defaultUsageSince = "30d" + + // usageNameLookupMax caps the per-bucket calls used to map the API's + // numeric storage_id to bucket names; above it the raw ids are shown. + usageNameLookupMax = 50 + // usageLookupConcurrency bounds those parallel calls. + usageLookupConcurrency = metricsConcurrency +) + +// Usage groupings. +const ( + groupByDay = "day" + groupByBucket = "bucket" + groupByTier = "tier" + groupByRegion = "region" +) + +var usageGroupings = []string{groupByDay, groupByBucket, groupByTier, groupByRegion} + +// usageSample is one daily usage record after name resolution. +type usageSample struct { + Date string // YYYY-MM-DD + Project string + StorageID string // the API's numeric storage id ("2028"), not the bkt_ id + Bucket string // display name when known, else StorageID + Named bool // Bucket is a resolved display name + Tier string + Region string + Bytes int64 +} + +// UsageRow is an aggregated usage record (one per group key). +type UsageRow struct { + Date string `json:"date,omitempty"` + Project string `json:"project,omitempty"` + Bucket string `json:"bucket,omitempty"` + StorageID string `json:"storage_id,omitempty"` + Tier string `json:"tier,omitempty"` + Region string `json:"region,omitempty"` + // Bytes is the sum of the daily samples in the group (byte-days when the + // group spans several days). + Bytes int64 `json:"bytes"` + // Days is the number of distinct days in the group; AvgBytes = Bytes/Days. + Days int `json:"days,omitempty"` + AvgBytes int64 `json:"avg_bytes,omitempty"` + Size string `json:"size,omitempty"` // human size of Bytes (-H only) + AvgSize string `json:"avg_size,omitempty"` // human size of AvgBytes (-H only) + + groupBy string + human bool + // rawIDs is set when no bucket name could be resolved (too many buckets + // for the lookup, or every bucket gone): the table then shows the API's + // storage_id under "Storage ID" instead of a "Bucket" column. + rawIDs bool +} + +func (u UsageRow) TableRow() table.Row { + bytesCell := fmt.Sprintf("%d", u.Bytes) + avgCell := fmt.Sprintf("%d", u.AvgBytes) + if u.human { + bytesCell = objectstorage.HumanSize(u.Bytes) + avgCell = objectstorage.HumanSize(u.AvgBytes) + } + row := table.Row{ + "project": {Label: "Project", Value: u.Project}, + "tier": {Label: "Tier", Value: u.Tier}, + "region": {Label: "Region", Value: u.Region}, + "bytes": {Label: "Bytes", Value: bytesCell}, + } + if u.rawIDs { + row["storage_id"] = table.Cell{Label: "Storage ID", Value: u.StorageID} + } else { + row["bucket"] = table.Cell{Label: "Bucket", Value: u.Bucket} + } + if u.groupBy == groupByDay || u.groupBy == "" { + row["date"] = table.Cell{Label: "Date", Value: u.Date} + } else { + row["days"] = table.Cell{Label: "Days", Value: fmt.Sprintf("%d", u.Days)} + row["avg_bytes"] = table.Cell{Label: "Avg/Day", Value: avgCell} + } + return row +} + +// parseUsageWindow resolves --since/--until (utils.ParseTimeRef syntax) with +// the defaults since=30d and until=now, and rejects an inverted window. +func parseUsageWindow(since, until string, now time.Time) (time.Time, time.Time, error) { + since = strings.TrimSpace(since) + if since == "" { + since = defaultUsageSince + } + start, err := utils.ParseTimeRef(since, now) + if err != nil { + return time.Time{}, time.Time{}, exitcode.Errorf(exitcode.Usage, "--%s: %v", flagSince, err) + } + end := now + if u := strings.TrimSpace(until); u != "" && !strings.EqualFold(u, "now") { + end, err = utils.ParseTimeRef(u, now) + if err != nil { + return time.Time{}, time.Time{}, exitcode.Errorf(exitcode.Usage, "--%s: %v", flagUntil, err) + } + } + if start.After(end) { + return time.Time{}, time.Time{}, exitcode.Errorf(exitcode.Usage, "--%s (%s) is after --%s (%s)", flagSince, start.Format("2006-01-02"), flagUntil, end.Format("2006-01-02")) + } + return start, end, nil +} + +// validateGroupBy checks the --group-by value. +func validateGroupBy(v string) (string, error) { + v = strings.ToLower(strings.TrimSpace(v)) + if v == "" { + return groupByDay, nil + } + for _, g := range usageGroupings { + if v == g { + return g, nil + } + } + return "", exitcode.Errorf(exitcode.Usage, "invalid --%s %q: expected one of %s", flagGroupBy, v, strings.Join(usageGroupings, ", ")) +} + +// usageSamples converts API rows for one project. names maps the API's +// numeric storage ids to display names (see usageStorageIDNames); +// fallbackBucket is used when the request was filtered to one bucket and the +// id is unknown. +func usageSamples(data []components.StorageUsageData, project string, names map[string]string, fallbackBucket string) []usageSample { + out := make([]usageSample, 0, len(data)) + for _, d := range data { + a := d.Attributes + if a == nil { + continue + } + // The endpoint has no storage_type filter and reports object, file and + // block rows; summing them would inflate object storage usage. + if t := strings.ToLower(utils.Str(a.StorageType)); t != "" && t != "object" { + continue + } + s := usageSample{Project: project, StorageID: utils.Str(a.StorageID), Tier: utils.Str(a.Tier), Region: utils.Str(a.Region)} + if a.Date != nil { + s.Date = a.Date.String() + } + if a.Bytes != nil { + s.Bytes = *a.Bytes + } + s.Bucket = s.StorageID + if n, ok := names[s.StorageID]; ok && n != "" { + s.Bucket, s.Named = n, true + } else if fallbackBucket != "" { + s.Bucket, s.Named = fallbackBucket, true + } + out = append(out, s) + } + return out +} + +// aggregateUsage sums bytes per group. Every grouping keeps the project in +// the key so --all-projects stays readable; dimensions that are not part of +// the key are shown when uniform inside the group and left empty otherwise. +func aggregateUsage(samples []usageSample, groupBy string, human bool) []UsageRow { + type acc struct { + row UsageRow + days map[string]struct{} + bucket map[string]struct{} + sid map[string]struct{} + tier map[string]struct{} + region map[string]struct{} + } + groups := map[string]*acc{} + var order []string + var named bool + for _, s := range samples { + named = named || s.Named + var key string + switch groupBy { + case groupByBucket: + key = s.Project + "\x00" + s.StorageID + case groupByTier: + key = s.Project + "\x00" + s.Tier + case groupByRegion: + key = s.Project + "\x00" + s.Region + default: + key = s.Project + "\x00" + s.Date + } + g, ok := groups[key] + if !ok { + g = &acc{ + row: UsageRow{Project: s.Project, groupBy: groupBy, human: human}, + days: map[string]struct{}{}, + bucket: map[string]struct{}{}, + sid: map[string]struct{}{}, + tier: map[string]struct{}{}, + region: map[string]struct{}{}, + } + groups[key] = g + order = append(order, key) + } + g.row.Bytes += s.Bytes + g.days[s.Date] = struct{}{} + g.bucket[s.Bucket] = struct{}{} + g.sid[s.StorageID] = struct{}{} + g.tier[s.Tier] = struct{}{} + g.region[s.Region] = struct{}{} + } + + rows := make([]UsageRow, 0, len(groups)) + for _, key := range order { + g := groups[key] + r := g.row + r.Days = len(g.days) + if groupBy == groupByDay { + r.Date = uniform(g.days) + } + r.Bucket = uniform(g.bucket) + r.StorageID = uniform(g.sid) + r.Tier = uniform(g.tier) + r.Region = uniform(g.region) + if r.Days > 0 { + r.AvgBytes = r.Bytes / int64(r.Days) + } + if human { + r.Size = objectstorage.HumanSize(r.Bytes) + r.AvgSize = objectstorage.HumanSize(r.AvgBytes) + } + if groupBy == groupByDay { + // Per-day rows: Days is always 1 and the average equals Bytes. + r.Days, r.AvgBytes, r.AvgSize = 0, 0, "" + } + r.rawIDs = !named + rows = append(rows, r) + } + + sort.SliceStable(rows, func(i, j int) bool { + if groupBy == groupByDay { + if rows[i].Date != rows[j].Date { + return rows[i].Date < rows[j].Date + } + return rows[i].Project < rows[j].Project + } + if rows[i].Bytes != rows[j].Bytes { + return rows[i].Bytes > rows[j].Bytes + } + if rows[i].Project != rows[j].Project { + return rows[i].Project < rows[j].Project + } + return rows[i].Bucket+rows[i].Tier+rows[i].Region < rows[j].Bucket+rows[j].Tier+rows[j].Region + }) + return rows +} + +// uniform returns the single value of set, or "" when it has several. +func uniform(set map[string]struct{}) string { + if len(set) != 1 { + return "" + } + for v := range set { + return v + } + return "" +} + +// usageStorageIDNames maps the numeric storage_id carried by usage rows to +// bucket display names. The API exposes that id only through the usage rows +// themselves, but filter[storage_id] accepts the bkt_ id, so the window is +// queried once per bucket and the storage_id seen in the answer is recorded +// for that bucket. Calls run with bounded concurrency; a bucket whose call +// fails or returns no rows simply stays unmapped (its rows, if any, show the +// raw id). project is used for buckets that do not carry their own. +func usageStorageIDNames(ctx context.Context, api *sdk.Latitudesh, buckets []*objectstorage.Bucket, project string, start, end *types.Date, opts []operations.Option) map[string]string { + names := map[string]string{} + var mu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, usageLookupConcurrency) + for _, b := range buckets { + if b.ID == "" || b.Name == "" { + continue + } + wg.Add(1) + go func(b *objectstorage.Bucket) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + p := b.ProjectRef() + if p == "" { + p = project + } + id := b.ID + resp, err := api.ObjectStorage.GetStorageUsage(ctx, p, &id, start, end, opts...) + if err != nil { + lsh.LogDebugf("usage: could not map storage id of %s: %v", b.Display(), err) + return + } + if resp.StorageUsage == nil { + return + } + mu.Lock() + defer mu.Unlock() + for _, d := range resp.StorageUsage.Data { + if d.Attributes == nil { + continue + } + if sid := utils.Str(d.Attributes.StorageID); sid != "" { + names[sid] = b.Name + } + } + }(b) + } + wg.Wait() + return names +} + +// NewUsageCmd builds `lsh s3 usage`. +func NewUsageCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "usage", + Short: "Daily storage usage history", + Long: `Show the daily storage usage of a project (bytes stored per day), optionally +restricted to one bucket and aggregated per bucket, tier or region. + +--project is required unless --all-projects or --bucket is given. Dates accept +durations (30d, 2w, 24h) or ISO dates (2026-06-01). Aggregated rows report the +sum of the daily samples (bytes), the number of days and the average per day.`, + Example: ` lsh s3 usage --project my-project + lsh s3 usage --project my-project --since 7d --group-by bucket --human-readable + lsh s3 usage --bucket s3://backups --since 2026-08-01 --until 2026-08-31 -o csv + lsh s3 usage --all-projects --group-by region -o json`, + Args: cobra.NoArgs, + RunE: runUsage, + }) + // Registered as optional so the root pre-run never prompts; the command + // enforces the --project / --all-projects / --bucket requirement itself. + addProjectFlag(cmd, true, "project to report (ID or slug); required unless --all-projects or --bucket") + cmd.Flags().Bool(optAllProjects, false, "report every project that has buckets") + cmd.Flags().String(flagSince, defaultUsageSince, "start of the window: duration (30d, 2w) or date (2026-06-01)") + cmd.Flags().String(flagUntil, "", "end of the window: duration or date (default: now)") + cmd.Flags().String(flagUsageBucket, "", "only this bucket (s3://, name or bkt_ id)") + cmd.Flags().String(flagGroupBy, groupByDay, "aggregate rows by day, bucket, tier or region") + cmd.Flags().BoolP(optHumanReadable, "H", false, "show sizes in human-readable units") + return cmd +} + +func runUsage(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "usage comes from the Latitude API; unset --endpoint-url / %s to use this command", objectstorage.EnvEndpointURL)) + } + sinceFlag, _ := cmd.Flags().GetString(flagSince) + untilFlag, _ := cmd.Flags().GetString(flagUntil) + start, end, err := parseUsageWindow(sinceFlag, untilFlag, time.Now()) + if err != nil { + return printErr(err) + } + groupFlag, _ := cmd.Flags().GetString(flagGroupBy) + groupBy, err := validateGroupBy(groupFlag) + if err != nil { + return printErr(err) + } + human, _ := cmd.Flags().GetBool(optHumanReadable) + allProjects, _ := cmd.Flags().GetBool(optAllProjects) + if allProjects && cmd.Flags().Changed(flagProject) { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s and --%s are mutually exclusive", flagProject, optAllProjects)) + } + bucketFlag, _ := cmd.Flags().GetString(flagUsageBucket) + project := projectFlag(cmd) + if allProjects { + project = "" + } + + // Optional bucket filter; it can also supply the project. + var storageID *string + var bucketName string + if bucketFlag != "" { + ref, err := objectstorage.ParseBucketOnly(bucketFlag) + if err != nil { + return printErr(err) + } + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + if b.ID == "" { + return printErr(exitcode.Errorf(exitcode.Usage, "bucket %s has no API id; usage needs the Latitude API", b.Display())) + } + id := b.ID + storageID = &id + bucketName = b.Name + if project == "" && !allProjects { + project = b.ProjectRef() + } + } + if project == "" && !allProjects { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s is required (pass --%s=, --%s, --%s or set LSH_PROJECT)", flagProject, flagProject, optAllProjects, flagUsageBucket)) + } + + // Bucket list: the buckets whose storage ids get a name and, with + // --all-projects, the set of projects to query. + r := newResolver(cmd) + r.Project = project + list, err := r.ListBuckets(ctx) + if err != nil { + return printErr(err) + } + buckets := make([]*objectstorage.Bucket, 0, len(list)) + projectSet := map[string]struct{}{} + for _, d := range list { + b := objectstorage.BucketFromData(d) + buckets = append(buckets, b) + if p := b.ProjectRef(); p != "" { + projectSet[p] = struct{}{} + } + } + projects := []string{project} + if allProjects { + projects = projects[:0] + for p := range projectSet { + projects = append(projects, p) + } + sort.Strings(projects) + if len(projects) == 0 { + objectstorage.Hintf("no buckets found in any project; nothing to report") + render(nil) + return nil + } + } + + api := apiClient() + opts := []operations.Option{operations.WithRetries(lsh.RetryConfig())} + startDate, endDate := types.NewDate(start), types.NewDate(end) + + // The rows carry a numeric storage_id, not the bkt_ id, so names come from + // one filtered call per bucket. Skipped with --bucket (the filtered rows + // all belong to that bucket) and for very large bucket sets, where the + // table falls back to a Storage ID column. + var names map[string]string + switch { + case storageID != nil: + case len(buckets) > usageNameLookupMax: + objectstorage.Hintf("more than %d buckets; showing storage ids instead of bucket names", usageNameLookupMax) + default: + names = usageStorageIDNames(ctx, api, buckets, project, startDate, endDate, opts) + } + + var samples []usageSample + for _, p := range projects { + resp, err := api.ObjectStorage.GetStorageUsage(ctx, p, storageID, startDate, endDate, opts...) + if err != nil { + return printErr(objectstorage.HumanizeAPI(err, fmt.Sprintf("usage for project %q", p))) + } + if resp.StorageUsage == nil { + continue + } + samples = append(samples, usageSamples(resp.StorageUsage.Data, p, names, bucketName)...) + } + if len(samples) == 0 && isHuman() { + objectstorage.Hintf("no usage recorded between %s and %s", start.Format("2006-01-02"), end.Format("2006-01-02")) + } + render(objectstorage.AsResponseData(aggregateUsage(samples, groupBy, human))) + return nil +} diff --git a/cmd/s3/usage_test.go b/cmd/s3/usage_test.go new file mode 100644 index 0000000..fab716a --- /dev/null +++ b/cmd/s3/usage_test.go @@ -0,0 +1,411 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/types" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func TestParseUsageWindow(t *testing.T) { + now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) + + start, end, err := parseUsageWindow("", "", now) + if err != nil { + t.Fatalf("defaults: %v", err) + } + if !start.Equal(now.AddDate(0, 0, -30)) || !end.Equal(now) { + t.Errorf("defaults = %s..%s, want 30d..now", start, end) + } + + start, end, err = parseUsageWindow("7d", "now", now) + if err != nil || !start.Equal(now.Add(-7*24*time.Hour)) || !end.Equal(now) { + t.Errorf("7d..now = %s..%s (%v)", start, end, err) + } + + start, end, err = parseUsageWindow("2026-08-01", "2026-08-31", now) + if err != nil || start.Format("2006-01-02") != "2026-08-01" || end.Format("2006-01-02") != "2026-08-31" { + t.Errorf("absolute dates = %s..%s (%v)", start, end, err) + } + + start, end, err = parseUsageWindow("2w", "1d", now) + if err != nil || !start.Equal(now.Add(-14*24*time.Hour)) || !end.Equal(now.Add(-24*time.Hour)) { + t.Errorf("2w..1d = %s..%s (%v)", start, end, err) + } + + for _, c := range [][2]string{{"yesterday", ""}, {"", "soon"}, {"1d", "7d"}, {"2026-09-10", "2026-09-01"}} { + _, _, err := parseUsageWindow(c[0], c[1], now) + if err == nil { + t.Errorf("parseUsageWindow(%q, %q): expected error", c[0], c[1]) + continue + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("parseUsageWindow(%q, %q): exit %d, want %d", c[0], c[1], exitcode.Of(err), exitcode.Usage) + } + } +} + +func TestValidateGroupBy(t *testing.T) { + for in, want := range map[string]string{"": "day", "day": "day", "Bucket": "bucket", " tier ": "tier", "REGION": "region"} { + got, err := validateGroupBy(in) + if err != nil || got != want { + t.Errorf("validateGroupBy(%q) = %q, %v; want %q", in, got, err, want) + } + } + if _, err := validateGroupBy("week"); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("validateGroupBy(week) = %v", err) + } +} + +func usageDatum(date, storageID, tier, region string, bytes int64) components.StorageUsageData { + d := types.MustDateFromString(date) + return components.StorageUsageData{Attributes: &components.StorageUsageAttributes{ + Date: &d, StorageID: ptrS(storageID), Tier: ptrS(tier), Region: ptrS(region), Bytes: ptrI(bytes), StorageType: ptrS("object"), + }} +} + +// usageFixtureData is the raw API answer for one project. The storage_id the +// API returns is a numeric string ("2028"), not the bkt_ id, which is why the +// names map is keyed by those numbers (see usageStorageIDNames). +func usageFixtureData() []components.StorageUsageData { + return []components.StorageUsageData{ + usageDatum("2026-09-01", "2028", "standard", "DAL", 1000), + usageDatum("2026-09-01", "42", "high", "TYO4", 100), + usageDatum("2026-09-02", "2028", "standard", "DAL", 2000), + usageDatum("2026-09-02", "42", "high", "TYO4", 200), + usageDatum("2026-09-03", "2028", "standard", "DAL", 3000), + usageDatum("2026-09-03", "7", "standard", "DAL", 50), // unknown storage id + {}, // row without attributes is skipped + } +} + +func usageFixture() []usageSample { + names := map[string]string{"2028": "backups", "42": "logs"} + return usageSamples(usageFixtureData(), "my-project", names, "") +} + +func TestUsageSamples(t *testing.T) { + samples := usageFixture() + if len(samples) != 6 { + t.Fatalf("got %d samples, want 6", len(samples)) + } + if samples[0].Bucket != "backups" || samples[1].Bucket != "logs" || samples[5].Bucket != "7" { + t.Errorf("bucket names = %q %q %q", samples[0].Bucket, samples[1].Bucket, samples[5].Bucket) + } + if !samples[0].Named || !samples[1].Named || samples[5].Named { + t.Errorf("named flags = %v %v %v, want true true false", samples[0].Named, samples[1].Named, samples[5].Named) + } + if samples[0].StorageID != "2028" { + t.Errorf("storage id = %q, want the API's numeric id", samples[0].StorageID) + } + if samples[0].Date != "2026-09-01" || samples[0].Project != "my-project" || samples[0].Bytes != 1000 { + t.Errorf("sample = %+v", samples[0]) + } + // The bkt_ id never appears in usage rows, so a map keyed by it resolves nothing. + if byBkt := usageSamples(usageFixtureData(), "p", map[string]string{"bkt_1": "backups"}, ""); byBkt[0].Named { + t.Errorf("bkt_ keyed names must not match numeric storage ids: %+v", byBkt[0]) + } + // With a bucket filter, unknown ids fall back to the filtered bucket's name. + filtered := usageSamples([]components.StorageUsageData{usageDatum("2026-09-01", "99", "standard", "DAL", 1)}, "p", nil, "backups") + if filtered[0].Bucket != "backups" || !filtered[0].Named { + t.Errorf("fallback bucket = %+v", filtered[0]) + } +} + +// usageStorageIDs is the fake API's bkt_ id -> numeric storage_id table. +var usageStorageIDs = map[string]string{"bkt_1": "2028", "bkt_2": "42", "bkt_3": "300"} + +// fakeUsageAPI serves GET /storage/usage. With filter[storage_id]= it +// answers rows carrying that bucket's numeric storage_id (the real API +// behaviour); bkt_err fails with 500 and bkt_empty has no rows. It records +// the peak concurrency and the projects it was asked for. +func fakeUsageAPI(t *testing.T, peak *int32, projects *sync.Map) *httptest.Server { + var inflight int32 + var mu sync.Mutex + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cur := atomic.AddInt32(&inflight, 1) + defer atomic.AddInt32(&inflight, -1) + mu.Lock() + if cur > *peak { + *peak = cur + } + mu.Unlock() + time.Sleep(20 * time.Millisecond) + + if r.URL.Path != "/storage/usage" { + http.Error(w, "unexpected path "+r.URL.Path, http.StatusNotFound) + return + } + q := r.URL.Query() + if q.Get("filter[start_date]") == "" || q.Get("filter[end_date]") == "" { + t.Errorf("lookup must carry the usage window, got %s", r.URL.RawQuery) + } + id := q.Get("filter[storage_id]") + projects.Store(id, q.Get("filter[project]")) + w.Header().Set("Content-Type", "application/vnd.api+json") + switch id { + case "bkt_err": + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"errors":[{"status":"500","title":"Internal Server Error","detail":"boom"}]}`) + return + case "bkt_empty": + fmt.Fprint(w, `{"data":[]}`) + return + } + sid := usageStorageIDs[id] + fmt.Fprintf(w, `{"data":[ + {"id":"u1","type":"storage_usage","attributes":{"date":"2026-09-01","storage_id":%q,"project_id":"1","storage_type":"object","tier":"standard","region":"DAL","bytes":1000}}, + {"id":"u2","type":"storage_usage","attributes":{"date":"2026-09-02","storage_id":%q,"project_id":"1","storage_type":"object","tier":"standard","region":"DAL","bytes":2000}} + ]}`, sid, sid) + })) +} + +func TestUsageStorageIDNames(t *testing.T) { + var peak int32 + var projects sync.Map + srv := fakeUsageAPI(t, &peak, &projects) + defer srv.Close() + api := sdk.New(sdk.WithServerURL(srv.URL), sdk.WithSecurity("test-token")) + + buckets := []*objectstorage.Bucket{ + {ID: "bkt_1", Name: "backups", ProjectSlug: "proj"}, + {ID: "bkt_2", Name: "logs", ProjectSlug: "proj"}, + {ID: "bkt_3", Name: "no-project"}, // falls back to the command's project + {ID: "bkt_err", Name: "broken", ProjectSlug: "proj"}, + {ID: "bkt_empty", Name: "idle", ProjectSlug: "proj"}, + {ID: "", Name: "no-id"}, // skipped: nothing to filter on + } + // Pad with more buckets than the concurrency bound to observe the fan-out. + for i := 10; i < 30; i++ { + buckets = append(buckets, &objectstorage.Bucket{ID: fmt.Sprintf("bkt_%d", i), Name: fmt.Sprintf("b%d", i), ProjectSlug: "proj"}) + } + start, end := types.NewDate(time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)), types.NewDate(time.Date(2026, 9, 7, 0, 0, 0, 0, time.UTC)) + names := usageStorageIDNames(context.Background(), api, buckets, "fallback-proj", start, end, nil) + + want := map[string]string{"2028": "backups", "42": "logs", "300": "no-project"} + for sid, name := range want { + if names[sid] != name { + t.Errorf("names[%q] = %q, want %q", sid, names[sid], name) + } + } + // Padding buckets have no storage id in the fake table and all answer "", + // which must not be recorded; failures and empty answers stay unmapped. + if _, ok := names[""]; ok { + t.Errorf("empty storage ids must not be recorded: %v", names) + } + for sid, name := range names { + if _, ok := want[sid]; !ok { + t.Errorf("unexpected mapping %q -> %q", sid, name) + } + } + if peak > usageLookupConcurrency { + t.Errorf("peak concurrency %d exceeds %d", peak, usageLookupConcurrency) + } + if peak < 2 { + t.Errorf("expected parallel requests, peak was %d", peak) + } + if p, _ := projects.Load("bkt_1"); p != "proj" { + t.Errorf("bkt_1 queried with project %v, want proj", p) + } + if p, _ := projects.Load("bkt_3"); p != "fallback-proj" { + t.Errorf("bucket without project should use the command's project, got %v", p) + } + if _, asked := projects.Load(""); asked { + t.Errorf("a bucket without id must not trigger a lookup") + } + + // The mapped names resolve the rows of the unfiltered call. + samples := usageSamples(usageFixtureData(), "proj", names, "") + if samples[0].Bucket != "backups" || samples[1].Bucket != "logs" || samples[5].Bucket != "7" { + t.Errorf("resolved buckets = %q %q %q", samples[0].Bucket, samples[1].Bucket, samples[5].Bucket) + } +} + +func TestAggregateUsageRawStorageIDs(t *testing.T) { + // No names at all (too many buckets for the lookup): the table shows the + // API's storage_id under "Storage ID" instead of an id-filled Bucket column. + raw := aggregateUsage(usageSamples(usageFixtureData(), "p", nil, ""), groupByBucket, false) + if len(raw) != 3 { + t.Fatalf("got %d rows, want 3", len(raw)) + } + row := raw[0].TableRow() + if _, ok := row["bucket"]; ok { + t.Errorf("unresolved rows must not have a Bucket column: %+v", row) + } + if got := row["storage_id"]; got.Label != "Storage ID" || got.Value != "2028" { + t.Errorf("storage id column = %+v", got) + } + // JSON keeps both fields; bucket falls back to the storage id. + out, _ := json.Marshal(raw[0]) + if !strings.Contains(string(out), `"storage_id":"2028"`) || !strings.Contains(string(out), `"bucket":"2028"`) { + t.Errorf("json = %s", out) + } + + // As soon as one name resolves the Bucket column is back (unknown ids are + // shown as-is inside it). + named := aggregateUsage(usageFixture(), groupByBucket, false) + for _, r := range named { + row := r.TableRow() + if _, ok := row["storage_id"]; ok { + t.Errorf("resolved rows must not have a Storage ID column: %+v", row) + } + if row["bucket"].Label != "Bucket" { + t.Errorf("bucket column = %+v", row["bucket"]) + } + } + if named[2].TableRow()["bucket"].Value != "7" { + t.Errorf("unknown id inside the Bucket column = %q", named[2].TableRow()["bucket"].Value) + } +} + +func TestAggregateUsageByDay(t *testing.T) { + rows := aggregateUsage(usageFixture(), groupByDay, false) + if len(rows) != 3 { + t.Fatalf("got %d rows, want 3: %+v", len(rows), rows) + } + wantBytes := map[string]int64{"2026-09-01": 1100, "2026-09-02": 2200, "2026-09-03": 3050} + for i, r := range rows { + if r.Bytes != wantBytes[r.Date] { + t.Errorf("%s bytes = %d, want %d", r.Date, r.Bytes, wantBytes[r.Date]) + } + if r.Project != "my-project" { + t.Errorf("project = %q", r.Project) + } + // Several buckets per day: bucket is never uniform; tier/region are + // blank when mixed (09-01, 09-02) and kept when shared (09-03). + if r.Bucket != "" || r.StorageID != "" { + t.Errorf("%s: mixed buckets should be blank: %+v", r.Date, r) + } + if r.Date == "2026-09-03" { + if r.Tier != "standard" || r.Region != "DAL" { + t.Errorf("%s: uniform tier/region should be kept: %+v", r.Date, r) + } + } else if r.Tier != "" || r.Region != "" { + t.Errorf("%s: mixed tier/region should be blank: %+v", r.Date, r) + } + if i > 0 && rows[i-1].Date > r.Date { + t.Errorf("rows not sorted by date: %v", rows) + } + if r.Days != 0 || r.AvgBytes != 0 { + t.Errorf("day rows must not carry days/avg: %+v", r) + } + row := r.TableRow() + if _, ok := row["date"]; !ok { + t.Errorf("day grouping should have a date column") + } + if _, ok := row["days"]; ok { + t.Errorf("day grouping should not have a days column") + } + } +} + +func TestAggregateUsageByBucket(t *testing.T) { + rows := aggregateUsage(usageFixture(), groupByBucket, true) + if len(rows) != 3 { + t.Fatalf("got %d rows, want 3: %+v", len(rows), rows) + } + // Sorted by bytes desc. + if rows[0].Bucket != "backups" || rows[0].Bytes != 6000 || rows[0].Days != 3 || rows[0].AvgBytes != 2000 { + t.Errorf("backups row = %+v", rows[0]) + } + if rows[0].Tier != "standard" || rows[0].Region != "DAL" || rows[0].StorageID != "2028" || rows[0].Date != "" { + t.Errorf("uniform dimensions should be kept: %+v", rows[0]) + } + if rows[1].Bucket != "logs" || rows[1].Bytes != 300 || rows[1].Days != 2 || rows[1].AvgBytes != 150 { + t.Errorf("logs row = %+v", rows[1]) + } + if rows[2].Bucket != "7" || rows[2].Bytes != 50 || rows[2].Days != 1 { + t.Errorf("unknown bucket row = %+v", rows[2]) + } + // -H fills the human sizes and the table shows them. + if rows[0].Size != "5.9 KiB" || rows[0].AvgSize != "2.0 KiB" { + t.Errorf("human sizes = %q %q", rows[0].Size, rows[0].AvgSize) + } + row := rows[0].TableRow() + if row["bytes"].Value != "5.9 KiB" || row["avg_bytes"].Value != "2.0 KiB" || row["days"].Value != "3" { + t.Errorf("table row = %+v", row) + } + if _, ok := row["date"]; ok { + t.Errorf("bucket grouping should not have a date column") + } +} + +func TestAggregateUsageByTierAndRegion(t *testing.T) { + tiers := aggregateUsage(usageFixture(), groupByTier, false) + if len(tiers) != 2 || tiers[0].Tier != "standard" || tiers[0].Bytes != 6050 || tiers[1].Tier != "high" || tiers[1].Bytes != 300 { + t.Errorf("tier rows = %+v", tiers) + } + // standard spans two buckets: bucket is blank, region is uniform. + if tiers[0].Bucket != "" || tiers[0].Region != "DAL" { + t.Errorf("standard tier dimensions = %+v", tiers[0]) + } + regions := aggregateUsage(usageFixture(), groupByRegion, false) + if len(regions) != 2 || regions[0].Region != "DAL" || regions[0].Bytes != 6050 || regions[1].Region != "TYO4" || regions[1].Bytes != 300 { + t.Errorf("region rows = %+v", regions) + } + + // Projects stay separate under --all-projects. + mixed := append(usageFixture(), usageSample{Date: "2026-09-01", Project: "other", StorageID: "x", Bucket: "x", Tier: "standard", Region: "DAL", Bytes: 1}) + byTier := aggregateUsage(mixed, groupByTier, false) + if len(byTier) != 3 { + t.Errorf("expected a separate row per project, got %+v", byTier) + } + if empty := aggregateUsage(nil, groupByDay, false); len(empty) != 0 { + t.Errorf("no samples should yield no rows, got %+v", empty) + } +} + +func TestUsageRowJSONShape(t *testing.T) { + rows := aggregateUsage(usageFixture(), groupByBucket, false) + raw, err := json.Marshal(rows[0]) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"project":"my-project"`, `"bucket":"backups"`, `"storage_id":"2028"`, `"tier":"standard"`, `"region":"DAL"`, `"bytes":6000`, `"days":3`, `"avg_bytes":2000`} { + if !strings.Contains(string(raw), want) { + t.Errorf("json %s lacks %s", raw, want) + } + } + if strings.Contains(string(raw), `"size"`) || strings.Contains(string(raw), `"date"`) || strings.Contains(string(raw), "groupBy") { + t.Errorf("unexpected fields in %s", raw) + } +} + +// TestUsageSamplesSkipsOtherStorageTypes covers the aggregation fix: the +// endpoint has no storage_type filter and returns object, file and block rows, +// so summing everything would inflate object storage usage. +func TestUsageSamplesSkipsOtherStorageTypes(t *testing.T) { + str := func(s string) *string { return &s } + i64 := func(n int64) *int64 { return &n } + data := []components.StorageUsageData{ + {Attributes: &components.StorageUsageAttributes{StorageID: str("1"), StorageType: str("object"), Bytes: i64(100)}}, + {Attributes: &components.StorageUsageAttributes{StorageID: str("2"), StorageType: str("file"), Bytes: i64(900)}}, + {Attributes: &components.StorageUsageAttributes{StorageID: str("3"), StorageType: str("block"), Bytes: i64(500)}}, + // An older payload without the field is kept: this endpoint is the + // object storage one, so an unset type is treated as object. + {Attributes: &components.StorageUsageAttributes{StorageID: str("4"), Bytes: i64(7)}}, + } + got := usageSamples(data, "proj_1", nil, "") + if len(got) != 2 { + t.Fatalf("kept %d samples, want the 2 object rows: %+v", len(got), got) + } + var total int64 + for _, s := range got { + total += s.Bytes + } + if total != 107 { + t.Errorf("total = %d bytes, want 107 (file/block rows must not be summed)", total) + } +} diff --git a/cmd/servers/rescue.go b/cmd/servers/rescue.go index 6c624b5..2ea5726 100644 --- a/cmd/servers/rescue.go +++ b/cmd/servers/rescue.go @@ -5,7 +5,6 @@ import ( "fmt" "os" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" "github.com/latitudesh/lsh/cmd/lsh" "github.com/latitudesh/lsh/internal/utils" @@ -57,8 +56,8 @@ func runRescueMode(cmd *cobra.Command, args []string) error { // dashboard. stderr keeps stdout clean for structured output. fmt.Fprintln(os.Stderr, "Note: rescue login credentials are available on the server's page in the dashboard.") - want := []components.ServerDataStatus{components.ServerDataStatusRescueMode} - fail := []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} + want := []wait.ServerStatus{wait.ServerStatusRescueMode} + fail := []wait.ServerStatus{wait.ServerStatusFailedDeployment} // Idempotent wait: a server already in the target state is already done — // requiring a transition here would hang until timeout. return waitForServerState(cmd, serverID, want, fail, false) @@ -104,11 +103,11 @@ func runExitRescueMode(cmd *cobra.Command, args []string) error { } // Leaving rescue mode reboots into the installed OS; it settles on/off. - want := []components.ServerDataStatus{ - components.ServerDataStatusOn, - components.ServerDataStatusOff, + want := []wait.ServerStatus{ + wait.ServerStatusOn, + wait.ServerStatusOff, } - fail := []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} + fail := []wait.ServerStatus{wait.ServerStatusFailedDeployment} // Idempotent wait: a server already in the target state is already done — // requiring a transition here would hang until timeout. return waitForServerState(cmd, serverID, want, fail, false) diff --git a/cmd/servers/servers_test.go b/cmd/servers/servers_test.go index 5c1547e..5350f97 100644 --- a/cmd/servers/servers_test.go +++ b/cmd/servers/servers_test.go @@ -5,6 +5,7 @@ import ( "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/wait" ) func strp(s string) *string { return &s } @@ -48,17 +49,17 @@ func TestActionsCmdRequiresArg(t *testing.T) { // TestPowerActionTargets verifies each action waits for the correct terminal // power state. func TestPowerActionTargets(t *testing.T) { - cases := map[string]components.ServerDataStatus{ - "power_on": components.ServerDataStatusOn, - "reboot": components.ServerDataStatusOn, - "power_off": components.ServerDataStatusOff, + cases := map[string]wait.ServerStatus{ + "power_on": wait.ServerStatusOn, + "reboot": wait.ServerStatusOn, + "power_off": wait.ServerStatusOff, } for action, wantState := range cases { want, fail := powerActionTargets(action) if len(want) != 1 || want[0] != wantState { t.Errorf("powerActionTargets(%q) want = %v, expected [%v]", action, want, wantState) } - if len(fail) != 1 || fail[0] != components.ServerDataStatusFailedDeployment { + if len(fail) != 1 || fail[0] != wait.ServerStatusFailedDeployment { t.Errorf("powerActionTargets(%q) fail = %v, expected [failed_deployment]", action, fail) } } diff --git a/cmd/servers/wait.go b/cmd/servers/wait.go index 6094ee6..2697afa 100644 --- a/cmd/servers/wait.go +++ b/cmd/servers/wait.go @@ -6,7 +6,6 @@ import ( "fmt" "os" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" "github.com/latitudesh/lsh/cmd/lsh" "github.com/latitudesh/lsh/internal/wait" @@ -21,7 +20,7 @@ import ( // requireTransition guards operations that act on a server which may already // sit in a target state (e.g. power_on on an already-on server) so the wait // does not return before the operation has actually taken effect. -func waitForServerState(cmd *cobra.Command, serverID string, want, fail []components.ServerDataStatus, requireTransition bool) error { +func waitForServerState(cmd *cobra.Command, serverID string, want, fail []wait.ServerStatus, requireTransition bool) error { o := wait.OptionsFrom(cmd) if !o.Enabled { if cmd.Flags().Changed("timeout") { @@ -58,13 +57,13 @@ func waitForServerState(cmd *cobra.Command, serverID string, want, fail []compon // powerActionTargets maps a power action to the server states that satisfy the // wait (want) and the states that abort it (fail). -func powerActionTargets(action string) (want, fail []components.ServerDataStatus) { - fail = []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} +func powerActionTargets(action string) (want, fail []wait.ServerStatus) { + fail = []wait.ServerStatus{wait.ServerStatusFailedDeployment} switch action { case "power_on", "reboot": - return []components.ServerDataStatus{components.ServerDataStatusOn}, fail + return []wait.ServerStatus{wait.ServerStatusOn}, fail case "power_off": - return []components.ServerDataStatus{components.ServerDataStatusOff}, fail + return []wait.ServerStatus{wait.ServerStatusOff}, fail default: return nil, fail } diff --git a/cmd/sshkeys/create.go b/cmd/sshkeys/create.go index 9cb45b9..c0eab49 100644 --- a/cmd/sshkeys/create.go +++ b/cmd/sshkeys/create.go @@ -49,8 +49,8 @@ func (o *CreateSSHKeyOperation) run(cmd *cobra.Command, args []string) error { Data: operations.PostSSHKeySSHKeysData{ Type: operations.PostSSHKeySSHKeysTypeSSHKeys, Attributes: &operations.PostSSHKeySSHKeysAttributes{ - Name: &name, - PublicKey: &publicKey, + Name: name, + PublicKey: publicKey, }, }, } diff --git a/cmd/sshkeys/sshkeys_test.go b/cmd/sshkeys/sshkeys_test.go index aa6f0b8..b9c5b16 100644 --- a/cmd/sshkeys/sshkeys_test.go +++ b/cmd/sshkeys/sshkeys_test.go @@ -106,15 +106,15 @@ func TestCreateBuildsRequestBody(t *testing.T) { Data: operations.PostSSHKeySSHKeysData{ Type: operations.PostSSHKeySSHKeysTypeSSHKeys, Attributes: &operations.PostSSHKeySSHKeysAttributes{ - Name: &name, - PublicKey: &publicKey, + Name: name, + PublicKey: publicKey, }, }, } if request.Data.Type != "ssh_keys" { t.Errorf("type = %q, want ssh_keys", request.Data.Type) } - if *request.Data.Attributes.Name != name || *request.Data.Attributes.PublicKey != publicKey { + if request.Data.Attributes.Name != name || request.Data.Attributes.PublicKey != publicKey { t.Error("attributes not wired through") } } diff --git a/cmd/sshkeys/update.go b/cmd/sshkeys/update.go index 7daed4f..2cc1f53 100644 --- a/cmd/sshkeys/update.go +++ b/cmd/sshkeys/update.go @@ -59,7 +59,7 @@ func (o *UpdateSSHKeyOperation) run(cmd *cobra.Command, args []string) error { request := operations.PutSSHKeySSHKeysRequestBody{ Data: operations.PutSSHKeySSHKeysData{ - ID: &sshKeyID, + ID: sshKeyID, Type: operations.PutSSHKeySSHKeysTypeSSHKeys, Attributes: attributes, }, diff --git a/cmd/storage_filesystems/update.go b/cmd/storage_filesystems/update.go index 1c5cc57..4df8331 100644 --- a/cmd/storage_filesystems/update.go +++ b/cmd/storage_filesystems/update.go @@ -33,7 +33,7 @@ type UpdateFilesystemOperation struct{} func buildUpdateRequest(sizeSet bool, size int64) operations.PatchStorageFilesystemsFilesystemStorageRequestBody { attributes := operations.PatchStorageFilesystemsFilesystemStorageAttributes{} if sizeSet { - attributes.SizeInGb = &size + attributes.SizeInGb = size } return operations.PatchStorageFilesystemsFilesystemStorageRequestBody{ diff --git a/cmd/storage_filesystems/update_test.go b/cmd/storage_filesystems/update_test.go index 201cd70..9259456 100644 --- a/cmd/storage_filesystems/update_test.go +++ b/cmd/storage_filesystems/update_test.go @@ -11,15 +11,15 @@ func TestBuildUpdateRequestSizeSet(t *testing.T) { if req.Data.Type != operations.PatchStorageFilesystemsFilesystemStorageTypeFilesystems { t.Errorf("Type = %q, want filesystems", req.Data.Type) } - if req.Data.Attributes.SizeInGb == nil || *req.Data.Attributes.SizeInGb != 3000 { + if req.Data.Attributes.SizeInGb != 3000 { t.Errorf("SizeInGb = %v, want 3000", req.Data.Attributes.SizeInGb) } } func TestBuildUpdateRequestSizeOmitted(t *testing.T) { req := buildUpdateRequest(false, 0) - if req.Data.Attributes.SizeInGb != nil { - t.Errorf("SizeInGb = %v, want nil when --size not set", req.Data.Attributes.SizeInGb) + if req.Data.Attributes.SizeInGb != 0 { + t.Errorf("SizeInGb = %v, want 0 when --size not set", req.Data.Attributes.SizeInGb) } } diff --git a/cmd/storage_objects/create.go b/cmd/storage_objects/create.go deleted file mode 100644 index 4d93ad2..0000000 --- a/cmd/storage_objects/create.go +++ /dev/null @@ -1,134 +0,0 @@ -package storage_objects - -import ( - "context" - "fmt" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" - "github.com/latitudesh/lsh/cmd/lsh" - "github.com/latitudesh/lsh/internal/tui" - "github.com/latitudesh/lsh/internal/utils" - cobra "github.com/spf13/cobra" -) - -func NewCreateCmd() *cobra.Command { - o := CreateBucketOperation{} - cmd := &cobra.Command{ - Long: "Create an object storage bucket in a project.\n\n" + - "Buckets are S3-compatible. Optional flags enable versioning, object\n" + - "lock (WORM) and a higher-performance storage class where available.\n", - RunE: o.run, - Short: "Create a bucket", - Example: ` lsh storage-objects create --project my-project --name my-bucket --region SAO2 - lsh storage-objects create --project my-project --name logs --region DAL --storage-class high_performance --versioning`, - Use: "create", - } - - cmd.Flags().String("project", "", "Project ID or slug to create the bucket in") - cmd.Flags().String("name", "", "Bucket name (no special characters or spaces)") - cmd.Flags().String("region", "", "Site slug representing the region (e.g. DAL, SAO2)") - cmd.Flags().String("storage-class", "", "Storage tier: standard or high_performance") - cmd.Flags().Bool("versioning", false, "Enable S3 object versioning") - cmd.Flags().Bool("locking", false, "Enable S3 Object Lock (WORM); implies versioning") - - return cmd -} - -type CreateBucketOperation struct{} - -// buildCreateRequest turns parsed flags into the SDK request body. Project, -// name and region are required. Optional toggles are only set when supplied so -// the API defaults apply otherwise. Split out for unit testing. -func buildCreateRequest(project, name, region, storageClass string, versioningSet, versioning, lockingSet, locking bool) (operations.PostStorageBucketsRequestBody, error) { - if project == "" { - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("--project is required") - } - if name == "" { - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("--name is required") - } - if region == "" { - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("--region is required") - } - - attributes := operations.PostStorageBucketsAttributes{ - Project: project, - Name: name, - Region: region, - } - - if storageClass != "" { - switch storageClass { - case string(operations.StorageClassStandard), string(operations.StorageClassHighPerformance): - sc := operations.StorageClass(storageClass) - attributes.StorageClass = &sc - default: - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("invalid --storage-class %q (expected standard or high_performance)", storageClass) - } - } - if versioningSet { - attributes.Versioning = &versioning - } - if lockingSet { - // The API requires versioning on locked buckets; enforce the documented - // implication client-side instead of surfacing an opaque server error. - if locking && versioningSet && !versioning { - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("--locking requires versioning; drop --versioning=false or enable it") - } - if locking && !versioningSet { - enabled := true - attributes.Versioning = &enabled - } - attributes.Locking = &locking - } - - return operations.PostStorageBucketsRequestBody{ - Data: operations.PostStorageBucketsData{ - Type: operations.PostStorageBucketsTypeObjects, - Attributes: attributes, - }, - }, nil -} - -func (o *CreateBucketOperation) run(cmd *cobra.Command, args []string) error { - project, _ := cmd.Flags().GetString("project") - name, _ := cmd.Flags().GetString("name") - region, _ := cmd.Flags().GetString("region") - storageClass, _ := cmd.Flags().GetString("storage-class") - versioning, _ := cmd.Flags().GetBool("versioning") - locking, _ := cmd.Flags().GetBool("locking") - - request, err := buildCreateRequest( - project, name, region, storageClass, - cmd.Flags().Changed("versioning"), versioning, - cmd.Flags().Changed("locking"), locking, - ) - if err != nil { - utils.PrintError(err) - return err - } - - if lsh.DryRun { - lsh.LogDebugf("dry-run flag specified. Skip sending request.") - return nil - } - - client := lsh.NewClient() - ctx := context.Background() - - response, err := client.ObjectStorage.PostStorageBuckets(ctx, request, operations.WithRetries(lsh.RetryConfig())) - if err != nil { - utils.PrintError(err) - return err - } - - if response.Object != nil && response.Object.Data != nil { - fmt.Println(tui.SuccessStyle.Render("✓ Bucket created successfully!")) - - if !lsh.Debug { - bucket := Bucket{ObjectStorageData: *response.Object.Data} - utils.RenderStatic(bucket.GetData()) - } - } - - return nil -} diff --git a/cmd/storage_objects/create_test.go b/cmd/storage_objects/create_test.go deleted file mode 100644 index 941931d..0000000 --- a/cmd/storage_objects/create_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package storage_objects - -import ( - "testing" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" -) - -func TestBuildCreateRequest(t *testing.T) { - req, err := buildCreateRequest("my-project", "my-bucket", "SAO2", "high_performance", true, true, false, false) - if err != nil { - t.Fatalf("buildCreateRequest returned error: %v", err) - } - - if req.Data.Type != operations.PostStorageBucketsTypeObjects { - t.Errorf("Type = %q, want objects", req.Data.Type) - } - attr := req.Data.Attributes - if attr.Project != "my-project" || attr.Name != "my-bucket" || attr.Region != "SAO2" { - t.Errorf("unexpected core attributes: %+v", attr) - } - if attr.StorageClass == nil || *attr.StorageClass != operations.StorageClassHighPerformance { - t.Errorf("StorageClass = %v, want high_performance", attr.StorageClass) - } - if attr.Versioning == nil || !*attr.Versioning { - t.Errorf("Versioning = %v, want true", attr.Versioning) - } - if attr.Locking != nil { - t.Errorf("Locking = %v, want nil when --locking not set", attr.Locking) - } -} - -func TestBuildCreateRequestDefaults(t *testing.T) { - req, err := buildCreateRequest("p", "b", "DAL", "", false, false, false, false) - if err != nil { - t.Fatalf("buildCreateRequest returned error: %v", err) - } - attr := req.Data.Attributes - if attr.StorageClass != nil { - t.Errorf("StorageClass = %v, want nil by default", attr.StorageClass) - } - if attr.Versioning != nil || attr.Locking != nil { - t.Errorf("optional toggles should be nil by default, got versioning=%v locking=%v", attr.Versioning, attr.Locking) - } -} - -func TestBuildCreateRequestValidation(t *testing.T) { - cases := []struct { - name string - project, bucket, region, sc string - }{ - {"missing project", "", "b", "DAL", ""}, - {"missing name", "p", "", "DAL", ""}, - {"missing region", "p", "b", "", ""}, - {"invalid storage-class", "p", "b", "DAL", "turbo"}, - } - for _, c := range cases { - if _, err := buildCreateRequest(c.project, c.bucket, c.region, c.sc, false, false, false, false); err == nil { - t.Errorf("%s: expected error, got nil", c.name) - } - } -} - -func TestCreateCmdFlags(t *testing.T) { - cmd := NewCreateCmd() - if err := cmd.Flags().Parse([]string{ - "--project", "p1", - "--name", "b1", - "--region", "SAO2", - "--storage-class", "standard", - "--versioning", - }); err != nil { - t.Fatalf("flag parse error: %v", err) - } - if !cmd.Flags().Changed("versioning") { - t.Error("expected --versioning to be marked changed") - } - if cmd.Flags().Changed("locking") { - t.Error("expected --locking to be unset") - } -} diff --git a/cmd/storage_objects/delete.go b/cmd/storage_objects/delete.go deleted file mode 100644 index 557b1c7..0000000 --- a/cmd/storage_objects/delete.go +++ /dev/null @@ -1,62 +0,0 @@ -package storage_objects - -import ( - "context" - "fmt" - "net/http" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" - "github.com/latitudesh/lsh/cmd/lsh" - "github.com/latitudesh/lsh/internal/utils" - cobra "github.com/spf13/cobra" -) - -func NewDeleteCmd() *cobra.Command { - op := DeleteBucketOperation{} - cmd := &cobra.Command{ - Long: "Delete an object storage bucket by ID.\n", - RunE: op.run, - Short: "Delete a bucket", - Example: ` lsh storage-objects delete bucket_xxxxxxxx`, - Use: "delete ", - Aliases: []string{"rm"}, - Args: cobra.ExactArgs(1), - } - - return cmd -} - -type DeleteBucketOperation struct{} - -func (o *DeleteBucketOperation) run(cmd *cobra.Command, args []string) error { - id := args[0] - - if lsh.DryRun { - lsh.LogDebugf("dry-run flag specified. Skip sending request.") - return nil - } - - client := lsh.NewClient() - ctx := context.Background() - - resp, err := client.ObjectStorage.DeleteStorageBuckets(ctx, id, operations.WithRetries(lsh.RetryConfig())) - if err != nil { - utils.PrintError(err) - return err - } - - if !lsh.Debug { - // The API answers deletes with 200 or 204 depending on the path. - status := 0 - if resp.HTTPMeta.Response != nil { - status = resp.HTTPMeta.Response.StatusCode - } - if status == http.StatusOK || status == http.StatusNoContent { - fmt.Printf("\nBucket deleted successfully!\n") - } else { - fmt.Printf("Warning: Unexpected status code: %d\n", status) - } - } - - return nil -} diff --git a/cmd/storage_objects/get.go b/cmd/storage_objects/get.go deleted file mode 100644 index a7780b9..0000000 --- a/cmd/storage_objects/get.go +++ /dev/null @@ -1,53 +0,0 @@ -package storage_objects - -import ( - "context" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" - "github.com/latitudesh/lsh/cmd/lsh" - "github.com/latitudesh/lsh/internal/utils" - cobra "github.com/spf13/cobra" -) - -func NewGetCmd() *cobra.Command { - op := GetBucketOperation{} - cmd := &cobra.Command{ - Long: "Retrieve a single object storage bucket by ID.\n", - RunE: op.run, - Short: "Get a bucket", - Example: ` lsh storage-objects get bucket_xxxxxxxx`, - Use: "get ", - Args: cobra.ExactArgs(1), - } - - return cmd -} - -type GetBucketOperation struct{} - -func (o *GetBucketOperation) run(cmd *cobra.Command, args []string) error { - id := args[0] - - if lsh.DryRun { - lsh.LogDebugf("dry-run flag specified. Skip sending request.") - return nil - } - - client := lsh.NewClient() - ctx := context.Background() - - response, err := client.ObjectStorage.GetStorageBucket(ctx, id, operations.WithRetries(lsh.RetryConfig())) - if err != nil { - utils.PrintError(err) - return err - } - - if response.Object != nil && response.Object.Data != nil { - if !lsh.Debug { - bucket := Bucket{ObjectStorageData: *response.Object.Data} - utils.Render(bucket.GetData()) - } - } - - return nil -} diff --git a/cmd/storage_objects/list.go b/cmd/storage_objects/list.go deleted file mode 100644 index ee5d5be..0000000 --- a/cmd/storage_objects/list.go +++ /dev/null @@ -1,66 +0,0 @@ -package storage_objects - -import ( - "context" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" - "github.com/latitudesh/lsh/cli" - "github.com/latitudesh/lsh/cmd/lsh" - "github.com/latitudesh/lsh/internal/utils" - cobra "github.com/spf13/cobra" -) - -func NewListCmd() *cobra.Command { - op := ListBucketsOperation{} - cmd := &cobra.Command{ - Long: "List object storage buckets in the project.\n", - RunE: op.run, - Short: "List buckets", - Example: ` lsh storage-objects list --project my-project - lsh storage-objects list`, - Use: "list", - Aliases: []string{"ls"}, - Annotations: map[string]string{cli.ProjectOptionalAnnotation: "true"}, - } - - cmd.Flags().String("project", "", "Project ID or slug to filter by") - - return cmd -} - -type ListBucketsOperation struct{} - -func (o *ListBucketsOperation) run(cmd *cobra.Command, args []string) error { - var filterProject *string - if cmd.Flags().Changed("project") { - value, _ := cmd.Flags().GetString("project") - filterProject = &value - } - - if lsh.DryRun { - lsh.LogDebugf("dry-run flag specified. Skip sending request.") - return nil - } - - client := lsh.NewClient() - ctx := context.Background() - - response, err := client.ObjectStorage.GetStorageBuckets(ctx, filterProject, operations.WithRetries(lsh.RetryConfig())) - if err != nil { - utils.PrintError(err) - return err - } - - buckets := Buckets{} - if response.ObjectStorages != nil { - for i := range response.ObjectStorages.Data { - buckets.Data = append(buckets.Data, &Bucket{ObjectStorageData: response.ObjectStorages.Data[i]}) - } - } - - if !lsh.Debug { - utils.Render(buckets.GetData()) - } - - return nil -} diff --git a/cmd/storage_objects/render_test.go b/cmd/storage_objects/render_test.go deleted file mode 100644 index 4f1b539..0000000 --- a/cmd/storage_objects/render_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package storage_objects - -import ( - "encoding/json" - "testing" - - "github.com/latitudesh/latitudesh-go-sdk/models/components" -) - -const bucketsFixture = `{ - "data": [ - { - "id": "bucket_abc123", - "type": "object_storages", - "attributes": { - "name": "my-bucket", - "bucket_name": "my-bucket-xyz", - "storage_class": "high_performance", - "endpoint": "https://s3.example.com", - "created_at": "2026-06-01T12:00:00Z", - "region": {"id": "SAO2", "city": "Sao Paulo", "country": "Brazil"} - } - } - ] -}` - -// TestBucketsPayloadDecoding pins the SDK's typed list envelope to the API -// shape and asserts the rendered row for the first bucket. -func TestBucketsPayloadDecoding(t *testing.T) { - var payload components.ObjectStorages - if err := json.Unmarshal([]byte(bucketsFixture), &payload); err != nil { - t.Fatalf("could not unmarshal buckets fixture: %v", err) - } - if len(payload.Data) != 1 { - t.Fatalf("expected 1 bucket, got %d", len(payload.Data)) - } - - bucket := Bucket{ObjectStorageData: payload.Data[0]} - row := bucket.TableRow() - - expectations := map[string]string{ - "id": "bucket_abc123", - "name": "my-bucket", - "bucket_name": "my-bucket-xyz", - "region": "SAO2", - "storage_class": "high_performance", - "endpoint": "https://s3.example.com", - "created_at": "2026-06-01T12:00:00Z", - } - for key, want := range expectations { - if got := row[key].Value; got != want { - t.Errorf("row[%q] = %q, want %q", key, got, want) - } - } - // Live gets return region without an id ({city, country}); the cell must - // fall back to the city so the column is not blank. - city := "Miami" - noID := Bucket{ObjectStorageData: components.ObjectStorageData{ - Attributes: &components.ObjectStorageDataAttributes{ - Region: &components.ObjectStorageDataRegion{City: &city}, - }, - }} - if got := noID.TableRow()["region"].Value; got != "Miami" { - t.Errorf("region fallback = %q, want Miami", got) - } -} - -func TestBucketTableRowEmpty(t *testing.T) { - bucket := Bucket{ObjectStorageData: components.ObjectStorageData{}} - row := bucket.TableRow() - if row["name"].Value != "" { - t.Errorf("expected blank cells for empty bucket, got %+v", row) - } -} - -func TestGetAndDeleteArgs(t *testing.T) { - if err := NewGetCmd().Args(NewGetCmd(), []string{}); err == nil { - t.Error("get: expected error with no args") - } - if err := NewGetCmd().Args(NewGetCmd(), []string{"id1"}); err != nil { - t.Errorf("get: unexpected error with one arg: %v", err) - } - if err := NewDeleteCmd().Args(NewDeleteCmd(), []string{"a", "b"}); err == nil { - t.Error("delete: expected error with two args") - } -} diff --git a/cmd/storage_objects/storage_objects.go b/cmd/storage_objects/storage_objects.go deleted file mode 100644 index dd3554b..0000000 --- a/cmd/storage_objects/storage_objects.go +++ /dev/null @@ -1,96 +0,0 @@ -package storage_objects - -import ( - "github.com/latitudesh/latitudesh-go-sdk/models/components" - "github.com/latitudesh/lsh/internal/output/table" - "github.com/latitudesh/lsh/internal/renderer" -) - -// Buckets is the renderable collection of object storage buckets. -type Buckets struct { - Data []*Bucket -} - -func (m *Buckets) GetData() []renderer.ResponseData { - var data []renderer.ResponseData - for _, v := range m.Data { - data = append(data, v) - } - return data -} - -// Bucket wraps the SDK ObjectStorageData so it can be rendered by the shared -// renderer (table / -o json,yaml / --query). -type Bucket struct { - components.ObjectStorageData -} - -func (m *Bucket) GetData() []renderer.ResponseData { - return []renderer.ResponseData{m} -} - -func (m *Bucket) TableRow() table.Row { - b := m.ObjectStorageData - - getStr := func(s *string) string { - if s != nil { - return *s - } - return "" - } - - var name, bucketName, region, storageClass, endpoint, createdAt string - if attr := b.Attributes; attr != nil { - name = getStr(attr.Name) - bucketName = getStr(attr.BucketName) - endpoint = getStr(attr.Endpoint) - if attr.StorageClass != nil { - storageClass = string(*attr.StorageClass) - } - // The API sends region as {city, country} on get (id is not populated - // and list omits the attribute entirely); prefer the most specific field. - if attr.Region != nil { - region = getStr(attr.Region.ID) - if region == "" { - region = getStr(attr.Region.City) - } - if region == "" { - region = getStr(attr.Region.Country) - } - } - if attr.CreatedAt != nil { - createdAt = attr.CreatedAt.Format("2006-01-02T15:04:05Z07:00") - } - } - - return table.Row{ - "id": table.Cell{ - Label: "ID", - Value: table.String(getStr(b.ID)), - }, - "name": table.Cell{ - Label: "Name", - Value: table.String(name), - }, - "bucket_name": table.Cell{ - Label: "Bucket Name", - Value: table.String(bucketName), - }, - "region": table.Cell{ - Label: "Region", - Value: table.String(region), - }, - "storage_class": table.Cell{ - Label: "Storage Class", - Value: table.String(storageClass), - }, - "endpoint": table.Cell{ - Label: "Endpoint", - Value: table.String(endpoint), - }, - "created_at": table.Cell{ - Label: "Created At", - Value: table.String(createdAt), - }, - } -} diff --git a/cmd/tags/create.go b/cmd/tags/create.go index bdf5778..3b00fb8 100644 --- a/cmd/tags/create.go +++ b/cmd/tags/create.go @@ -88,9 +88,9 @@ func (o *CreateTagOperation) run(cmd *cobra.Command, args []string) error { Data: &operations.CreateTagTagsData{ Type: &createTagType, Attributes: &operations.CreateTagTagsAttributes{ - Name: &name, + Name: name, Description: &description, - Color: &color, + Color: color, }, }, } diff --git a/cmd/virtualmachines/get.go b/cmd/virtualmachines/get.go index 4b17c48..9143542 100644 --- a/cmd/virtualmachines/get.go +++ b/cmd/virtualmachines/get.go @@ -36,7 +36,7 @@ func (o *GetVirtualMachineOperation) run(cmd *cobra.Command, args []string) erro client := lsh.NewClient() ctx := context.Background() - response, err := client.VirtualMachines.Get(ctx, id, operations.WithRetries(lsh.RetryConfig())) + response, err := client.VirtualMachines.Get(ctx, id, nil, operations.WithRetries(lsh.RetryConfig())) if err != nil { utils.PrintError(err) return err diff --git a/cmd/virtualmachines/list.go b/cmd/virtualmachines/list.go index 11f0f6b..59ead7d 100644 --- a/cmd/virtualmachines/list.go +++ b/cmd/virtualmachines/list.go @@ -53,7 +53,7 @@ func (o *ListVirtualMachineOperation) run(cmd *cobra.Command, args []string) err client := lsh.NewClient() ctx := context.Background() - response, err := client.VirtualMachines.List(ctx, filterProject, filterTags, nil, operations.WithRetries(lsh.RetryConfig())) + response, err := client.VirtualMachines.List(ctx, operations.IndexVirtualMachineRequest{FilterProject: filterProject, FilterTags: filterTags}, operations.WithRetries(lsh.RetryConfig())) if err != nil { utils.PrintError(err) return err diff --git a/cmd/virtualmachines/wait.go b/cmd/virtualmachines/wait.go index f13c66a..f55155e 100644 --- a/cmd/virtualmachines/wait.go +++ b/cmd/virtualmachines/wait.go @@ -6,7 +6,6 @@ import ( "fmt" "os" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" "github.com/latitudesh/lsh/cmd/lsh" "github.com/latitudesh/lsh/internal/utils" @@ -37,7 +36,7 @@ func waitForVirtualMachine(cmd *cobra.Command, vmID string) error { fmt.Fprintf(os.Stderr, "Waiting for virtual machine %s to finish provisioning… (Ctrl+C to stop)\n", vmID) - want := []components.VirtualMachineAttributesStatus{components.VirtualMachineAttributesStatusRunning} + want := []wait.VirtualMachineStatus{wait.VirtualMachineStatusRunning} status, err := wait.ForVirtualMachineState(ctx, client, vmID, want, o, operations.WithRetries(lsh.RetryConfig())) switch { case errors.Is(err, wait.ErrCanceled): @@ -54,7 +53,7 @@ func waitForVirtualMachine(cmd *cobra.Command, vmID string) error { // real, final state. The wait itself already succeeded, so a failed // re-fetch only degrades the display — surface it on stderr. if !lsh.Debug { - resp, err := client.VirtualMachines.Get(ctx, vmID, operations.WithRetries(lsh.RetryConfig())) + resp, err := client.VirtualMachines.Get(ctx, vmID, nil, operations.WithRetries(lsh.RetryConfig())) if err == nil && resp.VirtualMachine != nil && resp.VirtualMachine.Data != nil { vm := VirtualMachine{VirtualMachineAttributes: *resp.VirtualMachine.Data} utils.RenderStatic(vm.GetData()) diff --git a/go.mod b/go.mod index a36ae87..4fbddb1 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,9 @@ require ( github.com/go-openapi/swag v0.22.7 github.com/go-openapi/validate v0.22.6 github.com/jmespath/go-jmespath v0.4.0 - github.com/latitudesh/latitudesh-go-sdk v1.16.5 + github.com/latitudesh/latitudesh-go-sdk v1.19.20 github.com/manifoldco/promptui v0.9.0 + github.com/minio/minio-go/v7 v7.1.0 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pb33f/libopenapi v0.15.14 @@ -22,8 +23,8 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 - golang.org/x/term v0.37.0 - golang.org/x/text v0.14.0 + golang.org/x/term v0.45.0 + golang.org/x/text v0.41.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -33,14 +34,18 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/ansi v0.10.1 // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect - github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.22.0 // indirect @@ -48,43 +53,56 @@ require ( github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/loads v0.21.5 // indirect github.com/go-openapi/spec v0.20.13 // indirect - github.com/google/uuid v1.5.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.3.1 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sahilm/fuzzy v0.1.1 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spyzhov/ajson v0.8.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.mongodb.org/mongo-driver v1.13.1 // indirect go.opentelemetry.io/otel v1.17.0 // indirect go.opentelemetry.io/otel/metric v1.17.0 // indirect go.opentelemetry.io/otel/trace v1.17.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/sys v0.38.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect ) diff --git a/go.sum b/go.sum index 2239d02..1b1850f 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPn github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= @@ -24,14 +26,16 @@ github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0G github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= -github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= -github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/dave/jennifer v1.7.0 h1:uRbSBH9UTS64yXbh4FrMHfgfY762RD+C7bUPKODpSJE= github.com/dave/jennifer v1.7.0/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= @@ -42,6 +46,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -50,6 +56,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -91,11 +99,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -109,6 +117,13 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -118,10 +133,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/latitudesh/latitudesh-go-sdk v1.16.5 h1:lPaGS8Xz1DNQPOj2bHm6htx0j9S3G6iGRw/WNe1ckjQ= -github.com/latitudesh/latitudesh-go-sdk v1.16.5/go.mod h1:Uso3PR7vjaRxfpAzlDt6LUYLvBdiL4ZZBrYmKm64sp8= -github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= -github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/latitudesh/latitudesh-go-sdk v1.19.20 h1:hxEGwDu2l4Sb6uGELxz8nSbhDrrl/aMowYTDlHatM9A= +github.com/latitudesh/latitudesh-go-sdk v1.19.20/go.mod h1:ckEY2Redz9dr7OVvdLA6uUMvP57WAX2Dt5RP2sQdSo8= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -133,8 +148,14 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= -github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.1.0 h1:QEt5IStDpxgGjEdtOgpiZ5QhmSl3ax7qy61vi2SwHO8= +github.com/minio/minio-go/v7 v7.1.0/go.mod h1:Dm7WS1AgLmBa0NcQD6SeJnJf+K/EUW3GR7Ks6olB3OA= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -169,18 +190,21 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pb33f/libopenapi v0.15.14 h1:A0fn45jbthDyFGXfu5bYIZVsWyPI6hJYm3wG143MT8o= github.com/pb33f/libopenapi v0.15.14/go.mod h1:PEXNwvtT4KNdjrwudp5OYnD1ryqK6uJ68aMNyWvoMuc= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= +github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= @@ -192,8 +216,8 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= @@ -207,16 +231,20 @@ github.com/spyzhov/ajson v0.8.0/go.mod h1:63V+CGM6f1Bu/p4nLIN8885ojBdt88TbLoSFzy github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= @@ -229,6 +257,10 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM= @@ -239,15 +271,17 @@ go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0 go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ= go.opentelemetry.io/otel/trace v1.17.0 h1:/SWhSRHmDPOImIAetP1QAeMnZYiQXrTy4fMMYOdSKWQ= go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -262,14 +296,14 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -290,20 +324,20 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -325,8 +359,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/config/objectstorage.go b/internal/config/objectstorage.go new file mode 100644 index 0000000..e10fb1e --- /dev/null +++ b/internal/config/objectstorage.go @@ -0,0 +1,158 @@ +package config + +import ( + "sort" + "time" +) + +// Access-key scopes as reported by the API. +const ( + ScopeFullAccess = "fullaccess" + ScopeLimitedAccess = "limited_access" + // ScopeUnknown marks keys imported without being matched against the + // API. They are only used when selected explicitly with --access-key. + ScopeUnknown = "unknown" +) + +// Per-bucket permissions of a limited_access key. +const ( + PermissionRW = "rw" + PermissionReadOnly = "readonly" +) + +// Origins of a stored key (informational). +const ( + KeySourceConfigure = "configure" + KeySourceCreate = "access-key-create" + KeySourceMakeBkt = "mb" + KeySourceImport = "import" +) + +// ObjectStorageConfig is the per-profile store of S3 access keys. +type ObjectStorageConfig struct { + // DefaultKey breaks ties when several saved keys cover a bucket equally. + DefaultKey string `json:"default_key,omitempty"` + // Keys is indexed by the logical key name the user passes to --access-key. + Keys map[string]StoredAccessKey `json:"keys,omitempty"` +} + +// StoredAccessKey is one S3 access key saved locally. The scope metadata is +// what lets the CLI pick the least-privileged key covering a bucket without +// asking the API again. +type StoredAccessKey struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + // StorageClass is standard (Wasabi) or high_performance (VAST). + StorageClass string `json:"storage_class"` + // Site is the Latitude site slug (e.g. TYO4). Set for high_performance + // keys, which are bound to one VAST cluster; empty for standard keys, + // which cover every site of the project. + Site string `json:"site,omitempty"` + ProjectID string `json:"project_id,omitempty"` + // Scope is fullaccess, limited_access or unknown. + Scope string `json:"scope"` + // Buckets maps bkt_ IDs to rw|readonly for limited_access keys. + Buckets map[string]string `json:"buckets,omitempty"` + // Username is the backend identity the API needs on DELETE. + Username string `json:"username,omitempty"` + Source string `json:"source,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// String renders the key without its secret so accidental %v/%s formatting +// never leaks it into logs or errors. +func (k StoredAccessKey) String() string { + return "access_key_id=" + k.AccessKeyID + " secret=[redacted]" +} + +// GoString mirrors String for %#v. +func (k StoredAccessKey) GoString() string { return k.String() } + +// Covers reports whether the key grants access to bucketID with the +// requested permission ("rw" or "readonly"); a fullaccess key covers every +// bucket, a limited key only the listed ones. +func (k StoredAccessKey) Covers(bucketID string, needWrite bool) bool { + switch k.Scope { + case ScopeFullAccess: + return true + case ScopeLimitedAccess: + perm, ok := k.Buckets[bucketID] + if !ok { + return false + } + return !needWrite || perm == PermissionRW + } + return false +} + +// Permission returns rw, readonly or "" for bucketID. +func (k StoredAccessKey) Permission(bucketID string) string { + if k.Scope == ScopeFullAccess { + return PermissionRW + } + return k.Buckets[bucketID] +} + +// ObjectStorageKeys returns the saved keys of the profile (never nil). +func (p Profile) ObjectStorageKeys() map[string]StoredAccessKey { + if p.ObjectStorage == nil || p.ObjectStorage.Keys == nil { + return map[string]StoredAccessKey{} + } + return p.ObjectStorage.Keys +} + +// SortedObjectStorageKeyNames returns the saved key names alphabetically. +func (p Profile) SortedObjectStorageKeyNames() []string { + keys := p.ObjectStorageKeys() + names := make([]string, 0, len(keys)) + for n := range keys { + names = append(names, n) + } + sort.Strings(names) + return names +} + +// SetObjectStorageKey inserts or replaces a saved key. +func (p *Profile) SetObjectStorageKey(name string, k StoredAccessKey) { + if p.ObjectStorage == nil { + p.ObjectStorage = &ObjectStorageConfig{} + } + if p.ObjectStorage.Keys == nil { + p.ObjectStorage.Keys = map[string]StoredAccessKey{} + } + p.ObjectStorage.Keys[name] = k +} + +// RemoveObjectStorageKey deletes a saved key by name, clearing DefaultKey if +// it pointed at it. Returns false when the name was not stored. +func (p *Profile) RemoveObjectStorageKey(name string) bool { + if p.ObjectStorage == nil || p.ObjectStorage.Keys == nil { + return false + } + if _, ok := p.ObjectStorage.Keys[name]; !ok { + return false + } + delete(p.ObjectStorage.Keys, name) + if p.ObjectStorage.DefaultKey == name { + p.ObjectStorage.DefaultKey = "" + } + return true +} + +// FindObjectStorageKeyByID looks a saved key up by its access key ID. +func (p Profile) FindObjectStorageKeyByID(accessKeyID string) (string, StoredAccessKey, bool) { + for name, k := range p.ObjectStorageKeys() { + if k.AccessKeyID == accessKeyID { + return name, k, true + } + } + return "", StoredAccessKey{}, false +} + +// DefaultObjectStorageKey returns the configured tie-breaker key name. +func (p Profile) DefaultObjectStorageKey() string { + if p.ObjectStorage == nil { + return "" + } + return p.ObjectStorage.DefaultKey +} diff --git a/internal/config/profile.go b/internal/config/profile.go index 477d18b..f1a4ac2 100644 --- a/internal/config/profile.go +++ b/internal/config/profile.go @@ -13,6 +13,10 @@ type Profile struct { Email string `json:"email,omitempty"` Source string `json:"source,omitempty"` APIVersion string `json:"api_version,omitempty"` + + // ObjectStorage holds the S3 access keys saved for `lsh s3`. Optional: + // profiles written before this field existed load unchanged. + ObjectStorage *ObjectStorageConfig `json:"object_storage,omitempty"` } // SourceBrowser is set on profiles created via the browser-assisted diff --git a/internal/exitcode/exitcode.go b/internal/exitcode/exitcode.go new file mode 100644 index 0000000..5995b33 --- /dev/null +++ b/internal/exitcode/exitcode.go @@ -0,0 +1,89 @@ +// Package exitcode defines the process exit codes used by the newer command +// groups (starting with `lsh s3`) and the error type that carries them. +// +// Historically every lsh failure exited with 1. Object storage commands are +// used heavily in scripts and CI, where distinguishing "not found" from +// "no credentials" from "refused for safety" matters, so they attach an +// explicit code to their errors. main() maps the error to os.Exit through +// Of(); commands that do not opt in keep exiting with 1. +package exitcode + +import ( + "errors" + "fmt" +) + +const ( + // OK is a successful run (including empty listings). + OK = 0 + // Generic is an unexpected error, or one or more transfers failed. + Generic = 1 + // Usage is invalid usage: bad URI/flag, ambiguous bucket, --recursive + // without --all, and other argument errors. + Usage = 2 + // NotFound covers a missing bucket, object, access key or lifecycle rule. + NotFound = 3 + // Credentials means no usable S3 credential was found, or the backend + // rejected the credential (InvalidAccessKeyId, SignatureDoesNotMatch). + Credentials = 4 + // Permission is an authenticated request the backend refused (403). + Permission = 5 + // Partial means the operation completed for some objects and failed for + // others (recursive delete, bucket emptying). + Partial = 6 + // Refused is a safety refusal: non-empty bucket without --force, + // --max-delete exceeded, object lock, prompt declined, or a confirmation + // required in a non-interactive session without --yes. + Refused = 7 + // Interrupted is returned after a SIGINT once cleanup finished. + Interrupted = 130 +) + +// OptInAnnotation marks a cobra command whose subtree uses these exit codes. +// Command groups that predate them keep exiting 1 for every failure, so shared +// pre-run validation has to know which contract applies. Commands set it on +// themselves (see cmd/s3.Finalize) and callers walk up the parents. +const OptInAnnotation = "lsh.exit_codes" + +// Error pairs an error with the exit code the process should terminate with. +type Error struct { + Code int + Err error +} + +func (e *Error) Error() string { + if e.Err == nil { + return fmt.Sprintf("exit %d", e.Code) + } + return e.Err.Error() +} + +// Unwrap exposes the wrapped error to errors.Is/As. +func (e *Error) Unwrap() error { return e.Err } + +// New wraps err with an exit code. A nil err yields nil. +func New(code int, err error) error { + if err == nil { + return nil + } + return &Error{Code: code, Err: err} +} + +// Errorf formats an error and attaches an exit code. +func Errorf(code int, format string, a ...interface{}) error { + return &Error{Code: code, Err: fmt.Errorf(format, a...)} +} + +// Of returns the exit code carried by err (the outermost *Error in the chain +// wins, as errors.As walks from the outside in), Generic when err carries +// none, and OK for nil. +func Of(err error) int { + if err == nil { + return OK + } + var e *Error + if errors.As(err, &e) { + return e.Code + } + return Generic +} diff --git a/internal/objectstorage/apiclient.go b/internal/objectstorage/apiclient.go new file mode 100644 index 0000000..d5e3db5 --- /dev/null +++ b/internal/objectstorage/apiclient.go @@ -0,0 +1,93 @@ +package objectstorage + +import ( + "bytes" + "io" + "net/http" + "regexp" + "strings" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/spf13/viper" +) + +// NewAPIClient returns the Latitude SDK client used by the object storage +// commands. It is the regular authenticated client with one addition: a +// transport that normalizes known API/SDK mismatches in /storage responses +// before the generated models decode them. +// +// Today the API serializes `retention_period` as "" (or a numeric string) for +// buckets without object lock, while the v1.19.20 model declares it as +// *int64, which makes every bucket list/get fail with "cannot unmarshal string +// into Go value of type int64". Until the API returns null/number, the +// transport rewrites the field. Everything else passes through untouched. +func NewAPIClient() *sdk.Latitudesh { + token := viper.GetString("Authorization") + opts := []sdk.SDKOption{ + sdk.WithSecurity(token), + sdk.WithClient(&normalizingHTTPClient{inner: http.DefaultClient}), + } + // Honour the global --hostname/--scheme overrides (dev/staging, tests) + // the same way the legacy client does; the SDK default is production. + if host := viper.GetString("hostname"); host != "" && host != "api.latitude.sh" { + scheme := viper.GetString("scheme") + if scheme == "" { + scheme = "https" + } + opts = append(opts, sdk.WithServerURL(scheme+"://"+host)) + } + return sdk.New(opts...) +} + +type normalizingHTTPClient struct { + inner *http.Client +} + +// reRetentionString matches retention_period serialized as any JSON string. +var reRetentionString = regexp.MustCompile(`"retention_period"\s*:\s*"((?:[^"\\]|\\.)*)"`) + +// Do executes the request and rewrites storage payloads. +func (c *normalizingHTTPClient) Do(req *http.Request) (*http.Response, error) { + resp, err := c.inner.Do(req) + if err != nil || resp == nil || resp.Body == nil { + return resp, err + } + if !strings.Contains(req.URL.Path, "/storage/") || !strings.Contains(resp.Header.Get("Content-Type"), "json") { + return resp, nil + } + body, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + if readErr != nil { + return nil, readErr + } + fixed := NormalizeStorageJSON(body) + resp.Body = io.NopCloser(bytes.NewReader(fixed)) + resp.ContentLength = int64(len(fixed)) + resp.Header.Del("Content-Length") + return resp, nil +} + +// NormalizeStorageJSON applies the known field fixes to a /storage payload. +func NormalizeStorageJSON(body []byte) []byte { + if !bytes.Contains(body, []byte(`"retention_period"`)) { + return body + } + return reRetentionString.ReplaceAllFunc(body, func(m []byte) []byte { + sub := reRetentionString.FindSubmatch(m) + if len(sub) == 2 { + if v := strings.TrimSpace(string(sub[1])); v != "" && isDigits(v) { + return []byte(`"retention_period":` + v) + } + } + return []byte(`"retention_period":null`) + }) +} + +func isDigits(s string) bool { + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return s != "" +} diff --git a/internal/objectstorage/apiclient_test.go b/internal/objectstorage/apiclient_test.go new file mode 100644 index 0000000..e3dea8f --- /dev/null +++ b/internal/objectstorage/apiclient_test.go @@ -0,0 +1,21 @@ +package objectstorage + +import "testing" + +func TestNormalizeStorageJSON(t *testing.T) { + cases := map[string]string{ + `{"retention_period":""}`: `{"retention_period":null}`, + `{"retention_period": ""}`: `{"retention_period":null}`, + `{"retention_period":"30"}`: `{"retention_period":30}`, + `{"retention_period":"n/a"}`: `{"retention_period":null}`, + `{"retention_period":30}`: `{"retention_period":30}`, + `{"retention_period":null}`: `{"retention_period":null}`, + `{"name":"x"}`: `{"name":"x"}`, + `{"a":{"retention_period":"7"},"b":{"retention_period":""}}`: `{"a":{"retention_period":7},"b":{"retention_period":null}}`, + } + for in, want := range cases { + if got := string(NormalizeStorageJSON([]byte(in))); got != want { + t.Errorf("NormalizeStorageJSON(%s) = %s, want %s", in, got, want) + } + } +} diff --git a/internal/objectstorage/client.go b/internal/objectstorage/client.go new file mode 100644 index 0000000..7e27397 --- /dev/null +++ b/internal/objectstorage/client.go @@ -0,0 +1,138 @@ +package objectstorage + +import ( + "context" + "io" + "net" + "net/http" + "os" + "os/signal" + "regexp" + "syscall" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/version" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// Addressing styles for the escape hatch flag. +const ( + AddressingPath = "path" + AddressingVirtual = "virtual" +) + +// ClientOptions tunes NewS3Client. +type ClientOptions struct { + // Debug enables HTTP tracing (redacted) to Trace (stderr when nil). + Debug bool + Trace io.Writer + // Addressing is path (default) or virtual. + Addressing string + // MaxRetries caps request retries (minio default 10 when zero). + MaxRetries int +} + +// NewS3Client builds the minio client for a bucket with the settings every +// S3-compatible backend needs: explicit endpoint and signing region (never +// discovered through GetBucketLocation), path-style addressing, no trailing +// checksums (Content-MD5 is used instead), sane timeouts without a total +// deadline so long uploads are not killed. +func NewS3Client(b *Bucket, c Credential, o ClientOptions) (*minio.Client, error) { + if err := b.Validate(); err != nil { + return nil, err + } + host, secure, err := EndpointHost(b.Endpoint) + if err != nil { + return nil, exitcode.New(exitcode.Usage, err) + } + lookup := minio.BucketLookupPath + if o.Addressing == AddressingVirtual { + lookup = minio.BucketLookupDNS + } + region := b.SigningRegion + if region == "" { + region = DefaultSigningRegion + } + client, err := minio.New(host, &minio.Options{ + Creds: credentials.NewStaticV4(c.AccessKeyID, c.Secret(), ""), + Secure: secure, + Region: region, + BucketLookup: lookup, + TrailingHeaders: false, + Transport: newTransport(), + MaxRetries: o.MaxRetries, + }) + if err != nil { + return nil, exitcode.Errorf(exitcode.Generic, "could not create S3 client for %s: %v", host, err) + } + client.SetAppInfo("lsh", version.Version) + if o.Debug { + w := o.Trace + if w == nil { + w = os.Stderr + } + client.TraceOn(&redactingWriter{w: w}) + } + return client, nil +} + +// newTransport mirrors the minio default transport but with explicit +// connection timeouts and no overall request deadline. +func newTransport() http.RoundTripper { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 15 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 64, + MaxIdleConnsPerHost: 16, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 15 * time.Second, + ResponseHeaderTimeout: 90 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ForceAttemptHTTP2: true, + DisableCompression: true, + } +} + +var redactPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?i)(Signature=)[0-9a-f]+`), + regexp.MustCompile(`(?i)(X-Amz-Signature=)[0-9a-f]+`), + regexp.MustCompile(`(?i)(X-Amz-Security-Token=)[^&\s]+`), + regexp.MustCompile(`(?i)(x-amz-security-token: )\S+`), + regexp.MustCompile(`(?i)("?secret_access_key"?\s*[:=]\s*"?)[^",\s]+`), + regexp.MustCompile(`(?i)("?secret_key"?\s*[:=]\s*"?)[^",\s]+`), +} + +// redactingWriter strips signatures and secrets from minio's HTTP trace. +type redactingWriter struct{ w io.Writer } + +func (r *redactingWriter) Write(p []byte) (int, error) { + out := p + for _, re := range redactPatterns { + out = re.ReplaceAll(out, []byte("${1}[redacted]")) + } + if _, err := r.w.Write(out); err != nil { + return 0, err + } + return len(p), nil +} + +// Redact applies the same redaction to an arbitrary string (error messages, +// debug lines). +func Redact(s string) string { + for _, re := range redactPatterns { + s = re.ReplaceAllString(s, "${1}[redacted]") + } + return s +} + +// SignalContext returns a context cancelled on SIGINT/SIGTERM so transfers can +// abort cleanly (multipart uploads are aborted by minio on error). The +// returned stop function releases the signal handler. +func SignalContext(parent context.Context) (context.Context, context.CancelFunc) { + return signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM) +} diff --git a/internal/objectstorage/client_test.go b/internal/objectstorage/client_test.go new file mode 100644 index 0000000..7304286 --- /dev/null +++ b/internal/objectstorage/client_test.go @@ -0,0 +1,138 @@ +package objectstorage + +import ( + "bytes" + "context" + "io" + "regexp" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/objectstorage/s3test" + "github.com/minio/minio-go/v7" +) + +// TestClientAgainstFakeS3 exercises the client settings that matter for +// S3-compatible backends: path-style addressing, the derived signing region, +// no checksum trailers on writes, and that the fake server round-trips +// list/put/get/delete. +func TestClientAgainstFakeS3(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + srv.AddObject("backups-7f3a", "2026/09/a.sql", []byte("aaa"), "application/sql") + srv.AddObject("backups-7f3a", "2026/09/b.sql", []byte("bbbb"), "application/sql") + srv.AddObject("backups-7f3a", "readme.txt", []byte("hi"), "text/plain") + + b := &Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a", Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: "eu-west-2"} + var trace bytes.Buffer + client, err := NewS3Client(b, NewCredential("AKIAEXAMPLE", "topsecret", "test"), ClientOptions{Debug: true, Trace: &trace, MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + // Non-recursive listing: one prefix + one object. + var keys []string + var prefixes []string + for info := range client.ListObjects(ctx, b.BucketName, minio.ListObjectsOptions{Recursive: false}) { + if info.Err != nil { + t.Fatal(info.Err) + } + if strings.HasSuffix(info.Key, "/") { + prefixes = append(prefixes, info.Key) + } else { + keys = append(keys, info.Key) + } + } + if len(prefixes) != 1 || prefixes[0] != "2026/" || len(keys) != 1 || keys[0] != "readme.txt" { + t.Fatalf("listing: prefixes=%v keys=%v", prefixes, keys) + } + + // Upload, download, delete. + if _, err := client.PutObject(ctx, b.BucketName, "new/file.bin", bytes.NewReader([]byte("payload")), 7, minio.PutObjectOptions{ContentType: "application/octet-stream"}); err != nil { + t.Fatal(err) + } + obj, err := client.GetObject(ctx, b.BucketName, "new/file.bin", minio.GetObjectOptions{}) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(obj) + if err != nil || string(data) != "payload" { + t.Fatalf("download = %q, %v", data, err) + } + if err := client.RemoveObject(ctx, b.BucketName, "new/file.bin", minio.RemoveObjectOptions{}); err != nil { + t.Fatal(err) + } + if srv.Object("backups-7f3a", "new/file.bin") != nil { + t.Fatal("object still present after delete") + } + // Deleting a missing key is idempotent (S3 semantics). + if err := client.RemoveObject(ctx, b.BucketName, "missing", minio.RemoveObjectOptions{}); err != nil { + t.Fatalf("delete of missing key must succeed: %v", err) + } + + reqs := srv.Requests() + if len(reqs) == 0 { + t.Fatal("no requests recorded") + } + for _, r := range reqs { + if !strings.HasPrefix(r.Path, "/backups-7f3a") { + t.Errorf("expected path-style request, got %s", r.Path) + } + if region := s3test.SigningRegionOf(r); region != "eu-west-2" { + t.Errorf("signing region = %q, want eu-west-2 (%s %s)", region, r.Method, r.Path) + } + } + if srv.HasChecksumHeaders() { + t.Error("writes must not carry x-amz-checksum-*/aws-chunked headers (Wasabi rejects them)") + } + if strings.Contains(trace.String(), "topsecret") { + t.Error("trace leaked the secret") + } + // minio already masks the SigV4 signature in its trace; our writer must + // keep it that way (no 64-hex signature anywhere in the output). + if regexp.MustCompile(`Signature=[0-9a-f]{64}`).MatchString(trace.String()) { + t.Error("trace did not redact the signature") + } +} + +func TestHumanizeMapsS3Errors(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bkt-test") + b := &Bucket{ID: "bkt_1", Name: "b", BucketName: "bkt-test", Endpoint: srv.URL(), SigningRegion: "us-east-1"} + cred := NewCredential("AK", "SK", "saved key \"x\"") + client, err := NewS3Client(b, cred, ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + _, err = client.StatObject(ctx, "bkt-test", "nope", minio.StatObjectOptions{}) + if h := Humanize(err, b, &cred); h == nil || !strings.Contains(h.Error(), "not found") { + t.Errorf("NoSuchKey → %v", h) + } + + srv.DenyAll = true + _, err = client.StatObject(ctx, "bkt-test", "x", minio.StatObjectOptions{}) + h := Humanize(err, b, &cred) + if h == nil || !strings.Contains(h.Error(), "access denied") { + t.Errorf("AccessDenied → %v", h) + } + srv.DenyAll = false + + // HEAD responses carry no body, so use a GET (listing) to exercise the + // XML error mapping. + srv.FailNext = &s3test.ErrorResponse{Code: "AuthorizationHeaderMalformed", Message: "wrong region", Region: "eu-west-2", Status: 400} + err = nil + for info := range client.ListObjects(ctx, "bkt-test", minio.ListObjectsOptions{}) { + if info.Err != nil { + err = info.Err + } + } + h = Humanize(err, b, &cred) + if h == nil || !strings.Contains(h.Error(), "eu-west-2") { + t.Errorf("AuthorizationHeaderMalformed → %v", h) + } +} diff --git a/internal/objectstorage/creds.go b/internal/objectstorage/creds.go new file mode 100644 index 0000000..8511652 --- /dev/null +++ b/internal/objectstorage/creds.go @@ -0,0 +1,330 @@ +package objectstorage + +import ( + "errors" + "fmt" + "os" + "sort" + "strings" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" +) + +// Environment variables understood by the S3 layer. +const ( + EnvAccessKeyID = "LSH_S3_ACCESS_KEY_ID" + EnvSecretAccessKey = "LSH_S3_SECRET_ACCESS_KEY" + // EnvUseAWSEnv opts in to reading AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY + // as a fallback. Off by default: those variables almost always point at + // the real AWS and would produce opaque 403s against Latitude. + EnvUseAWSEnv = "LSH_S3_USE_AWS_ENV" + EnvEndpointURL = "LSH_S3_ENDPOINT_URL" + EnvSigningRegion = "LSH_S3_SIGNING_REGION" +) + +// Credential is a resolved S3 credential plus where it came from. The secret +// is unexported so that formatting the value never prints it. +type Credential struct { + // Name is the saved key name; empty for environment credentials. + Name string + AccessKeyID string + secret string + // Source describes the origin for messages and --debug: + // "LSH_S3_ACCESS_KEY_ID (environment)", "saved key \"ci\" (profile acme)". + Source string + // FromEnv is true when the credential came from environment variables. + FromEnv bool + // Key carries the scope metadata of a saved key (zero for env). + Key config.StoredAccessKey + // Profile is the profile name the key was loaded from. + Profile string +} + +// NewCredential builds a credential from explicit values (tests, imports). +func NewCredential(id, secret, source string) Credential { + return Credential{AccessKeyID: id, secret: secret, Source: source} +} + +// Secret returns the secret access key. +func (c Credential) Secret() string { return c.secret } + +// String never includes the secret. +func (c Credential) String() string { + return fmt.Sprintf("access_key_id=%s source=%s secret=[redacted]", c.AccessKeyID, c.Source) +} + +// GoString mirrors String for %#v. +func (c Credential) GoString() string { return c.String() } + +// Describe renders the credential for user-facing messages, including its +// permission on bucketID when known. +func (c Credential) Describe(bucketID string) string { + if c.FromEnv || c.Name == "" { + return c.Source + } + perm := c.Key.Permission(bucketID) + switch { + case c.Key.Scope == config.ScopeFullAccess: + return fmt.Sprintf("saved key %q (fullaccess)", c.Name) + case perm != "": + return fmt.Sprintf("saved key %q (limited_access, %s on this bucket)", c.Name, perm) + default: + return fmt.Sprintf("saved key %q (%s)", c.Name, c.Key.Scope) + } +} + +// CredentialOptions controls ResolveCredential. +type CredentialOptions struct { + // AccessKeyName is the --access-key flag: force a saved key by name. + AccessKeyName string + // ProfileOverride is the --profile flag. + ProfileOverride string + // Write is true when the operation needs write permission on the bucket + // (upload, delete, presign PUT). + Write bool +} + +// EnvCredential reads LSH_S3_* (and, when opted in, AWS_*) from the +// environment. ok is false when nothing is set; err reports a half-set pair. +func EnvCredential() (Credential, bool, error) { + id, secret := os.Getenv(EnvAccessKeyID), os.Getenv(EnvSecretAccessKey) + switch { + case id != "" && secret != "": + return Credential{AccessKeyID: id, secret: secret, Source: EnvAccessKeyID + " (environment)", FromEnv: true}, true, nil + case id != "" && secret == "": + return Credential{}, false, exitcode.Errorf(exitcode.Credentials, "%s is set but %s is missing", EnvAccessKeyID, EnvSecretAccessKey) + case id == "" && secret != "": + return Credential{}, false, exitcode.Errorf(exitcode.Credentials, "%s is set but %s is missing", EnvSecretAccessKey, EnvAccessKeyID) + } + if os.Getenv(EnvUseAWSEnv) == "1" || strings.EqualFold(os.Getenv(EnvUseAWSEnv), "true") { + if os.Getenv("AWS_SESSION_TOKEN") != "" { + // STS credentials are never Latitude access keys. + return Credential{}, false, nil + } + aid, asecret := os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY") + if aid != "" && asecret != "" { + fmt.Fprintf(os.Stderr, "warning: using AWS_ACCESS_KEY_ID from the environment (%s=1)\n", EnvUseAWSEnv) + return Credential{AccessKeyID: aid, secret: asecret, Source: "AWS_ACCESS_KEY_ID (environment, opt-in fallback)", FromEnv: true}, true, nil + } + } + return Credential{}, false, nil +} + +// ActiveProfile loads the config file and resolves the active profile +// (--profile > LSH_PROFILE > default_profile). +func ActiveProfile(override string) (*config.File, string, config.Profile, error) { + f, err := config.Load() + if err != nil { + return nil, "", config.Profile{}, err + } + name, p, err := f.Resolve(override) + if err != nil { + if errors.Is(err, config.ErrProfileNotFound) { + if override != "" { + return f, name, p, exitcode.Errorf(exitcode.Usage, "profile %q not found — run 'lsh profile list'", override) + } + return f, "", config.Profile{}, exitcode.Errorf(exitcode.Credentials, "no active profile — run 'lsh login' (or use LSH_S3_ACCESS_KEY_ID/LSH_S3_SECRET_ACCESS_KEY)") + } + return f, name, p, err + } + return f, name, p, nil +} + +// ResolveCredential picks the credential for an operation on b: +// +// 1. LSH_S3_ACCESS_KEY_ID + LSH_S3_SECRET_ACCESS_KEY (environment) +// 2. --access-key (saved key, must cover the bucket) +// 3. the least-privileged saved key that covers the bucket +// +// In endpoint-override mode only the environment is consulted: saved keys are +// never sent to a host the API did not vouch for. +func ResolveCredential(b *Bucket, o CredentialOptions) (Credential, error) { + if c, ok, err := EnvCredential(); err != nil { + return Credential{}, err + } else if ok { + return c, nil + } + if b != nil && b.EndpointOverride { + return Credential{}, exitcode.Errorf(exitcode.Credentials, + "saved access keys are not sent to custom endpoints; set %s and %s in the environment", EnvAccessKeyID, EnvSecretAccessKey) + } + _, profileName, profile, err := ActiveProfile(o.ProfileOverride) + if err != nil { + return Credential{}, err + } + keys := profile.ObjectStorageKeys() + + if o.AccessKeyName != "" { + k, ok := keys[o.AccessKeyName] + if !ok { + return Credential{}, exitcode.Errorf(exitcode.NotFound, "no saved access key named %q in profile %s; run 'lsh s3 access-keys list --saved'", o.AccessKeyName, profileName) + } + c := fromStored(o.AccessKeyName, k, profileName) + if b != nil && !b.EndpointOverride && b.ID != "" { + if k.Scope != config.ScopeUnknown && !k.Covers(b.ID, false) { + return Credential{}, exitcode.Errorf(exitcode.Permission, "saved key %q does not cover bucket %s", o.AccessKeyName, b.Display()) + } + if o.Write && k.Scope == config.ScopeLimitedAccess && k.Permission(b.ID) != config.PermissionRW { + return Credential{}, exitcode.Errorf(exitcode.Permission, "saved key %q is readonly on bucket %s; pick a key with rw permission (lsh s3 access-keys list) or create one: lsh s3 access-keys create --bucket %s --save", o.AccessKeyName, b.Display(), b.Name) + } + } + return c, nil + } + + if b == nil { + return Credential{}, exitcode.Errorf(exitcode.Credentials, "no credential selected; pass --access-key or set %s/%s", EnvAccessKeyID, EnvSecretAccessKey) + } + name, k, ok := SelectKey(keys, profile.DefaultObjectStorageKey(), b, o.Write) + if !ok { + return Credential{}, NoCredentialError(b, profileName, o.Write, keys) + } + return fromStored(name, k, profileName), nil +} + +func fromStored(name string, k config.StoredAccessKey, profile string) Credential { + return Credential{ + Name: name, + AccessKeyID: k.AccessKeyID, + secret: k.SecretAccessKey, + Source: fmt.Sprintf("saved key %q (profile %s)", name, profile), + Key: k, + Profile: profile, + } +} + +// SelectKey applies the automatic selection rules over the saved keys: +// same storage class, compatible site and project, covering the bucket with +// the needed permission. Ranking: limited rw > limited readonly (reads only) +// > fullaccess; ties go to defaultKey, then to the newest key, then by name. +func SelectKey(keys map[string]config.StoredAccessKey, defaultKey string, b *Bucket, write bool) (string, config.StoredAccessKey, bool) { + type cand struct { + name string + key config.StoredAccessKey + rank int + } + var cands []cand + for name, k := range keys { + if !keyMatchesBucket(k, b) { + continue + } + if !k.Covers(b.ID, write) { + continue + } + rank := 2 + if k.Scope == config.ScopeLimitedAccess { + if k.Permission(b.ID) == config.PermissionRW { + rank = 0 + } else { + rank = 1 + } + } + cands = append(cands, cand{name, k, rank}) + } + if len(cands) == 0 { + return "", config.StoredAccessKey{}, false + } + sort.Slice(cands, func(i, j int) bool { + a, c := cands[i], cands[j] + if a.rank != c.rank { + return a.rank < c.rank + } + if (a.name == defaultKey) != (c.name == defaultKey) { + return a.name == defaultKey + } + if !a.key.CreatedAt.Equal(c.key.CreatedAt) { + return a.key.CreatedAt.After(c.key.CreatedAt) + } + return a.name < c.name + }) + return cands[0].name, cands[0].key, true +} + +// keyMatchesBucket checks class, site and project compatibility. Empty values +// on either side are treated as wildcards so keys saved before the site was +// known keep working. +func keyMatchesBucket(k config.StoredAccessKey, b *Bucket) bool { + if k.Scope == config.ScopeUnknown { + return false + } + if k.StorageClass != "" && b.StorageClass != "" && k.StorageClass != b.StorageClass { + return false + } + if k.Site != "" && b.Site != "" && !strings.EqualFold(k.Site, b.Site) { + return false + } + if k.ProjectID != "" && b.ProjectID != "" && k.ProjectID != b.ProjectID { + return false + } + return true +} + +// NoCredentialError explains how to obtain a credential for b. +func NoCredentialError(b *Bucket, profile string, write bool, keys map[string]config.StoredAccessKey) error { + what := "no saved S3 access key covers" + if write { + // Point out read-only keys that exist but cannot write. + for name, k := range keys { + if keyMatchesBucket(k, b) && k.Covers(b.ID, false) { + what = fmt.Sprintf("saved key %q is readonly on this bucket; no saved key can write to", name) + break + } + } + } + return exitcode.Errorf(exitcode.Credentials, + "%s bucket %s (%s, profile %s).\n create one: lsh s3 access-keys create --bucket %s --save\n or run: lsh s3 configure\n or export: %s and %s", + what, b.Display(), b.StorageClass, profile, b.Name, EnvAccessKeyID, EnvSecretAccessKey) +} + +// SaveKey stores k under name in the active profile and persists the file. +func SaveKey(profileOverride, name string, k config.StoredAccessKey) (string, error) { + f, profileName, p, err := ActiveProfile(profileOverride) + if err != nil { + return "", err + } + p.SetObjectStorageKey(name, k) + f.SetProfile(profileName, p) + if err := config.Save(f); err != nil { + return profileName, err + } + return profileName, nil +} + +// ForgetKey removes a saved key by name from the active profile. +func ForgetKey(profileOverride, name string) (string, bool, error) { + f, profileName, p, err := ActiveProfile(profileOverride) + if err != nil { + return "", false, err + } + removed := p.RemoveObjectStorageKey(name) + if removed { + f.SetProfile(profileName, p) + if err := config.Save(f); err != nil { + return profileName, false, err + } + } + return profileName, removed, nil +} + +// ForgetKeyByID removes every saved key with the given access key ID. +func ForgetKeyByID(profileOverride, accessKeyID string) (string, []string, error) { + f, profileName, p, err := ActiveProfile(profileOverride) + if err != nil { + return "", nil, err + } + var removed []string + for name, k := range p.ObjectStorageKeys() { + if k.AccessKeyID == accessKeyID { + p.RemoveObjectStorageKey(name) + removed = append(removed, name) + } + } + if len(removed) > 0 { + f.SetProfile(profileName, p) + if err := config.Save(f); err != nil { + return profileName, nil, err + } + } + sort.Strings(removed) + return profileName, removed, nil +} diff --git a/internal/objectstorage/creds_test.go b/internal/objectstorage/creds_test.go new file mode 100644 index 0000000..ad6b496 --- /dev/null +++ b/internal/objectstorage/creds_test.go @@ -0,0 +1,124 @@ +package objectstorage + +import ( + "os" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func TestEnvCredential(t *testing.T) { + t.Setenv(EnvAccessKeyID, "") + t.Setenv(EnvSecretAccessKey, "") + t.Setenv(EnvUseAWSEnv, "") + t.Setenv("AWS_ACCESS_KEY_ID", "AKIAREAL") + t.Setenv("AWS_SECRET_ACCESS_KEY", "realsecret") + t.Setenv("AWS_SESSION_TOKEN", "") + + if _, ok, err := EnvCredential(); ok || err != nil { + t.Fatalf("AWS_* must be ignored without opt-in: ok=%v err=%v", ok, err) + } + + t.Setenv(EnvUseAWSEnv, "1") + c, ok, err := EnvCredential() + if !ok || err != nil || c.AccessKeyID != "AKIAREAL" || c.Secret() != "realsecret" || !c.FromEnv { + t.Fatalf("opt-in fallback failed: %+v ok=%v err=%v", c, ok, err) + } + t.Setenv("AWS_SESSION_TOKEN", "sts") + if _, ok, _ := EnvCredential(); ok { + t.Fatal("STS credentials must never be used as Latitude keys") + } + + t.Setenv(EnvAccessKeyID, "LSHID") + if _, _, err := EnvCredential(); err == nil || exitcode.Of(err) != exitcode.Credentials { + t.Fatalf("half-set pair must fail with exit %d, got %v", exitcode.Credentials, err) + } + t.Setenv(EnvSecretAccessKey, "LSHSECRET") + c, ok, err = EnvCredential() + if !ok || err != nil || c.AccessKeyID != "LSHID" || c.Secret() != "LSHSECRET" { + t.Fatalf("LSH_S3_* must win: %+v", c) + } + if strings.Contains(c.String(), "LSHSECRET") { + t.Fatal("String() leaked the secret") + } +} + +func TestSelectKeyLeastPrivilege(t *testing.T) { + b := &Bucket{ID: "bkt_1", StorageClass: config.ScopeFullAccess, Site: "DAL", ProjectID: "proj_1"} + b.StorageClass = "standard" + now := time.Now() + keys := map[string]config.StoredAccessKey{ + "full-std": {AccessKeyID: "F", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now.Add(-time.Hour)}, + "ro-1": {AccessKeyID: "R", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": config.PermissionReadOnly}, CreatedAt: now}, + "rw-1": {AccessKeyID: "W", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": config.PermissionRW}, CreatedAt: now.Add(-2 * time.Hour)}, + "other": {AccessKeyID: "O", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_9": config.PermissionRW}, CreatedAt: now}, + "hp": {AccessKeyID: "H", StorageClass: "high_performance", Site: "DAL", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now}, + "unknown": {AccessKeyID: "U", StorageClass: "standard", Scope: config.ScopeUnknown, CreatedAt: now}, + } + + name, _, ok := SelectKey(keys, "", b, false) + if !ok || name != "rw-1" { + t.Fatalf("read: want rw-1 (limited rw ranks first), got %q ok=%v", name, ok) + } + name, _, ok = SelectKey(keys, "", b, true) + if !ok || name != "rw-1" { + t.Fatalf("write: want rw-1, got %q", name) + } + + delete(keys, "rw-1") + name, _, _ = SelectKey(keys, "", b, false) + if name != "ro-1" { + t.Fatalf("read without rw key: want ro-1 over fullaccess, got %q", name) + } + name, _, _ = SelectKey(keys, "", b, true) + if name != "full-std" { + t.Fatalf("write must skip readonly and use fullaccess, got %q", name) + } + + delete(keys, "full-std") + if _, _, ok := SelectKey(keys, "", b, true); ok { + t.Fatal("no writable key must yield no selection") + } + + // Fullaccess ties: default key wins, then newest. + keys = map[string]config.StoredAccessKey{ + "old": {AccessKeyID: "A", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now.Add(-time.Hour)}, + "new": {AccessKeyID: "B", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now}, + "dflt": {AccessKeyID: "C", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now.Add(-2 * time.Hour)}, + } + if name, _, _ := SelectKey(keys, "dflt", b, false); name != "dflt" { + t.Fatalf("default key must break ties, got %q", name) + } + if name, _, _ := SelectKey(keys, "", b, false); name != "new" { + t.Fatalf("newest key must break ties, got %q", name) + } + + // A high_performance bucket in another site is not covered by a DAL key. + hp := &Bucket{ID: "bkt_2", StorageClass: "high_performance", Site: "NYC", ProjectID: "proj_1"} + keys = map[string]config.StoredAccessKey{"hp-dal": {AccessKeyID: "H", StorageClass: "high_performance", Site: "DAL", ProjectID: "proj_1", Scope: config.ScopeFullAccess}} + if _, _, ok := SelectKey(keys, "", hp, false); ok { + t.Fatal("DAL key must not cover NYC high_performance bucket") + } + hp.Site = "dal" + if _, _, ok := SelectKey(keys, "", hp, false); !ok { + t.Fatal("site comparison must be case-insensitive") + } +} + +func TestResolveCredentialEndpointOverrideRequiresEnv(t *testing.T) { + t.Setenv(EnvAccessKeyID, "") + t.Setenv(EnvSecretAccessKey, "") + t.Setenv(EnvUseAWSEnv, "") + os.Unsetenv("AWS_ACCESS_KEY_ID") + b := &Bucket{BucketName: "raw", Endpoint: "https://s3.example.test", EndpointOverride: true} + _, err := ResolveCredential(b, CredentialOptions{}) + if err == nil || exitcode.Of(err) != exitcode.Credentials { + t.Fatalf("saved keys must not be used with --endpoint-url; got %v", err) + } + if !strings.Contains(err.Error(), EnvAccessKeyID) { + t.Errorf("error should tell the user which variables to set: %v", err) + } +} diff --git a/internal/objectstorage/errors.go b/internal/objectstorage/errors.go new file mode 100644 index 0000000..f85f3db --- /dev/null +++ b/internal/objectstorage/errors.go @@ -0,0 +1,327 @@ +package objectstorage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "strconv" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/tui" + "github.com/minio/minio-go/v7" +) + +// Humanize rewrites an S3 or API error into an actionable message with the +// right exit code. Errors that already carry an exit code pass through. +func Humanize(err error, b *Bucket, c *Credential) error { + if err == nil { + return nil + } + var already *exitcode.Error + if errors.As(err, &already) { + return err + } + if errors.Is(err, context.Canceled) { + return exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + if errors.Is(err, context.DeadlineExceeded) { + return exitcode.Errorf(exitcode.Generic, "timed out: %v", err) + } + + bucketName := "" + if b != nil { + bucketName = b.Display() + } + cred := "the credential" + if c != nil { + if b != nil { + cred = c.Describe(b.ID) + } else { + cred = c.Source + } + } + + if resp := minio.ToErrorResponse(err); resp.Code != "" || resp.StatusCode != 0 { + switch resp.Code { + case "NoSuchBucket": + endpoint := "" + backend := resp.BucketName + if b != nil { + endpoint = b.Endpoint + backend = b.BucketName + } + return exitcode.Errorf(exitcode.NotFound, "bucket %s (backend name %q) was not found at %s; it may still be provisioning — check 'lsh s3 stat %s'", bucketName, backend, endpoint, displayOrBackend(b)) + case "NoSuchKey", "NotFound": + key := resp.Key + if key == "" { + return exitcode.Errorf(exitcode.NotFound, "object not found in bucket %s", bucketName) + } + return exitcode.Errorf(exitcode.NotFound, "object %q not found in bucket %s", key, bucketName) + case "AccessDenied", "AllAccessDisabled": + hint := "" + if b != nil { + hint = fmt.Sprintf("; use a key with rw permission (lsh s3 access-keys list) or create one: lsh s3 access-keys create --bucket %s --save", displayOrBackend(b)) + } + return exitcode.Errorf(exitcode.Permission, "access denied to %s using %s%s", bucketName, cred, hint) + case "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidToken", "ExpiredToken": + hint := "" + if c != nil && !c.FromEnv && c.Name != "" { + hint = fmt.Sprintf("; if it was deleted or rotated, run 'lsh s3 access-keys forget %s' and create a new one", c.Name) + } + return exitcode.Errorf(exitcode.Credentials, "the backend rejected %s (%s)%s", cred, resp.Code, hint) + case "AuthorizationHeaderMalformed": + expected := resp.Region + signed := "" + if b != nil { + signed = b.SigningRegion + } + if expected != "" { + return exitcode.Errorf(exitcode.Credentials, "signing region mismatch: the backend expects %q but the request was signed for %q; retry with --signing-region %s (or %s=%s)", expected, signed, expected, EnvSigningRegion, expected) + } + return exitcode.Errorf(exitcode.Credentials, "signing region mismatch (signed for %q); retry with --signing-region ", signed) + case "BucketNotEmpty": + return exitcode.Errorf(exitcode.Refused, "bucket %s is not empty; re-run with --force to delete its objects first", bucketName) + case "BucketAlreadyOwnedByYou", "BucketAlreadyExists": + return exitcode.Errorf(exitcode.Usage, "bucket %s already exists", bucketName) + case "EntityTooLarge": + return exitcode.Errorf(exitcode.Generic, "object exceeds the backend size limit: %s", resp.Message) + case "RequestTimeTooSkewed": + return exitcode.Errorf(exitcode.Generic, "the local clock is off by more than 15 minutes; fix the system time and retry") + case "InvalidArgument", "InvalidRequest", "XMinioInvalidObjectName", "KeyTooLongError", "InvalidBucketName": + return exitcode.Errorf(exitcode.Usage, "%s: %s", resp.Code, resp.Message) + case "ObjectLocked", "InvalidRetentionPeriod", "ObjectLockConfigurationNotFoundError": + return exitcode.Errorf(exitcode.Refused, "%s: %s", resp.Code, resp.Message) + case "PreconditionFailed": + return exitcode.Errorf(exitcode.Generic, "precondition failed: %s", resp.Message) + case "SlowDown", "ServiceUnavailable", "InternalError": + return exitcode.Errorf(exitcode.Generic, "the backend is busy (%s); retry in a moment", resp.Code) + } + if resp.StatusCode == 403 { + return exitcode.Errorf(exitcode.Permission, "access denied to %s using %s", bucketName, cred) + } + if resp.StatusCode == 404 { + return exitcode.Errorf(exitcode.NotFound, "not found: %s", firstNonEmpty(resp.Message, resp.Code, "404")) + } + msg := firstNonEmpty(resp.Message, resp.Code) + if resp.Code != "" && resp.Message != "" { + msg = resp.Code + ": " + resp.Message + } + return exitcode.Errorf(exitcode.Generic, "%s", Redact(msg)) + } + + if e := humanizeLatitudeError(err, ""); e != nil { + return e + } + + var urlErr *url.Error + var netErr net.Error + var opErr *net.OpError + if errors.As(err, &urlErr) || errors.As(err, &netErr) || errors.As(err, &opErr) { + target := "" + if b != nil { + target = b.Endpoint + } + return exitcode.Errorf(exitcode.Generic, "could not reach %s: %v", target, Redact(err.Error())) + } + return exitcode.New(exitcode.Generic, errors.New(Redact(err.Error()))) +} + +func displayOrBackend(b *Bucket) string { + if b == nil { + return "" + } + if b.Name != "" { + return b.Name + } + return b.BucketName +} + +// humanizeAPIError maps Latitude API errors for control-plane calls. +func humanizeAPIError(err error, what string) error { + if e := humanizeLatitudeError(err, what); e != nil { + return e + } + // Same interruption contract as Humanize: the exit code must not depend on + // whether the signal landed during an API call or an S3 call. + if errors.Is(err, context.Canceled) { + return exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + if errors.Is(err, context.DeadlineExceeded) { + return exitcode.Errorf(exitcode.Generic, "timed out: %v", err) + } + var already *exitcode.Error + if errors.As(err, &already) { + return err + } + return exitcode.New(exitcode.Generic, err) +} + +// HumanizeAPI is the exported form of humanizeAPIError for commands that +// only talk to the Latitude API (access keys, lifecycle, metrics, usage). +func HumanizeAPI(err error, what string) error { + if err == nil { + return nil + } + return humanizeAPIError(err, what) +} + +// humanizeLatitudeError recognizes the two error shapes the SDK produces: +// *components.APIError (unmodeled statuses; carries the HTTP status and raw +// body) and *components.ErrorObject (JSON:API errors for the statuses the +// spec models — 403/404/422/500 — where the status is a string per entry). +// It returns nil for any other error. +func humanizeLatitudeError(err error, what string) error { + var apiErr *components.APIError + if errors.As(err, &apiErr) { + return humanizeAPIStatus(apiErr.StatusCode, []byte(apiErr.Body), what) + } + var eo *components.ErrorObject + if errors.As(err, &eo) { + status := 0 + var parts []string + for _, e := range eo.Errors { + if status == 0 && e.Status != nil { + if n, convErr := strconv.Atoi(strings.TrimSpace(*e.Status)); convErr == nil { + status = n + } + } + msg := firstNonEmpty(str(e.Detail), str(e.Title), str(e.Code)) + if msg == "" { + continue + } + if e.Source != nil && e.Source.Pointer != nil && *e.Source.Pointer != "" { + msg = strings.TrimPrefix(*e.Source.Pointer, "/data/attributes/") + " " + msg + } + parts = append(parts, msg) + } + detail := strings.Join(parts, "; ") + if status == 0 { + switch { + case strings.Contains(strings.ToLower(detail), "not found"): + status = 404 + case strings.Contains(strings.ToLower(detail), "forbidden"), strings.Contains(strings.ToLower(detail), "permission"): + status = 403 + default: + status = 422 + } + } + body, _ := json.Marshal(map[string]interface{}{"errors": []map[string]string{{"detail": detail}}}) + return humanizeAPIStatus(status, body, what) + } + return nil +} + +// humanizeAPIStatus turns an HTTP status + JSON:API body into an exit-coded +// error with the API's own detail message when present. +func humanizeAPIStatus(status int, body []byte, what string) error { + detail := apiDetail(body) + switch status { + case 401: + return exitcode.Errorf(exitcode.Credentials, "your API token is invalid or revoked; run 'lsh login' to sign in again") + case 403: + return exitcode.Errorf(exitcode.Permission, "your API token does not have permission for this action%s", suffix(detail)) + case 404: + if what != "" { + return exitcode.Errorf(exitcode.NotFound, "%s not found%s", what, suffix(detail)) + } + return exitcode.Errorf(exitcode.NotFound, "not found%s", suffix(detail)) + case 409: + if strings.Contains(strings.ToLower(detail), "not empty") { + return exitcode.Errorf(exitcode.Refused, "%s", detail) + } + return exitcode.Errorf(exitcode.Usage, "conflict%s", suffix(detail)) + case 422, 400: + return exitcode.Errorf(exitcode.Usage, "the API rejected the request%s", suffix(detail)) + case 429: + return exitcode.Errorf(exitcode.Generic, "rate limited by the API; retry in a moment") + } + if status >= 500 { + return exitcode.Errorf(exitcode.Generic, "the API returned %d%s", status, suffix(detail)) + } + return exitcode.Errorf(exitcode.Generic, "API error %d%s", status, suffix(detail)) +} + +func suffix(detail string) string { + if detail == "" { + return "" + } + return ": " + detail +} + +// apiDetail extracts the most useful text from a JSON:API error body. +func apiDetail(body []byte) string { + if len(body) == 0 { + return "" + } + var env struct { + Errors []struct { + Title string `json:"title"` + Detail string `json:"detail"` + Code string `json:"code"` + Source struct { + Pointer string `json:"pointer"` + } `json:"source"` + } `json:"errors"` + Error string `json:"error"` + Message string `json:"message"` + } + if err := json.Unmarshal(body, &env); err != nil { + s := strings.TrimSpace(string(body)) + if len(s) > 200 { + s = s[:200] + "…" + } + return s + } + var parts []string + for _, e := range env.Errors { + msg := firstNonEmpty(e.Detail, e.Title, e.Code) + if msg == "" { + continue + } + if e.Source.Pointer != "" { + msg = strings.TrimPrefix(e.Source.Pointer, "/data/attributes/") + " " + msg + } + parts = append(parts, msg) + } + if len(parts) > 0 { + return strings.Join(parts, "; ") + } + return firstNonEmpty(env.Message, env.Error) +} + +// PrintedError marks an error that was already written to stderr by +// PrintError, so the command wrapper in cmd/s3 does not print it a second +// time. It unwraps to the original error (exit codes are preserved). +type PrintedError struct{ Err error } + +func (p *PrintedError) Error() string { return p.Err.Error() } + +// Unwrap exposes the wrapped error to errors.Is/As. +func (p *PrintedError) Unwrap() error { return p.Err } + +// PrintError writes a humanized error to stderr in the CLI's error style and +// returns it wrapped in PrintedError (with its exit code) so RunE can +// propagate it without cobra echoing it again. +func PrintError(err error) error { + if err == nil { + return nil + } + var already *PrintedError + if errors.As(err, &already) { + return err + } + fmt.Fprintln(os.Stderr, tui.ErrorStyle.Render("✗ Error: ")+err.Error()) + return &PrintedError{Err: err} +} + +// IsPrinted reports whether err was already printed by PrintError. +func IsPrinted(err error) bool { + var p *PrintedError + return errors.As(err, &p) +} diff --git a/internal/objectstorage/filter.go b/internal/objectstorage/filter.go new file mode 100644 index 0000000..8d3e9a6 --- /dev/null +++ b/internal/objectstorage/filter.go @@ -0,0 +1,120 @@ +package objectstorage + +import ( + "fmt" + "regexp" + "strings" + + "github.com/spf13/pflag" +) + +// Filters implements the aws s3 --exclude/--include semantics: everything is +// included by default, rules apply in the order given on the command line and +// the last matching rule wins. Patterns are shell globs where `*` also +// matches `/` (fnmatch-style), evaluated against the path relative to the +// source directory or prefix. +type Filters struct { + rules []filterRule +} + +type filterRule struct { + include bool + pattern string + re *regexp.Regexp +} + +// Add appends a rule; returns an error for an invalid pattern. +func (f *Filters) Add(include bool, pattern string) error { + re, err := globToRegexp(pattern) + if err != nil { + return err + } + f.rules = append(f.rules, filterRule{include: include, pattern: pattern, re: re}) + return nil +} + +// Empty reports whether no rules were given. +func (f *Filters) Empty() bool { return f == nil || len(f.rules) == 0 } + +// Include decides whether relPath passes the filters. +func (f *Filters) Include(relPath string) bool { + if f == nil { + return true + } + include := true + for _, r := range f.rules { + if r.re.MatchString(relPath) { + include = r.include + } + } + return include +} + +// Describe lists the rules (for --dry-run/--debug). +func (f *Filters) Describe() string { + if f.Empty() { + return "(no filters)" + } + parts := make([]string, 0, len(f.rules)) + for _, r := range f.rules { + kind := "exclude" + if r.include { + kind = "include" + } + parts = append(parts, fmt.Sprintf("--%s %q", kind, r.pattern)) + } + return strings.Join(parts, " ") +} + +// globToRegexp translates a fnmatch-style pattern into an anchored regexp. +func globToRegexp(pattern string) (*regexp.Regexp, error) { + var sb strings.Builder + sb.WriteString("^") + for i := 0; i < len(pattern); i++ { + c := pattern[i] + switch c { + case '*': + sb.WriteString(".*") + case '?': + sb.WriteString(".") + case '[': + j := strings.IndexByte(pattern[i:], ']') + if j < 0 { + sb.WriteString(regexp.QuoteMeta(string(c))) + continue + } + class := pattern[i+1 : i+j] + if strings.HasPrefix(class, "!") { + class = "^" + class[1:] + } + sb.WriteString("[" + strings.ReplaceAll(class, `\`, `\\`) + "]") + i += j + default: + sb.WriteString(regexp.QuoteMeta(string(c))) + } + } + sb.WriteString("$") + re, err := regexp.Compile(sb.String()) + if err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", pattern, err) + } + return re, nil +} + +// FilterFlags registers --exclude and --include on a flag set, preserving the +// relative order in which they appear (cobra's per-flag slices cannot). +func FilterFlags(fs *pflag.FlagSet, f *Filters) { + fs.Var(&filterValue{f: f, include: false}, "exclude", "exclude paths matching this pattern (repeatable; rules apply in order, last match wins)") + fs.Var(&filterValue{f: f, include: true}, "include", "re-include paths matching this pattern after an --exclude (repeatable)") +} + +type filterValue struct { + f *Filters + include bool +} + +func (v *filterValue) String() string { return "" } +func (v *filterValue) Type() string { return "pattern" } +func (v *filterValue) Set(s string) error { + return v.f.Add(v.include, s) +} diff --git a/internal/objectstorage/filter_test.go b/internal/objectstorage/filter_test.go new file mode 100644 index 0000000..e6f3d84 --- /dev/null +++ b/internal/objectstorage/filter_test.go @@ -0,0 +1,64 @@ +package objectstorage + +import ( + "testing" + + "github.com/spf13/pflag" +) + +// The three examples from the aws user guide ("Use of Exclude and Include +// Filters"): include alone is a no-op, exclude-all then include narrows, +// order matters. +func TestFiltersAWSSemantics(t *testing.T) { + var f Filters + if !f.Include("anything.txt") { + t.Fatal("no rules must include everything") + } + + var onlyInclude Filters + _ = onlyInclude.Add(true, "*.log") + if !onlyInclude.Include("a.txt") { + t.Error("--include alone must not exclude other files (aws semantics)") + } + + var narrow Filters + _ = narrow.Add(false, "*") + _ = narrow.Add(true, "*.log") + if narrow.Include("a.txt") || !narrow.Include("dir/b.log") { + t.Error("--exclude '*' --include '*.log' must include only .log files") + } + + var reversed Filters + _ = reversed.Add(true, "*.log") + _ = reversed.Add(false, "*") + if reversed.Include("b.log") { + t.Error("last matching rule wins: --include then --exclude '*' excludes everything") + } + + var dirOnly Filters + _ = dirOnly.Add(false, "*") + _ = dirOnly.Add(true, "2026/09/*") + if !dirOnly.Include("2026/09/dump.sql") || dirOnly.Include("2026/08/dump.sql") { + t.Error("star must match across slashes like fnmatch in aws") + } +} + +func TestFilterFlagsPreserveOrder(t *testing.T) { + var f Filters + fs := pflag.NewFlagSet("t", pflag.ContinueOnError) + FilterFlags(fs, &f) + if err := fs.Parse([]string{"--exclude", "*", "--include", "*.log", "--exclude", "tmp/*"}); err != nil { + t.Fatal(err) + } + if !f.Include("a.log") || f.Include("tmp/a.log") || f.Include("a.txt") { + t.Errorf("order not preserved: %s", f.Describe()) + } +} + +func TestGlobToRegexpInvalid(t *testing.T) { + var f Filters + if err := f.Add(true, "[a-"); err != nil { + // An unterminated class is treated literally, not as an error. + t.Errorf("unterminated class should be literal, got %v", err) + } +} diff --git a/internal/objectstorage/output.go b/internal/objectstorage/output.go new file mode 100644 index 0000000..5d0bcdd --- /dev/null +++ b/internal/objectstorage/output.go @@ -0,0 +1,250 @@ +package objectstorage + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" +) + +// Object is a listed or inspected S3 object (also used for versions). +type Object struct { + Type string `json:"type"` + Key string `json:"key"` + Size int64 `json:"size"` + LastModified time.Time `json:"last_modified"` + ETag string `json:"etag,omitempty"` + ContentType string `json:"content_type,omitempty"` + StorageClass string `json:"storage_class,omitempty"` + VersionID string `json:"version_id,omitempty"` + IsLatest *bool `json:"is_latest,omitempty"` + IsDeleteMarker bool `json:"is_delete_marker,omitempty"` + // Metadata holds x-amz-meta-* values (stat only). + Metadata map[string]string `json:"metadata,omitempty"` +} + +// ObjectFromInfo converts a minio listing entry. +func ObjectFromInfo(info minio.ObjectInfo, withVersions bool) Object { + o := Object{ + Type: "object", + Key: info.Key, + Size: info.Size, + LastModified: info.LastModified, + ETag: strings.Trim(info.ETag, `"`), + ContentType: info.ContentType, + StorageClass: info.StorageClass, + IsDeleteMarker: info.IsDeleteMarker, + } + // VersionID is copied whenever the backend returns one; IsLatest is only + // meaningful in a versions listing (HEAD responses never carry it). + o.VersionID = info.VersionID + if withVersions { + latest := info.IsLatest + o.IsLatest = &latest + } + if info.IsDeleteMarker { + o.Type = "delete_marker" + } + return o +} + +func (o Object) TableRow() table.Row { + row := table.Row{ + "type": {Label: "Type", Value: strings.ToUpper(shortType(o.Type))}, + "key": {Label: "Key", Value: o.Key, MaxLength: 80}, + "size": {Label: "Size", Value: HumanSize(o.Size)}, + "last_modified": {Label: "Last Modified", Value: FormatTime(o.LastModified)}, + } + if o.VersionID != "" { + row["version_id"] = table.Cell{Label: "Version ID", Value: o.VersionID} + } + if o.ContentType != "" { + row["content_type"] = table.Cell{Label: "Content Type", Value: o.ContentType} + } + if o.ETag != "" { + row["etag"] = table.Cell{Label: "ETag", Value: o.ETag} + } + return row +} + +func shortType(t string) string { + switch t { + case "prefix": + return "PRE" + case "delete_marker": + return "DEL" + default: + return "OBJ" + } +} + +// Prefix is a common prefix ("directory") in a non-recursive listing. +type Prefix struct { + Type string `json:"type"` + Prefix string `json:"prefix"` +} + +// NewPrefix builds a Prefix entry. +func NewPrefix(p string) Prefix { return Prefix{Type: "prefix", Prefix: p} } + +func (p Prefix) TableRow() table.Row { + return table.Row{ + "type": {Label: "Type", Value: "PRE"}, + "key": {Label: "Key", Value: p.Prefix, MaxLength: 80}, + "size": {Label: "Size", Value: ""}, + "last_modified": {Label: "Last Modified", Value: ""}, + } +} + +// TransferResult is one cp/mv operation (or its dry-run plan). +type TransferResult struct { + Op string `json:"op"` // upload | download | copy | move + Source string `json:"source"` + Destination string `json:"destination"` + Size int64 `json:"size"` + ETag string `json:"etag,omitempty"` + ContentType string `json:"content_type,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + Error string `json:"error,omitempty"` +} + +func (t TransferResult) TableRow() table.Row { + status := "ok" + if t.DryRun { + status = "dryrun" + } + if t.Error != "" { + status = "error" + } + return table.Row{ + "op": {Label: "Op", Value: t.Op}, + "source": {Label: "Source", Value: t.Source, MaxLength: 60}, + "destination": {Label: "Destination", Value: t.Destination, MaxLength: 60}, + "size": {Label: "Size", Value: HumanSize(t.Size)}, + "status": {Label: "Status", Value: status}, + } +} + +// HumanLine renders the aws-style line: `upload: ./a to s3://b/a`. +func (t TransferResult) HumanLine() string { + prefix := "" + if t.DryRun { + prefix = "(dryrun) " + } + if t.Error != "" { + return fmt.Sprintf("%s%s failed: %s to %s: %s", prefix, t.Op, t.Source, t.Destination, t.Error) + } + return fmt.Sprintf("%s%s: %s to %s", prefix, t.Op, t.Source, t.Destination) +} + +// DeleteResult is one rm/rb deletion (or its dry-run plan). +type DeleteResult struct { + Bucket string `json:"bucket"` + Key string `json:"key,omitempty"` + VersionID string `json:"version_id,omitempty"` + Deleted bool `json:"deleted"` + DryRun bool `json:"dry_run,omitempty"` + Error string `json:"error,omitempty"` +} + +func (d DeleteResult) TableRow() table.Row { + status := "deleted" + if d.DryRun { + status = "dryrun" + } + if d.Error != "" { + status = "error: " + d.Error + } + return table.Row{ + "bucket": {Label: "Bucket", Value: d.Bucket}, + "key": {Label: "Key", Value: d.Key, MaxLength: 80}, + "status": {Label: "Status", Value: status}, + } +} + +// HumanLine renders `delete: s3://b/k` (or `remove_bucket: s3://b`). +func (d DeleteResult) HumanLine() string { + prefix := "" + if d.DryRun { + prefix = "(dryrun) " + } + target := "s3://" + d.Bucket + verb := "remove_bucket" + if d.Key != "" { + target += "/" + d.Key + verb = "delete" + } + if d.VersionID != "" { + target += " (version " + d.VersionID + ")" + } + if d.Error != "" { + return fmt.Sprintf("%s%s failed: %s: %s", prefix, verb, target, d.Error) + } + return fmt.Sprintf("%s%s: %s", prefix, verb, target) +} + +// PresignResult is the output of `presign`. +type PresignResult struct { + URL string `json:"url"` + Method string `json:"method"` + ExpiresAt time.Time `json:"expires_at"` +} + +func (p PresignResult) TableRow() table.Row { + return table.Row{ + "url": {Label: "URL", Value: p.URL}, + "method": {Label: "Method", Value: p.Method}, + "expires_at": {Label: "Expires At", Value: FormatTime(p.ExpiresAt)}, + } +} + +// AsResponseData adapts a slice of any ResponseData implementation. +func AsResponseData[T renderer.ResponseData](items []T) []renderer.ResponseData { + out := make([]renderer.ResponseData, 0, len(items)) + for _, it := range items { + out = append(out, it) + } + return out +} + +// FormatTime renders timestamps the way `aws s3 ls` does, in local time. +func FormatTime(t time.Time) string { + if t.IsZero() { + return "" + } + return t.Local().Format("2006-01-02 15:04:05") +} + +// HumanSize renders sizes exactly like `aws s3 ls --human-readable`: +// "1 Byte", "12 Bytes", then "1.0 KiB", "3.4 MiB"… promoting the unit when the +// rounded value would reach 1024 (so 1048575 bytes is "1.0 MiB"). +func HumanSize(n int64) string { + if n == 1 { + return "1 Byte" + } + if n < 1024 { + return fmt.Sprintf("%d Bytes", n) + } + units := []string{"KiB", "MiB", "GiB", "TiB", "PiB", "EiB"} + unit := 1024.0 + for i, u := range units { + unit *= 1024 + if math.Round(float64(n)/unit*1024) < 1024 || i == len(units)-1 { + return fmt.Sprintf("%.1f %s", 1024*float64(n)/unit, u) + } + } + return fmt.Sprintf("%d Bytes", n) +} + +// LsSummary renders the --summarize footer for object listings. +func LsSummary(count int, bytes int64, human bool) string { + size := fmt.Sprintf("%d", bytes) + if human { + size = HumanSize(bytes) + } + return fmt.Sprintf("\nTotal Objects: %d\n Total Size: %s", count, size) +} diff --git a/internal/objectstorage/prompt.go b/internal/objectstorage/prompt.go new file mode 100644 index 0000000..982fca0 --- /dev/null +++ b/internal/objectstorage/prompt.go @@ -0,0 +1,228 @@ +package objectstorage + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/tui" + "github.com/spf13/cobra" + "golang.org/x/term" +) + +// stdin/stderr are package variables so tests can substitute them. +var ( + promptIn io.Reader = os.Stdin + promptOut io.Writer = os.Stderr + isTTY = func() bool { + return term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stderr.Fd())) + } +) + +// CanPrompt reports whether the command may ask the user something: --no-input +// was not given, and both stdin and stderr are terminals. Prompts are written +// to stderr so stdout stays clean for pipes. +func CanPrompt(cmd *cobra.Command) bool { + if cmd != nil { + if noInput, err := cmd.Flags().GetBool("no-input"); err == nil && noInput { + return false + } + } + return isTTY() +} + +// Confirm asks a yes/no question. In a real terminal it uses the shared +// Bubble Tea confirm widget (the same y/n prompt as the rest of the CLI); +// otherwise (tests, redirected streams) it falls back to a plain [y/N] line so +// the flow stays scriptable. EOF counts as No. +func Confirm(question string) (bool, error) { + if isTTY() { + return tui.RunConfirm(question) + } + return confirmLine(question) +} + +// confirmLine is the non-interactive fallback of Confirm. +func confirmLine(question string) (bool, error) { + fmt.Fprintf(promptOut, "? %s [y/N] ", question) + line, err := readLine() + if err != nil && line == "" { + fmt.Fprintln(promptOut) + return false, nil + } + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + return true, nil + } + return false, nil +} + +// Choose lets the user pick one option and returns its index (0-based), or -1 +// when the selection is cancelled. In a real terminal it uses the shared +// Bubble Tea list (arrow keys, filtering — the same picker as the rest of the +// CLI); otherwise it falls back to a numbered menu read from stdin. +func Choose(question string, options []string, defaultIndex int) (int, error) { + if isTTY() { + choice, err := tui.RunList(question, options, nil) + if err != nil { + // Cancelled (esc/ctrl+c) or no selection. + return -1, nil + } + for i, opt := range options { + if opt == choice { + return i, nil + } + } + return -1, nil + } + return chooseLine(question, options, defaultIndex) +} + +// chooseLine is the non-interactive fallback of Choose. +func chooseLine(question string, options []string, defaultIndex int) (int, error) { + fmt.Fprintln(promptOut, "? "+question) + for i, opt := range options { + marker := " " + if i == defaultIndex { + marker = "*" + } + fmt.Fprintf(promptOut, " %s[%d] %s\n", marker, i+1, opt) + } + fmt.Fprintf(promptOut, "> ") + line, err := readLine() + line = strings.TrimSpace(line) + if err != nil && line == "" { + fmt.Fprintln(promptOut) + return -1, nil + } + if line == "" { + return defaultIndex, nil + } + n, convErr := strconv.Atoi(line) + if convErr != nil || n < 1 || n > len(options) { + return -1, nil + } + return n - 1, nil +} + +// ReadSecret reads a secret from stdin without echo when stdin is a terminal, +// or a single line otherwise (pipes, heredocs). cmd is used to honour +// --no-input: a session that forbids prompts must fail instead of blocking on +// a terminal read. Piping the value on stdin keeps working either way. +func ReadSecret(cmd *cobra.Command, prompt string) (string, error) { + if f, ok := promptIn.(*os.File); ok && term.IsTerminal(int(f.Fd())) && !CanPrompt(cmd) { + return "", exitcode.Errorf(exitcode.Refused, "a secret is required but prompts are disabled; pipe it on stdin instead: echo \"$SECRET\" | lsh ...") + } + fmt.Fprint(promptOut, prompt) + if f, ok := promptIn.(*os.File); ok && term.IsTerminal(int(f.Fd())) { + b, err := term.ReadPassword(int(f.Fd())) + fmt.Fprintln(promptOut) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil + } + line, err := readLine() + if err != nil && line == "" { + return "", err + } + return strings.TrimSpace(line), nil +} + +func readLine() (string, error) { + r := bufio.NewReader(promptIn) + line, err := r.ReadString('\n') + return strings.TrimRight(line, "\r\n"), err +} + +// ConfirmOrRefuse implements the confirmation contract of destructive +// commands: --yes skips the prompt; otherwise an interactive session is asked +// and a non-interactive one fails fast with exit 7 instead of hanging. +func ConfirmOrRefuse(cmd *cobra.Command, yes bool, question string) error { + if yes { + return nil + } + if !CanPrompt(cmd) { + return exitcode.Errorf(exitcode.Refused, "%s — refusing to continue without --yes in a non-interactive session", question) + } + ok, err := Confirm(question) + if err != nil { + return err + } + if !ok { + return exitcode.Errorf(exitcode.Refused, "cancelled") + } + return nil +} + +// Warnf prints a warning line to stderr. +func Warnf(format string, a ...interface{}) { + fmt.Fprintf(os.Stderr, "warning: "+format+"\n", a...) +} + +// Hintf prints an informational line to stderr (never stdout). +func Hintf(format string, a ...interface{}) { + fmt.Fprintf(os.Stderr, format+"\n", a...) +} + +// ReadLine prints a prompt on stderr and reads one line from stdin (the +// empty string on EOF). Use it for free-text answers such as names. +func ReadLine(prompt string) (string, error) { + fmt.Fprint(promptOut, "? "+prompt) + line, err := readLine() + if err != nil && line == "" { + fmt.Fprintln(promptOut) + return "", nil + } + return strings.TrimSpace(line), nil +} + +// ChooseMany lets the user pick several options and returns their indices. In +// a real terminal it uses the shared Bubble Tea checkbox list (space to toggle, +// enter to confirm); otherwise it falls back to reading a comma/space separated +// list of numbers. Aborting (esc/ctrl+c, or EOF in the fallback) is a refusal, +// not a usage error; a confirmed but empty selection returns no indices. +func ChooseMany(question string, options []string) ([]int, error) { + if isTTY() { + idx, err := tui.RunMultiSelect(question, options, nil) + if errors.Is(err, tui.ErrCanceled) { + return nil, exitcode.Errorf(exitcode.Refused, "cancelled") + } + return idx, err + } + return chooseManyLine(question, options) +} + +// chooseManyLine is the non-interactive fallback of ChooseMany. +func chooseManyLine(question string, options []string) ([]int, error) { + // The numbered list only exists in this fallback, so the hint about typing + // numbers belongs here and not in the callers' question text. + fmt.Fprintln(promptOut, "? "+question+" (comma-separated numbers)") + for i, opt := range options { + fmt.Fprintf(promptOut, " [%d] %s\n", i+1, opt) + } + fmt.Fprint(promptOut, "> ") + line, err := readLine() + if err != nil && strings.TrimSpace(line) == "" { + fmt.Fprintln(promptOut) + return nil, exitcode.Errorf(exitcode.Refused, "cancelled") + } + seen := map[int]bool{} + var out []int + for _, tok := range strings.FieldsFunc(line, func(r rune) bool { return r == ',' || r == ' ' || r == ';' }) { + n, convErr := strconv.Atoi(strings.TrimSpace(tok)) + if convErr != nil || n < 1 || n > len(options) { + return nil, nil + } + if !seen[n-1] { + seen[n-1] = true + out = append(out, n-1) + } + } + return out, nil +} diff --git a/internal/objectstorage/prompt_test.go b/internal/objectstorage/prompt_test.go new file mode 100644 index 0000000..89854b5 --- /dev/null +++ b/internal/objectstorage/prompt_test.go @@ -0,0 +1,63 @@ +package objectstorage + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" +) + +// withPrompt swaps the prompt streams for the duration of a test. +func withPrompt(t *testing.T, stdin string) *bytes.Buffer { + t.Helper() + out := &bytes.Buffer{} + inSaved, outSaved := promptIn, promptOut + promptIn, promptOut = strings.NewReader(stdin), out + t.Cleanup(func() { promptIn, promptOut = inSaved, outSaved }) + return out +} + +// TestChooseManyLineCancelIsRefused covers the exit-code fix: aborting a +// selection is a refusal (7), not a usage error (2), and the numbered-list +// hint belongs to this fallback rather than to the callers' question text. +func TestChooseManyLineCancelIsRefused(t *testing.T) { + out := withPrompt(t, "") // EOF: nothing typed + idx, err := chooseManyLine("Which buckets should the key cover?", []string{"a", "b"}) + if idx != nil { + t.Errorf("selected %v on EOF", idx) + } + if code := exitcode.Of(err); code != exitcode.Refused { + t.Errorf("exit code = %d, want %d", code, exitcode.Refused) + } + if !strings.Contains(out.String(), "comma-separated numbers") { + t.Errorf("the fallback must explain how to answer, got %q", out.String()) + } +} + +func TestChooseManyLineSelection(t *testing.T) { + withPrompt(t, "1,3\n") + idx, err := chooseManyLine("Which buckets?", []string{"a", "b", "c"}) + if err != nil { + t.Fatalf("chooseManyLine: %v", err) + } + if len(idx) != 2 || idx[0] != 0 || idx[1] != 2 { + t.Errorf("selected %v, want [0 2]", idx) + } +} + +// TestHumanizeAPIInterruption covers the exit-code fix on the control plane: +// Ctrl-C must yield 130 whichever phase of a command it lands in. +func TestHumanizeAPIInterruption(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := HumanizeAPI(ctx.Err(), "project \"x\"") + if code := exitcode.Of(err); code != exitcode.Interrupted { + t.Errorf("exit code = %d, want %d (err=%v)", code, exitcode.Interrupted, err) + } + // The S3 path already behaved this way; the two must agree. + if code := exitcode.Of(Humanize(ctx.Err(), nil, nil)); code != exitcode.Interrupted { + t.Errorf("Humanize disagrees: %d", code) + } +} diff --git a/internal/objectstorage/region.go b/internal/objectstorage/region.go new file mode 100644 index 0000000..706e7f9 --- /dev/null +++ b/internal/objectstorage/region.go @@ -0,0 +1,60 @@ +package objectstorage + +import ( + "fmt" + "net/url" + "strings" +) + +// DefaultSigningRegion is used when the endpoint hostname does not encode a +// region. VAST clusters accept any region string; Wasabi requires the one in +// the hostname, which SigningRegion extracts. +const DefaultSigningRegion = "us-east-1" + +// SigningRegion derives the SigV4 signing region from a bucket endpoint. +// +// Latitude publishes two endpoint shapes (see the plan, §3): +// +// https://s3..storage.sh standard tier (CNAME to Wasabi) → +// https://objects..storage.sh high_performance tier (VAST) → +// +// Wasabi rejects requests signed with the wrong region, so for `s3.*` hosts +// the second label is authoritative. VAST does not validate the region, so the +// site label is used only for consistency; any value would work there. +// Anything else falls back to DefaultSigningRegion. +func SigningRegion(endpoint string) string { + host, _, err := EndpointHost(endpoint) + if err != nil { + return DefaultSigningRegion + } + labels := strings.Split(host, ".") + if len(labels) >= 3 && (labels[0] == "s3" || labels[0] == "objects") && labels[1] != "" { + return labels[1] + } + return DefaultSigningRegion +} + +// EndpointHost splits an endpoint into the host[:port] minio expects and +// whether TLS should be used. Bare hosts (no scheme) default to https. +func EndpointHost(endpoint string) (host string, secure bool, err error) { + e := strings.TrimSpace(endpoint) + if e == "" { + return "", false, fmt.Errorf("empty endpoint") + } + if !strings.Contains(e, "://") { + e = "https://" + e + } + u, err := url.Parse(e) + if err != nil || u.Host == "" { + return "", false, fmt.Errorf("invalid endpoint %q", endpoint) + } + switch strings.ToLower(u.Scheme) { + case "https": + secure = true + case "http": + secure = false + default: + return "", false, fmt.Errorf("unsupported endpoint scheme %q (use http or https)", u.Scheme) + } + return u.Host, secure, nil +} diff --git a/internal/objectstorage/region_test.go b/internal/objectstorage/region_test.go new file mode 100644 index 0000000..eecd70f --- /dev/null +++ b/internal/objectstorage/region_test.go @@ -0,0 +1,41 @@ +package objectstorage + +import "testing" + +func TestSigningRegion(t *testing.T) { + cases := map[string]string{ + "https://s3.us-east-1.storage.sh": "us-east-1", + "https://s3.eu-west-2.storage.sh": "eu-west-2", + "https://s3.ap-northeast-1.storage.sh": "ap-northeast-1", + "https://objects.tyo.storage.sh": "tyo", + "https://objects.nyc.storage.sh": "nyc", + "objects.lon2.storage.sh": "lon2", + "https://127.0.0.1:9000": DefaultSigningRegion, + "http://localhost:9000": DefaultSigningRegion, + "https://example.com": DefaultSigningRegion, + "": DefaultSigningRegion, + } + for in, want := range cases { + if got := SigningRegion(in); got != want { + t.Errorf("SigningRegion(%q) = %q, want %q", in, got, want) + } + } +} + +func TestEndpointHost(t *testing.T) { + host, secure, err := EndpointHost("https://s3.us-east-1.storage.sh") + if err != nil || host != "s3.us-east-1.storage.sh" || !secure { + t.Errorf("https: %q %v %v", host, secure, err) + } + host, secure, err = EndpointHost("http://127.0.0.1:9000") + if err != nil || host != "127.0.0.1:9000" || secure { + t.Errorf("http: %q %v %v", host, secure, err) + } + host, secure, err = EndpointHost("objects.tyo.storage.sh") + if err != nil || host != "objects.tyo.storage.sh" || !secure { + t.Errorf("bare: %q %v %v", host, secure, err) + } + if _, _, err := EndpointHost("ftp://x"); err == nil { + t.Error("ftp should be rejected") + } +} diff --git a/internal/objectstorage/resolver.go b/internal/objectstorage/resolver.go new file mode 100644 index 0000000..b6b1507 --- /dev/null +++ b/internal/objectstorage/resolver.go @@ -0,0 +1,506 @@ +package objectstorage + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/version" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/spf13/viper" +) + +// Storage classes as reported by the API. +const ( + ClassStandard = "standard" + ClassHighPerformance = "high_performance" +) + +// Bucket is the resolved view of an object storage bucket: everything the S3 +// layer needs (endpoint, real bucket name, signing region) plus the metadata +// shown by `stat` and used for safety checks (versioning, object lock). +type Bucket struct { + ID string + Name string // display name + BucketName string // real name on the S3 endpoint + Endpoint string // https://… + StorageClass string + // Site is the Latitude site slug (DAL, TYO4…). The SDK model drops it, so + // it is filled lazily by Resolver.FillSite when needed. + Site string + City string + Country string + ProjectID string + ProjectSlug string + ProjectName string + Versioning bool + Locking bool + // RetentionMode is NONE, GOVERNANCE or COMPLIANCE when object lock is on. + RetentionMode string + RetentionDays int64 + // Source is "default" (created through the API) or "synchronized" + // (imported from the provider; may lack an endpoint). + Source string + CreatedAt *time.Time + SigningRegion string + // EndpointOverride is true when the bucket was not looked up in the API + // (--endpoint-url / LSH_S3_ENDPOINT_URL): only BucketName, Endpoint and + // SigningRegion are meaningful. + EndpointOverride bool + // Data keeps the SDK payload for table/JSON rendering. + Data components.ObjectStorageData +} + +// ProjectRef returns the best human identifier for the bucket's project. +func (b *Bucket) ProjectRef() string { + if b.ProjectSlug != "" { + return b.ProjectSlug + } + return b.ProjectID +} + +// Display returns "name (bkt_…)" for messages. +func (b *Bucket) Display() string { + if b.EndpointOverride { + return b.BucketName + } + if b.ID == "" { + return b.Name + } + return fmt.Sprintf("%s (%s)", b.Name, b.ID) +} + +// Validate makes sure the bucket can be addressed over S3. +func (b *Bucket) Validate() error { + if b.BucketName == "" { + return exitcode.Errorf(exitcode.Generic, "bucket %s has no backend bucket name yet (it may still be provisioning); retry in a few seconds", b.Display()) + } + if b.Endpoint == "" { + return exitcode.Errorf(exitcode.Generic, "bucket %s has no S3 endpoint (source: %s); pass --endpoint-url if you know where it lives", b.Display(), b.Source) + } + // Validate up front so an invalid backend name is a usage error (exit 2) + // on every command instead of surfacing as a generic minio error. + if err := s3utils.CheckValidBucketName(b.BucketName); err != nil { + return exitcode.Errorf(exitcode.Usage, "invalid bucket name %q: %v", b.BucketName, err) + } + return nil +} + +// BucketFromData converts the SDK model into a Bucket. +func BucketFromData(d components.ObjectStorageData) *Bucket { + b := &Bucket{Data: d} + if d.ID != nil { + b.ID = *d.ID + } + a := d.Attributes + if a == nil { + return b + } + b.Name = str(a.Name) + b.BucketName = str(a.BucketName) + b.Endpoint = str(a.Endpoint) + if a.StorageClass != nil { + b.StorageClass = string(*a.StorageClass) + } + if a.Versioning != nil { + b.Versioning = *a.Versioning + } + if a.Locking != nil { + b.Locking = *a.Locking + } + if a.RetentionMode != nil { + b.RetentionMode = string(*a.RetentionMode) + } + if a.RetentionPeriod != nil { + b.RetentionDays = *a.RetentionPeriod + } + b.Source = str(a.Source) + b.CreatedAt = a.CreatedAt + if a.Region != nil { + b.Site = str(a.Region.ID) + b.City = str(a.Region.City) + b.Country = str(a.Region.Country) + } + if a.Project != nil { + b.ProjectID = str(a.Project.ID) + b.ProjectSlug = str(a.Project.Slug) + b.ProjectName = str(a.Project.Name) + } + b.SigningRegion = SigningRegion(b.Endpoint) + return b +} + +func str(s *string) string { + if s == nil { + return "" + } + return *s +} + +// Resolver turns a bucket token (display name, bkt_ ID or backend name) into +// a Bucket using the Latitude API, or bypasses the API when an endpoint +// override is in effect. +type Resolver struct { + API *sdk.Latitudesh + // Project narrows name lookups to one project (ID or slug). Empty means + // the whole team. + Project string + // EndpointURL enables the API-less mode: the token is the real bucket name + // on that endpoint. Saved credentials are never used in this mode. + EndpointURL string + // SigningRegion overrides the region derived from the endpoint. + SigningRegion string + // ClassFilter, when set, keeps only buckets of this storage class when a + // name matches several (already normalized to standard|high_performance). + ClassFilter string + // SiteFilter, when set, keeps only buckets in this Latitude site (slug, + // e.g. DAL, TYO4) when a name matches several. + SiteFilter string + // HasFilterFlags reports whether the command registered + // --storage-class/-c and --site. The ambiguity error only advertises them + // when it does; cp/mv/sync spend --storage-class on a different meaning. + HasFilterFlags bool + // FilterErr, when set, is returned by Resolve/ListBuckets before any call + // (e.g. an invalid --storage-class value). + FilterErr error + // RetryOptions are passed to every SDK call. + RetryOptions []operations.Option +} + +// Resolve resolves ref.Bucket. +func (r *Resolver) Resolve(ctx context.Context, token string) (*Bucket, error) { + if r.FilterErr != nil { + return nil, r.FilterErr + } + token = strings.TrimSpace(token) + if token == "" { + return nil, exitcode.Errorf(exitcode.Usage, "missing bucket name") + } + if r.EndpointURL != "" { + b := &Bucket{ + Name: token, + BucketName: token, + Endpoint: r.EndpointURL, + EndpointOverride: true, + SigningRegion: SigningRegion(r.EndpointURL), + } + if r.SigningRegion != "" { + b.SigningRegion = r.SigningRegion + } + return b, nil + } + if r.API == nil { + return nil, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login', or set LSH_S3_ENDPOINT_URL to address the bucket without the API") + } + + var b *Bucket + if strings.HasPrefix(token, "bkt_") { + resp, err := r.API.ObjectStorage.GetStorageBucket(ctx, token, r.RetryOptions...) + if err != nil { + return nil, humanizeAPIError(err, fmt.Sprintf("bucket %q", token)) + } + if resp.Object == nil || resp.Object.Data == nil { + return nil, bucketNotFound(token, r.Project) + } + b = BucketFromData(*resp.Object.Data) + } else { + list, err := r.ListBuckets(ctx) + if err != nil { + return nil, err + } + var matches []components.ObjectStorageData + for _, d := range list { + if d.Attributes == nil { + continue + } + if str(d.Attributes.Name) == token || str(d.Attributes.BucketName) == token { + matches = append(matches, d) + } + } + nameMatches := len(matches) + matches, err = r.applyFilters(ctx, matches) + if err != nil { + return nil, err + } + switch len(matches) { + case 0: + if nameMatches > 0 { + // The name existed but the --storage-class/--site filters ruled + // every candidate out. + return nil, exitcode.Errorf(exitcode.NotFound, "no bucket named %q matches the given filters (%s); run 'lsh s3 ls' to see the buckets", token, r.filterDesc()) + } + return nil, bucketNotFound(token, r.Project) + case 1: + b = BucketFromData(matches[0]) + default: + return nil, ambiguousBucket(token, matches, r.HasFilterFlags) + } + // The list payload omits some attributes; fetch the full record. + if b.ID != "" && (b.Endpoint == "" || b.BucketName == "") { + resp, err := r.API.ObjectStorage.GetStorageBucket(ctx, b.ID, r.RetryOptions...) + if err == nil && resp.Object != nil && resp.Object.Data != nil { + b = BucketFromData(*resp.Object.Data) + } + } + } + if r.SigningRegion != "" { + b.SigningRegion = r.SigningRegion + } + return b, nil +} + +// ListBuckets returns the team's buckets (filtered by r.Project when set). +func (r *Resolver) ListBuckets(ctx context.Context) ([]components.ObjectStorageData, error) { + if r.FilterErr != nil { + return nil, r.FilterErr + } + if r.API == nil { + return nil, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login' first") + } + var filter *string + if r.Project != "" { + p := r.Project + filter = &p + } + resp, err := r.API.ObjectStorage.GetStorageBuckets(ctx, filter, r.RetryOptions...) + if err != nil { + return nil, humanizeAPIError(err, "buckets") + } + if resp.ObjectStorages == nil { + return nil, nil + } + return resp.ObjectStorages.Data, nil +} + +// applyFilters narrows name matches by storage class and site. Class comes +// straight from the list payload; the site slug does not (the SDK model drops +// it), so it is fetched once from the API only when a site filter is set. +func (r *Resolver) applyFilters(ctx context.Context, matches []components.ObjectStorageData) ([]components.ObjectStorageData, error) { + if (r.ClassFilter == "" && r.SiteFilter == "") || len(matches) == 0 { + return matches, nil + } + var sites map[string]string + if r.SiteFilter != "" { + var err error + if sites, err = RawBucketSitesForProject(ctx, "", r.Project); err != nil { + return nil, err + } + } + out := make([]components.ObjectStorageData, 0, len(matches)) + for _, d := range matches { + b := BucketFromData(d) + if r.ClassFilter != "" && !strings.EqualFold(b.StorageClass, r.ClassFilter) { + continue + } + if r.SiteFilter != "" { + site := b.Site + if site == "" && d.ID != nil { + site = sites[*d.ID] + } + if !strings.EqualFold(site, r.SiteFilter) { + continue + } + } + out = append(out, d) + } + return out, nil +} + +// filterDesc renders the active filters for error messages. +func (r *Resolver) filterDesc() string { + var parts []string + if r.ClassFilter != "" { + parts = append(parts, "--storage-class "+r.ClassFilter) + } + if r.SiteFilter != "" { + parts = append(parts, "--site "+r.SiteFilter) + } + return strings.Join(parts, ", ") +} + +func ambiguousBucket(token string, matches []components.ObjectStorageData, hasFilterFlags bool) error { + lines := make([]string, 0, len(matches)) + for _, d := range matches { + b := BucketFromData(d) + lines = append(lines, fmt.Sprintf(" %s project=%s class=%s site=%s", b.ID, b.ProjectRef(), b.StorageClass, firstNonEmpty(b.Site, b.City))) + } + sort.Strings(lines) + narrow := "narrow it with --project, or use the bkt_ ID" + if hasFilterFlags { + narrow = "narrow it with --project, --storage-class/-c or --site, or use the bkt_ ID" + } + return exitcode.Errorf(exitcode.Usage, "bucket name %q is ambiguous:\n%s\n%s", token, strings.Join(lines, "\n"), narrow) +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +// FillSite fetches the bucket's site slug (e.g. TYO4) straight from the API +// when the SDK model did not carry it. It is a no-op when already known or in +// endpoint-override mode. +func (r *Resolver) FillSite(ctx context.Context, b *Bucket) error { + if b == nil || b.Site != "" || b.EndpointOverride || b.ID == "" { + return nil + } + sites, err := RawBucketSites(ctx, b.ID) + if err != nil { + return err + } + b.Site = sites[b.ID] + return nil +} + +// RawBucketSites returns bucket ID → site slug for one bucket (bucketID set) +// or for every bucket of the team (bucketID empty). It talks to the API +// directly because the generated model discards `region.site`. +func RawBucketSites(ctx context.Context, bucketID string) (map[string]string, error) { + return RawBucketSitesForProject(ctx, bucketID, "") +} + +// RawBucketSitesForProject is RawBucketSites restricted to one project (ID or +// slug) when bucketID is empty, avoiding a team-wide listing. +func RawBucketSitesForProject(ctx context.Context, bucketID, project string) (map[string]string, error) { + path := "/storage/buckets" + var query url.Values + if bucketID != "" { + path += "/" + url.PathEscape(bucketID) + } else if project != "" { + query = url.Values{"filter[project]": {project}} + } + body, err := rawAPIGet(ctx, path, query) + if err != nil { + return nil, err + } + type envelope struct { + Data json.RawMessage `json:"data"` + } + type item struct { + ID string `json:"id"` + Attributes struct { + Region struct { + Site struct { + Slug string `json:"slug"` + } `json:"site"` + } `json:"region"` + } `json:"attributes"` + } + var env envelope + if err := json.Unmarshal(body, &env); err != nil { + return nil, fmt.Errorf("could not parse bucket response: %w", err) + } + out := map[string]string{} + var many []item + if err := json.Unmarshal(env.Data, &many); err == nil { + for _, it := range many { + out[it.ID] = it.Attributes.Region.Site.Slug + } + return out, nil + } + var one item + if err := json.Unmarshal(env.Data, &one); err != nil { + return nil, fmt.Errorf("could not parse bucket response: %w", err) + } + out[one.ID] = one.Attributes.Region.Site.Slug + return out, nil +} + +// RawAccessKeyScopes returns, per access key ID, the bucket names and access +// level the API reports (`buckets[]` and `access`), which the SDK model does +// not expose. project is the project ID or slug. +func RawAccessKeyScopes(ctx context.Context, project string) (map[string]AccessKeyScope, error) { + body, err := rawAPIGet(ctx, "/storage/access_keys", url.Values{"project": {project}}) + if err != nil { + return nil, err + } + var env struct { + Data map[string][]struct { + AccessKeyID string `json:"access_key_id"` + Username string `json:"username"` + Access string `json:"access"` + Buckets []string `json:"buckets"` + Region string `json:"region"` + } `json:"data"` + } + if err := json.Unmarshal(body, &env); err != nil { + return nil, fmt.Errorf("could not parse access keys response: %w", err) + } + out := map[string]AccessKeyScope{} + for class, keys := range env.Data { + for _, k := range keys { + out[k.AccessKeyID] = AccessKeyScope{Access: k.Access, Buckets: k.Buckets, StorageClass: class, Site: k.Region, Username: k.Username} + } + } + return out, nil +} + +// AccessKeyScope is the scope information the API returns for a key. +type AccessKeyScope struct { + // Access is fullaccess, rw, readonly or "" (unknown). + Access string + Buckets []string // backend bucket names + StorageClass string + Site string + Username string +} + +// rawAPIGet performs an authenticated GET against the Latitude API and returns +// the body. It reuses the same connection settings (hostname, scheme, token, +// API version) the SDK client is configured with. +func rawAPIGet(ctx context.Context, path string, query url.Values) ([]byte, error) { + token := viper.GetString("Authorization") + if token == "" { + return nil, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login' first") + } + host := viper.GetString("hostname") + if host == "" { + host = "api.latitude.sh" + } + scheme := viper.GetString("scheme") + if scheme == "" { + scheme = "https" + } + u := scheme + "://" + host + path + if len(query) > 0 { + u += "?" + query.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+strings.TrimPrefix(token, "Bearer ")) + req.Header.Set("Accept", "application/vnd.api+json, application/json") + req.Header.Set("User-Agent", "lsh/"+version.Version) + if v := viper.GetString("api-version"); v != "" { + req.Header.Set("API-Version", v) + } + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, exitcode.Errorf(exitcode.Generic, "could not reach %s: %v", host, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, humanizeAPIStatus(resp.StatusCode, body, path) + } + return body, nil +} diff --git a/internal/objectstorage/resolver_filter_test.go b/internal/objectstorage/resolver_filter_test.go new file mode 100644 index 0000000..f5165fb --- /dev/null +++ b/internal/objectstorage/resolver_filter_test.go @@ -0,0 +1,45 @@ +package objectstorage + +import ( + "context" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func classData(id, name, class string) components.ObjectStorageData { + i, n := id, name + c := components.StorageClass(class) + return components.ObjectStorageData{ID: &i, Attributes: &components.ObjectStorageDataAttributes{Name: &n, StorageClass: &c}} +} + +func TestApplyFiltersByClass(t *testing.T) { + matches := []components.ObjectStorageData{ + classData("bkt_std", "backups", "standard"), + classData("bkt_hp", "backups", "high_performance"), + } + r := &Resolver{ClassFilter: "high_performance"} + out, err := r.applyFilters(context.Background(), matches) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || *out[0].ID != "bkt_hp" { + t.Fatalf("class filter kept %d matches, want just bkt_hp", len(out)) + } + // No filter keeps everything. + out, _ = (&Resolver{}).applyFilters(context.Background(), matches) + if len(out) != 2 { + t.Fatalf("no filter must keep both, got %d", len(out)) + } +} + +func TestResolveFilterErr(t *testing.T) { + r := &Resolver{FilterErr: exitcode.Errorf(exitcode.Usage, "bad class")} + if _, err := r.Resolve(context.Background(), "backups"); exitcode.Of(err) != exitcode.Usage { + t.Fatalf("Resolve must return FilterErr, got %v", err) + } + if _, err := r.ListBuckets(context.Background()); exitcode.Of(err) != exitcode.Usage { + t.Fatalf("ListBuckets must return FilterErr, got %v", err) + } +} diff --git a/internal/objectstorage/s3test/server.go b/internal/objectstorage/s3test/server.go new file mode 100644 index 0000000..ae911f0 --- /dev/null +++ b/internal/objectstorage/s3test/server.go @@ -0,0 +1,808 @@ +// Package s3test is an in-memory, path-style S3 server for unit tests. It +// implements the subset of the API the CLI uses (ListObjectsV2, ListObjectVersions, +// Put/Get/Head/Delete object, multi-delete, multipart upload, server-side copy) +// and records every request so tests can assert on headers (no checksum +// trailers, signing region, path style) without a real backend. +package s3test + +import ( + "crypto/md5" + "encoding/hex" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// Object is a stored object. +type Object struct { + Key string + Data []byte + ContentType string + Metadata map[string]string // x-amz-meta-* (lower-case keys without prefix) + Headers map[string]string // other response headers (Cache-Control…) + LastModified time.Time + ETag string + VersionID string +} + +// Bucket is a stored bucket. +type Bucket struct { + Name string + Objects map[string]*Object + // Versions keeps non-current versions when Versioned is true. + Versions map[string][]*Object + Versioned bool +} + +// Request is a recorded incoming request. +type Request struct { + Method string + Path string + Query url.Values + Header http.Header + Body []byte +} + +// Server is the fake S3 endpoint. +type Server struct { + mu sync.Mutex + buckets map[string]*Bucket + requests []Request + uploads map[string]*multipart + httptest *httptest.Server + // DenyAll makes every request fail with AccessDenied (403). + DenyAll bool + // FailNext, when set, is returned once for the next request. + FailNext *ErrorResponse + // SlowListPages forces ListObjectsV2 to paginate with at most N keys per + // page when max-keys is larger, to exercise continuation tokens. + MaxListKeys int + now func() time.Time +} + +type multipart struct { + bucket, key string + parts map[int][]byte + contentType string + metadata map[string]string + headers map[string]string +} + +// ErrorResponse is an S3 XML error. +type ErrorResponse struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + Resource string `xml:"Resource,omitempty"` + RequestID string `xml:"RequestId"` + Region string `xml:"Region,omitempty"` + Status int `xml:"-"` +} + +// New starts a server. Call Close when done. +func New() *Server { + s := &Server{ + buckets: map[string]*Bucket{}, + uploads: map[string]*multipart{}, + now: time.Now, + } + s.httptest = httptest.NewServer(http.HandlerFunc(s.handle)) + return s +} + +// URL returns the endpoint (http://127.0.0.1:port). +func (s *Server) URL() string { return s.httptest.URL } + +// Close shuts the server down. +func (s *Server) Close() { s.httptest.Close() } + +// CreateBucket adds an empty bucket. +func (s *Server) CreateBucket(name string) *Bucket { + s.mu.Lock() + defer s.mu.Unlock() + b := &Bucket{Name: name, Objects: map[string]*Object{}, Versions: map[string][]*Object{}} + s.buckets[name] = b + return b +} + +// AddObject stores an object (creating the bucket if needed). +func (s *Server) AddObject(bucket, key string, data []byte, contentType string) *Object { + s.mu.Lock() + b := s.buckets[bucket] + if b == nil { + b = &Bucket{Name: bucket, Objects: map[string]*Object{}, Versions: map[string][]*Object{}} + s.buckets[bucket] = b + } + s.mu.Unlock() + return s.put(b, key, data, contentType, nil, nil) +} + +// Object returns a stored object or nil. +func (s *Server) Object(bucket, key string) *Object { + s.mu.Lock() + defer s.mu.Unlock() + b := s.buckets[bucket] + if b == nil { + return nil + } + return b.Objects[key] +} + +// Keys returns the sorted object keys of a bucket. +func (s *Server) Keys(bucket string) []string { + s.mu.Lock() + defer s.mu.Unlock() + b := s.buckets[bucket] + if b == nil { + return nil + } + keys := make([]string, 0, len(b.Objects)) + for k := range b.Objects { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// Requests returns a copy of the recorded requests. +func (s *Server) Requests() []Request { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]Request, len(s.requests)) + copy(out, s.requests) + return out +} + +// WriteRequests returns the recorded requests that mutate state. +func (s *Server) WriteRequests() []Request { + var out []Request + for _, r := range s.Requests() { + if r.Method == http.MethodPut || r.Method == http.MethodPost || r.Method == http.MethodDelete { + out = append(out, r) + } + } + return out +} + +// ResetRequests clears the request log. +func (s *Server) ResetRequests() { + s.mu.Lock() + defer s.mu.Unlock() + s.requests = nil +} + +func etagOf(data []byte) string { + sum := md5.Sum(data) + return `"` + hex.EncodeToString(sum[:]) + `"` +} + +func (s *Server) put(b *Bucket, key string, data []byte, contentType string, meta, headers map[string]string) *Object { + s.mu.Lock() + defer s.mu.Unlock() + if contentType == "" { + contentType = "binary/octet-stream" + } + o := &Object{Key: key, Data: data, ContentType: contentType, Metadata: meta, Headers: headers, LastModified: s.now().UTC().Truncate(time.Second), ETag: etagOf(data)} + if b.Versioned { + if prev := b.Objects[key]; prev != nil { + b.Versions[key] = append(b.Versions[key], prev) + } + o.VersionID = fmt.Sprintf("v%d", len(b.Versions[key])+1) + } + b.Objects[key] = o + return o +} + +func (s *Server) record(r *http.Request, body []byte) { + s.mu.Lock() + defer s.mu.Unlock() + s.requests = append(s.requests, Request{Method: r.Method, Path: r.URL.Path, Query: r.URL.Query(), Header: r.Header.Clone(), Body: body}) +} + +func writeError(w http.ResponseWriter, e ErrorResponse) { + if e.RequestID == "" { + e.RequestID = "s3test" + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(e.Status) + _ = xml.NewEncoder(w).Encode(e) +} + +func (s *Server) handle(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + if isAWSChunked(r.Header) { + body = decodeAWSChunked(body) + } + s.record(r, body) + + s.mu.Lock() + failNext := s.FailNext + s.FailNext = nil + deny := s.DenyAll + s.mu.Unlock() + if failNext != nil { + writeError(w, *failNext) + return + } + if deny { + writeError(w, ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: 403}) + return + } + + // Path-style: /{bucket}[/{key}] + path := strings.TrimPrefix(r.URL.Path, "/") + if path == "" { + s.listBuckets(w) + return + } + bucketName, key := path, "" + if i := strings.Index(path, "/"); i >= 0 { + bucketName, key = path[:i], path[i+1:] + } + key, _ = url.PathUnescape(key) + + s.mu.Lock() + b := s.buckets[bucketName] + s.mu.Unlock() + if b == nil { + writeError(w, ErrorResponse{Code: "NoSuchBucket", Message: "The specified bucket does not exist", Resource: "/" + bucketName, Status: 404}) + return + } + q := r.URL.Query() + + if key == "" { + switch { + case r.Method == http.MethodGet && q.Has("versions"): + s.listVersions(w, b, q) + case r.Method == http.MethodGet && q.Get("list-type") == "2": + s.listV2(w, b, q) + case r.Method == http.MethodGet && q.Has("location"): + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, `us-east-1`) + case r.Method == http.MethodGet: + // ListObjects v1 fallback: treat as v2 without tokens. + s.listV2(w, b, q) + case r.Method == http.MethodHead: + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && q.Has("delete"): + s.multiDelete(w, b, body) + default: + writeError(w, ErrorResponse{Code: "MethodNotAllowed", Message: "method not allowed on bucket", Status: 405}) + } + return + } + + switch r.Method { + case http.MethodPut: + if src := r.Header.Get("X-Amz-Copy-Source"); src != "" { + s.copyObject(w, b, key, src, r.Header) + return + } + if q.Has("uploadId") { + s.uploadPart(w, q.Get("uploadId"), q.Get("partNumber"), body) + return + } + meta, headers := extractMeta(r.Header) + o := s.put(b, key, body, r.Header.Get("Content-Type"), meta, headers) + w.Header().Set("ETag", o.ETag) + if o.VersionID != "" { + w.Header().Set("x-amz-version-id", o.VersionID) + } + w.WriteHeader(http.StatusOK) + case http.MethodPost: + switch { + case q.Has("uploads"): + s.initiateMultipart(w, b, key, r.Header) + case q.Has("uploadId"): + s.completeMultipart(w, b, key, q.Get("uploadId"), body) + default: + writeError(w, ErrorResponse{Code: "MethodNotAllowed", Message: "bad POST", Status: 405}) + } + case http.MethodGet, http.MethodHead: + s.getObject(w, r, b, key, q.Get("versionId")) + case http.MethodDelete: + if q.Has("uploadId") { + s.mu.Lock() + delete(s.uploads, q.Get("uploadId")) + s.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + return + } + s.mu.Lock() + if vid := q.Get("versionId"); vid != "" { + s.deleteVersion(b, key, vid) + } else { + delete(b.Objects, key) + } + s.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + writeError(w, ErrorResponse{Code: "MethodNotAllowed", Message: "method not allowed", Status: 405}) + } +} + +func (s *Server) deleteVersion(b *Bucket, key, vid string) { + if cur := b.Objects[key]; cur != nil && cur.VersionID == vid { + vs := b.Versions[key] + if len(vs) > 0 { + b.Objects[key] = vs[len(vs)-1] + b.Versions[key] = vs[:len(vs)-1] + } else { + delete(b.Objects, key) + } + return + } + vs := b.Versions[key] + for i, v := range vs { + if v.VersionID == vid { + b.Versions[key] = append(vs[:i], vs[i+1:]...) + return + } + } +} + +func extractMeta(h http.Header) (map[string]string, map[string]string) { + meta := map[string]string{} + headers := map[string]string{} + for k, v := range h { + lk := strings.ToLower(k) + if strings.HasPrefix(lk, "x-amz-meta-") && len(v) > 0 { + meta[strings.TrimPrefix(lk, "x-amz-meta-")] = v[0] + } + switch lk { + case "cache-control", "content-encoding", "content-disposition", "content-language", "expires": + if len(v) > 0 { + headers[http.CanonicalHeaderKey(lk)] = v[0] + } + } + } + return meta, headers +} + +type listBucketsResult struct { + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Buckets []struct { + Name string `xml:"Name"` + CreationDate string `xml:"CreationDate"` + } `xml:"Buckets>Bucket"` +} + +func (s *Server) listBuckets(w http.ResponseWriter) { + s.mu.Lock() + defer s.mu.Unlock() + var res listBucketsResult + for name := range s.buckets { + res.Buckets = append(res.Buckets, struct { + Name string `xml:"Name"` + CreationDate string `xml:"CreationDate"` + }{name, s.now().UTC().Format(time.RFC3339)}) + } + w.Header().Set("Content-Type", "application/xml") + _ = xml.NewEncoder(w).Encode(res) +} + +type listV2Result struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Delimiter string `xml:"Delimiter,omitempty"` + KeyCount int `xml:"KeyCount"` + MaxKeys int `xml:"MaxKeys"` + IsTruncated bool `xml:"IsTruncated"` + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + Contents []contentsEntry `xml:"Contents"` + CommonPrefixes []commonPrefixes `xml:"CommonPrefixes"` +} + +type contentsEntry struct { + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` +} + +type commonPrefixes struct { + Prefix string `xml:"Prefix"` +} + +func (s *Server) listV2(w http.ResponseWriter, b *Bucket, q url.Values) { + s.mu.Lock() + defer s.mu.Unlock() + prefix := q.Get("prefix") + delimiter := q.Get("delimiter") + maxKeys := 1000 + if mk := q.Get("max-keys"); mk != "" { + if n, err := strconv.Atoi(mk); err == nil && n > 0 { + maxKeys = n + } + } + if s.MaxListKeys > 0 && maxKeys > s.MaxListKeys { + maxKeys = s.MaxListKeys + } + startAfter := q.Get("start-after") + if tok := q.Get("continuation-token"); tok != "" { + startAfter = tok + } + + keys := make([]string, 0, len(b.Objects)) + for k := range b.Objects { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + sort.Strings(keys) + + res := listV2Result{Name: b.Name, Prefix: prefix, Delimiter: delimiter, MaxKeys: maxKeys, ContinuationToken: q.Get("continuation-token")} + seenPrefix := map[string]bool{} + count := 0 + var last string + for _, k := range keys { + if startAfter != "" && k <= startAfter { + continue + } + if count >= maxKeys { + res.IsTruncated = true + res.NextContinuationToken = last + break + } + if delimiter != "" { + rest := k[len(prefix):] + if i := strings.Index(rest, delimiter); i >= 0 { + cp := prefix + rest[:i+len(delimiter)] + if !seenPrefix[cp] { + seenPrefix[cp] = true + res.CommonPrefixes = append(res.CommonPrefixes, commonPrefixes{Prefix: cp}) + count++ + } + last = cp + "￿" // skip everything under this prefix + continue + } + } + o := b.Objects[k] + res.Contents = append(res.Contents, contentsEntry{Key: k, LastModified: o.LastModified.Format("2006-01-02T15:04:05.000Z"), ETag: o.ETag, Size: int64(len(o.Data)), StorageClass: "STANDARD"}) + count++ + last = k + } + if res.IsTruncated && strings.HasSuffix(res.NextContinuationToken, "￿") { + // Keys sorted after a prefix marker: use the marker's prefix as token. + res.NextContinuationToken = strings.TrimSuffix(res.NextContinuationToken, "￿") + "￿" + } + res.KeyCount = count + w.Header().Set("Content-Type", "application/xml") + _ = xml.NewEncoder(w).Encode(res) +} + +type listVersionsResult struct { + XMLName xml.Name `xml:"ListVersionsResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + IsTruncated bool `xml:"IsTruncated"` + Versions []versionEntry `xml:"Version"` + DeleteMarkers []versionEntry `xml:"DeleteMarker"` + CommonPrefixes []commonPrefixes `xml:"CommonPrefixes"` +} + +type versionEntry struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` +} + +func (s *Server) listVersions(w http.ResponseWriter, b *Bucket, q url.Values) { + s.mu.Lock() + defer s.mu.Unlock() + prefix := q.Get("prefix") + res := listVersionsResult{Name: b.Name, Prefix: prefix} + keys := make([]string, 0, len(b.Objects)) + for k := range b.Objects { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + sort.Strings(keys) + for _, k := range keys { + o := b.Objects[k] + vid := o.VersionID + if vid == "" { + vid = "null" + } + res.Versions = append(res.Versions, versionEntry{Key: k, VersionID: vid, IsLatest: true, LastModified: o.LastModified.Format("2006-01-02T15:04:05.000Z"), ETag: o.ETag, Size: int64(len(o.Data))}) + for i := len(b.Versions[k]) - 1; i >= 0; i-- { + v := b.Versions[k][i] + res.Versions = append(res.Versions, versionEntry{Key: k, VersionID: v.VersionID, LastModified: v.LastModified.Format("2006-01-02T15:04:05.000Z"), ETag: v.ETag, Size: int64(len(v.Data))}) + } + } + w.Header().Set("Content-Type", "application/xml") + _ = xml.NewEncoder(w).Encode(res) +} + +func (s *Server) getObject(w http.ResponseWriter, r *http.Request, b *Bucket, key, versionID string) { + s.mu.Lock() + o := b.Objects[key] + if o != nil && versionID != "" && o.VersionID != versionID { + o = nil + for _, v := range b.Versions[key] { + if v.VersionID == versionID { + o = v + } + } + } + s.mu.Unlock() + if o == nil { + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusNotFound) + return + } + writeError(w, ErrorResponse{Code: "NoSuchKey", Message: "The specified key does not exist.", Resource: "/" + b.Name + "/" + key, Status: 404}) + return + } + h := w.Header() + h.Set("Content-Type", o.ContentType) + h.Set("ETag", o.ETag) + h.Set("Last-Modified", o.LastModified.UTC().Format(http.TimeFormat)) + h.Set("Accept-Ranges", "bytes") + if o.VersionID != "" { + h.Set("x-amz-version-id", o.VersionID) + } + for k, v := range o.Metadata { + h.Set("X-Amz-Meta-"+k, v) + } + for k, v := range o.Headers { + h.Set(k, v) + } + data := o.Data + status := http.StatusOK + if rng := r.Header.Get("Range"); rng != "" && strings.HasPrefix(rng, "bytes=") { + parts := strings.SplitN(strings.TrimPrefix(rng, "bytes="), "-", 2) + start, _ := strconv.Atoi(parts[0]) + end := len(data) - 1 + if len(parts) == 2 && parts[1] != "" { + if e, err := strconv.Atoi(parts[1]); err == nil && e < end { + end = e + } + } + if start > end || start >= len(data) { + writeError(w, ErrorResponse{Code: "InvalidRange", Message: "The requested range is not satisfiable", Status: 416}) + return + } + h.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + data = data[start : end+1] + status = http.StatusPartialContent + } + h.Set("Content-Length", strconv.Itoa(len(data))) + w.WriteHeader(status) + if r.Method == http.MethodGet { + _, _ = w.Write(data) + } +} + +type deleteRequest struct { + XMLName xml.Name `xml:"Delete"` + Quiet bool `xml:"Quiet"` + Objects []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + } `xml:"Object"` +} + +type deleteResult struct { + XMLName xml.Name `xml:"DeleteResult"` + Deleted []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId,omitempty"` + } `xml:"Deleted"` + Errors []struct { + Key string `xml:"Key"` + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` +} + +func (s *Server) multiDelete(w http.ResponseWriter, b *Bucket, body []byte) { + var req deleteRequest + if err := xml.Unmarshal(body, &req); err != nil { + writeError(w, ErrorResponse{Code: "MalformedXML", Message: err.Error(), Status: 400}) + return + } + var res deleteResult + s.mu.Lock() + for _, o := range req.Objects { + if o.VersionID != "" { + s.deleteVersion(b, o.Key, o.VersionID) + } else { + delete(b.Objects, o.Key) + } + if !req.Quiet { + res.Deleted = append(res.Deleted, struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId,omitempty"` + }{o.Key, o.VersionID}) + } + } + s.mu.Unlock() + w.Header().Set("Content-Type", "application/xml") + _ = xml.NewEncoder(w).Encode(res) +} + +func (s *Server) copyObject(w http.ResponseWriter, b *Bucket, key, src string, h http.Header) { + src = strings.TrimPrefix(src, "/") + src, _ = url.PathUnescape(src) + srcBucket, srcKey := src, "" + if i := strings.Index(src, "/"); i >= 0 { + srcBucket, srcKey = src[:i], src[i+1:] + } + if i := strings.Index(srcKey, "?versionId="); i >= 0 { + srcKey = srcKey[:i] + } + s.mu.Lock() + sb := s.buckets[srcBucket] + var so *Object + if sb != nil { + so = sb.Objects[srcKey] + } + s.mu.Unlock() + if so == nil { + writeError(w, ErrorResponse{Code: "NoSuchKey", Message: "The specified key does not exist.", Status: 404}) + return + } + meta, headers := so.Metadata, so.Headers + contentType := so.ContentType + if strings.EqualFold(h.Get("X-Amz-Metadata-Directive"), "REPLACE") { + meta, headers = extractMeta(h) + if ct := h.Get("Content-Type"); ct != "" { + contentType = ct + } + } + o := s.put(b, key, append([]byte(nil), so.Data...), contentType, meta, headers) + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, `%s%s`, o.LastModified.Format(time.RFC3339), o.ETag) +} + +func (s *Server) initiateMultipart(w http.ResponseWriter, b *Bucket, key string, h http.Header) { + s.mu.Lock() + id := fmt.Sprintf("upload-%d", len(s.uploads)+1) + meta, headers := extractMeta(h) + s.uploads[id] = &multipart{bucket: b.Name, key: key, parts: map[int][]byte{}, contentType: h.Get("Content-Type"), metadata: meta, headers: headers} + s.mu.Unlock() + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, `%s%s%s`, b.Name, xmlEscape(key), id) +} + +func (s *Server) uploadPart(w http.ResponseWriter, uploadID, partNumber string, body []byte) { + n, err := strconv.Atoi(partNumber) + s.mu.Lock() + up := s.uploads[uploadID] + s.mu.Unlock() + if err != nil || up == nil { + writeError(w, ErrorResponse{Code: "NoSuchUpload", Message: "The specified upload does not exist.", Status: 404}) + return + } + s.mu.Lock() + up.parts[n] = append([]byte(nil), body...) + s.mu.Unlock() + w.Header().Set("ETag", etagOf(body)) + w.WriteHeader(http.StatusOK) +} + +func (s *Server) completeMultipart(w http.ResponseWriter, b *Bucket, key, uploadID string, body []byte) { + s.mu.Lock() + up := s.uploads[uploadID] + if up != nil { + delete(s.uploads, uploadID) + } + s.mu.Unlock() + if up == nil { + writeError(w, ErrorResponse{Code: "NoSuchUpload", Message: "The specified upload does not exist.", Status: 404}) + return + } + nums := make([]int, 0, len(up.parts)) + for n := range up.parts { + nums = append(nums, n) + } + sort.Ints(nums) + var data []byte + for _, n := range nums { + data = append(data, up.parts[n]...) + } + o := s.put(b, key, data, up.contentType, up.metadata, up.headers) + o.ETag = fmt.Sprintf(`"%s-%d"`, strings.Trim(etagOf(data), `"`), len(nums)) + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, `%s%s%s`, b.Name, xmlEscape(key), o.ETag) +} + +func xmlEscape(s string) string { + var sb strings.Builder + _ = xml.EscapeText(&sb, []byte(s)) + return sb.String() +} + +// HasChecksumHeaders reports whether any recorded write request carried +// x-amz-checksum-* / x-amz-sdk-checksum-algorithm headers or declared +// checksum trailers — what S3-compatible backends such as Wasabi reject. +// (Plain SigV4 streaming over http is not a checksum trailer.) +func (s *Server) HasChecksumHeaders() bool { + for _, r := range s.WriteRequests() { + for k := range r.Header { + lk := strings.ToLower(k) + if strings.HasPrefix(lk, "x-amz-checksum-") || lk == "x-amz-sdk-checksum-algorithm" || lk == "x-amz-trailer" { + return true + } + } + if strings.Contains(strings.ToUpper(r.Header.Get("X-Amz-Content-Sha256")), "TRAILER") { + return true + } + } + return false +} + +// SigningRegionOf extracts the region from a request's SigV4 Authorization +// header ("" when absent). +func SigningRegionOf(r Request) string { + auth := r.Header.Get("Authorization") + i := strings.Index(auth, "Credential=") + if i < 0 { + return "" + } + parts := strings.Split(auth[i+len("Credential="):], "/") + if len(parts) < 3 { + return "" + } + return parts[2] +} + +// isAWSChunked reports whether the request body uses the SigV4 streaming +// framing (`aws-chunked`), which minio-go emits over plain http. +func isAWSChunked(h http.Header) bool { + if strings.Contains(strings.ToLower(h.Get("Content-Encoding")), "aws-chunked") { + return true + } + return strings.HasPrefix(strings.ToUpper(h.Get("X-Amz-Content-Sha256")), "STREAMING-") +} + +// decodeAWSChunked strips the `;chunk-signature=…\r\n\r\n` +// framing (and any trailing headers after the terminating 0 chunk). +func decodeAWSChunked(body []byte) []byte { + var out []byte + rest := body + for { + nl := strings.Index(string(rest), "\r\n") + if nl < 0 { + return out + } + header := string(rest[:nl]) + rest = rest[nl+2:] + sizeHex := header + if i := strings.IndexByte(header, ';'); i >= 0 { + sizeHex = header[:i] + } + size, err := strconv.ParseInt(strings.TrimSpace(sizeHex), 16, 64) + if err != nil { + // Not framed after all: return the original body. + return body + } + if size == 0 { + return out + } + if int64(len(rest)) < size { + return out + } + out = append(out, rest[:size]...) + rest = rest[size:] + if len(rest) >= 2 && rest[0] == '\r' && rest[1] == '\n' { + rest = rest[2:] + } + } +} diff --git a/internal/objectstorage/storageclass.go b/internal/objectstorage/storageclass.go new file mode 100644 index 0000000..b456b3a --- /dev/null +++ b/internal/objectstorage/storageclass.go @@ -0,0 +1,39 @@ +package objectstorage + +import ( + "fmt" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" +) + +// ParseStorageClass normalizes a user-supplied storage class. It is the single +// alias table shared by every command so the same spelling is accepted +// everywhere: standard|std|wasabi and high_performance|high-performance|hp| +// high|vast (case-insensitive). An empty value returns "" without error so +// callers can apply their own default. +func ParseStorageClass(s string) (string, error) { + v := strings.ToLower(strings.TrimSpace(s)) + switch v { + case "": + return "", nil + case ClassStandard, "std", "wasabi": + return ClassStandard, nil + case ClassHighPerformance, "high-performance", "highperformance", "hp", "high", "vast": + return ClassHighPerformance, nil + } + return "", exitcode.Errorf(exitcode.Usage, "invalid storage class %q (expected standard or high_performance)", s) +} + +// StorageClassLabel returns a short human label for a storage class. +func StorageClassLabel(class string) string { + switch class { + case ClassHighPerformance: + return "high_performance (VAST)" + case ClassStandard: + return "standard (Wasabi)" + } + return class +} + +var _ = fmt.Sprintf diff --git a/internal/objectstorage/uri.go b/internal/objectstorage/uri.go new file mode 100644 index 0000000..489dee2 --- /dev/null +++ b/internal/objectstorage/uri.go @@ -0,0 +1,197 @@ +package objectstorage + +import ( + "fmt" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" +) + +// Ref is a parsed command-line reference to a local path, stdin/stdout or an +// object storage location. +// +// Remote references follow the aws shape: `s3://[/]`. `lsh://` is +// accepted as a synonym. may be the bucket's display name, its +// `bkt_…` ID or the backend bucket name; the resolver decides which. +type Ref struct { + // Raw is the argument as typed. + Raw string + // Bucket is the bucket token (display name, bkt_ id or backend name). + Bucket string + // Key is the object key or prefix; empty when the reference is the bucket + // itself. + Key string + // Remote is true for s3:// and lsh:// references (or bare bucket tokens + // parsed through ParseRemote). + Remote bool + // Stdio is true when the argument is "-" (stdin for sources, stdout for + // destinations). + Stdio bool + // HadScheme records whether an explicit scheme was present. + HadScheme bool +} + +const ( + schemeS3 = "s3://" + schemeLsh = "lsh://" +) + +// IsDir reports whether the reference denotes a prefix (empty key or a key +// ending in "/"), following the aws convention for trailing slashes. +func (r Ref) IsDir() bool { + return r.Remote && (r.Key == "" || strings.HasSuffix(r.Key, "/")) +} + +// String renders the canonical s3:// form for remote references and the raw +// argument otherwise. +func (r Ref) String() string { + if !r.Remote { + return r.Raw + } + if r.Key == "" { + return schemeS3 + r.Bucket + } + return schemeS3 + r.Bucket + "/" + r.Key +} + +// WithKey returns a copy of r pointing at key. +func (r Ref) WithKey(key string) Ref { + r.Key = key + return r +} + +// stripScheme removes a leading s3:// or lsh:// and reports whether one was +// present. +func stripScheme(s string) (string, bool) { + lower := strings.ToLower(s) + switch { + case strings.HasPrefix(lower, schemeS3): + return s[len(schemeS3):], true + case strings.HasPrefix(lower, schemeLsh): + return s[len(schemeLsh):], true + } + return s, false +} + +// ParseRemote parses an argument that must name a bucket or object. The +// scheme is optional (`ls`, `stat`, `rm`, `presign`, `lifecycle`, `metrics` +// accept `my-bucket/prefix/` as well as `s3://my-bucket/prefix/`). +func ParseRemote(arg string) (Ref, error) { + s := strings.TrimSpace(arg) + if s == "" { + return Ref{}, exitcode.Errorf(exitcode.Usage, "missing bucket: expected s3://[/]") + } + if s == "-" { + return Ref{}, exitcode.Errorf(exitcode.Usage, "%q is not a valid bucket reference", arg) + } + rest, had := stripScheme(s) + if strings.HasPrefix(rest, "/") || strings.HasPrefix(rest, ".") { + return Ref{}, exitcode.Errorf(exitcode.Usage, "%q looks like a local path; expected s3://[/]", arg) + } + bucket, key := rest, "" + if i := strings.Index(rest, "/"); i >= 0 { + bucket, key = rest[:i], rest[i+1:] + } + if bucket == "" { + return Ref{}, exitcode.Errorf(exitcode.Usage, "%q has no bucket name; expected s3://[/]", arg) + } + return Ref{Raw: arg, Bucket: bucket, Key: key, Remote: true, HadScheme: had}, nil +} + +// ParseBucketOnly parses an argument that must name a bucket without a key +// (`mb`, `rb`). It mirrors the aws error for `aws s3 rb s3://b/key`. +func ParseBucketOnly(arg string) (Ref, error) { + r, err := ParseRemote(arg) + if err != nil { + return r, err + } + if r.Key != "" { + return r, exitcode.Errorf(exitcode.Usage, "please specify a valid bucket name only (got %q)", arg) + } + return r, nil +} + +// ParseTransferArg parses a `cp`/`mv`/`sync` operand. Only an explicit +// scheme makes it remote; "-" is stdin/stdout; anything else is a local path. +// This is the aws rule that keeps `cp ./file s3://b/` unambiguous. +func ParseTransferArg(arg string) (Ref, error) { + s := strings.TrimSpace(arg) + if s == "" { + return Ref{}, exitcode.Errorf(exitcode.Usage, "empty path argument") + } + if s == "-" { + return Ref{Raw: arg, Stdio: true}, nil + } + if _, had := stripScheme(s); had { + return ParseRemote(s) + } + return Ref{Raw: arg}, nil +} + +// ObjectRef parses an argument that must name an object (bucket and non-empty +// key). Keys ending in "/" are rejected unless allowPrefix is true. +func ObjectRef(arg string, allowPrefix bool) (Ref, error) { + r, err := ParseRemote(arg) + if err != nil { + return r, err + } + if r.Key == "" { + return r, exitcode.Errorf(exitcode.Usage, "%s names a bucket, not an object; expected s3:///", r) + } + if !allowPrefix && strings.HasSuffix(r.Key, "/") { + return r, exitcode.Errorf(exitcode.Usage, "key %q ends with '/'; use --recursive to operate on a prefix", r.Key) + } + return r, nil +} + +// JoinKey appends name to a prefix, inserting "/" when needed. +func JoinKey(prefix, name string) string { + if prefix == "" { + return name + } + if strings.HasSuffix(prefix, "/") { + return prefix + name + } + return prefix + "/" + name +} + +// BaseName returns the last path segment of key ("" when key ends in "/"). +func BaseName(key string) string { + if i := strings.LastIndex(key, "/"); i >= 0 { + return key[i+1:] + } + return key +} + +// NormalizePrefix ensures a non-empty prefix ends with "/" — the aws rule for +// `--recursive` targets, so `rm s3://b/path --recursive` does not touch +// `path2/`. +func NormalizePrefix(prefix string) string { + if prefix == "" || strings.HasSuffix(prefix, "/") { + return prefix + } + return prefix + "/" +} + +// DescribeRef renders a ref for messages, always in s3:// form for remotes. +func DescribeRef(r Ref) string { + if r.Stdio { + return "-" + } + return r.String() +} + +// ErrUsagef is a convenience for usage errors. +func ErrUsagef(format string, a ...interface{}) error { + return exitcode.Errorf(exitcode.Usage, format, a...) +} + +// bucketNotFound formats the "no such bucket" error shared by the resolver. +func bucketNotFound(ref, project string) error { + if project != "" { + return exitcode.Errorf(exitcode.NotFound, "bucket %q not found in project %s; run 'lsh s3 ls' to see available buckets", ref, project) + } + return exitcode.Errorf(exitcode.NotFound, "bucket %q not found; run 'lsh s3 ls' to see available buckets", ref) +} + +var _ = fmt.Sprintf diff --git a/internal/objectstorage/uri_test.go b/internal/objectstorage/uri_test.go new file mode 100644 index 0000000..8b2237f --- /dev/null +++ b/internal/objectstorage/uri_test.go @@ -0,0 +1,105 @@ +package objectstorage + +import ( + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" +) + +func TestParseRemote(t *testing.T) { + cases := []struct { + in string + bucket string + key string + hadScheme bool + wantErr bool + wantIsDir bool + wantString string + }{ + {"s3://backups", "backups", "", true, false, true, "s3://backups"}, + {"s3://backups/", "backups", "", true, false, true, "s3://backups"}, + {"S3://backups/2026/09/", "backups", "2026/09/", true, false, true, "s3://backups/2026/09/"}, + {"lsh://bkt_123/a/b.txt", "bkt_123", "a/b.txt", true, false, false, "s3://bkt_123/a/b.txt"}, + {"backups/logs/", "backups", "logs/", false, false, true, "s3://backups/logs/"}, + {"backups", "backups", "", false, false, true, "s3://backups"}, + {"", "", "", false, true, false, ""}, + {"-", "", "", false, true, false, ""}, + {"./file", "", "", false, true, false, ""}, + {"/tmp/x", "", "", false, true, false, ""}, + {"s3:///nobucket", "", "", false, true, false, ""}, + } + for _, c := range cases { + r, err := ParseRemote(c.in) + if c.wantErr { + if err == nil { + t.Errorf("ParseRemote(%q): expected error", c.in) + } else if exitcode.Of(err) != exitcode.Usage { + t.Errorf("ParseRemote(%q): exit code %d, want %d", c.in, exitcode.Of(err), exitcode.Usage) + } + continue + } + if err != nil { + t.Errorf("ParseRemote(%q): unexpected error %v", c.in, err) + continue + } + if r.Bucket != c.bucket || r.Key != c.key || r.HadScheme != c.hadScheme || !r.Remote { + t.Errorf("ParseRemote(%q) = %+v", c.in, r) + } + if r.IsDir() != c.wantIsDir { + t.Errorf("ParseRemote(%q).IsDir() = %v, want %v", c.in, r.IsDir(), c.wantIsDir) + } + if r.String() != c.wantString { + t.Errorf("ParseRemote(%q).String() = %q, want %q", c.in, r.String(), c.wantString) + } + } +} + +func TestParseBucketOnlyRejectsKeys(t *testing.T) { + if _, err := ParseBucketOnly("s3://b/key"); err == nil { + t.Fatal("expected error for key in bucket-only argument") + } + r, err := ParseBucketOnly("b") + if err != nil || r.Bucket != "b" { + t.Fatalf("ParseBucketOnly(b) = %+v, %v", r, err) + } +} + +func TestParseTransferArg(t *testing.T) { + if r, _ := ParseTransferArg("-"); !r.Stdio { + t.Error("- should be stdio") + } + if r, _ := ParseTransferArg("./dir/file.txt"); r.Remote || r.Stdio { + t.Error("bare path should be local") + } + if r, _ := ParseTransferArg("backups/file.txt"); r.Remote { + t.Error("bare bucket/key without scheme must stay local in transfer args") + } + r, err := ParseTransferArg("s3://backups/dir/") + if err != nil || !r.Remote || r.Bucket != "backups" || r.Key != "dir/" { + t.Errorf("ParseTransferArg(s3://backups/dir/) = %+v, %v", r, err) + } +} + +func TestObjectRef(t *testing.T) { + if _, err := ObjectRef("s3://b", false); err == nil { + t.Error("bucket-only must be rejected") + } + if _, err := ObjectRef("s3://b/dir/", false); err == nil { + t.Error("trailing slash must be rejected without allowPrefix") + } + if _, err := ObjectRef("s3://b/dir/", true); err != nil { + t.Errorf("trailing slash with allowPrefix: %v", err) + } +} + +func TestKeyHelpers(t *testing.T) { + if JoinKey("", "a") != "a" || JoinKey("p/", "a") != "p/a" || JoinKey("p", "a") != "p/a" { + t.Error("JoinKey") + } + if BaseName("a/b/c.txt") != "c.txt" || BaseName("c.txt") != "c.txt" || BaseName("a/") != "" { + t.Error("BaseName") + } + if NormalizePrefix("") != "" || NormalizePrefix("a") != "a/" || NormalizePrefix("a/") != "a/" { + t.Error("NormalizePrefix") + } +} diff --git a/internal/output/table/utils.go b/internal/output/table/utils.go index d411e1b..4838935 100644 --- a/internal/output/table/utils.go +++ b/internal/output/table/utils.go @@ -62,6 +62,8 @@ func RenderEmptyState(message string) { var PreferredColumnOrder = []string{ "id", "name", + "bucket_name", + "storage_class", "slug", "environment", "description", @@ -80,9 +82,48 @@ var PreferredColumnOrder = []string{ "primary_ipv4", "region", "endpoint", - "storage_class", + "versioning", + "locking", "created_at", "updated_at", + // Object storage (lsh s3) columns. + "type", + "key", + "size", + "last_modified", + "etag", + "content_type", + "version_id", + "op", + "source", + "destination", + "url", + "method", + "expires_at", + "access_key_id", + "scope", + "buckets", + "site", + "saved", + "project", + "prefix", + "expiration_days", + "noncurrent_days", + "abort_mpu_days", + "enabled", + "bucket", + "current_gb", + "consumed_gb", + "estimated_cost", + "currency", + "period_start", + "period_end", + "date", + "tier", + "bytes", + "days", + "avg_bytes", + "action", } // SortColumnsByPreference orders column ids by PreferredColumnOrder, with diff --git a/internal/renderer/format.go b/internal/renderer/format.go index d852c18..d1a1b50 100644 --- a/internal/renderer/format.go +++ b/internal/renderer/format.go @@ -9,8 +9,8 @@ import ( ) // Format is the output format selected for a command's results. table is the -// human-facing default; json, yaml and csv are the machine-readable formats -// meant for automation, piping and (in the future) an MCP server. +// human-facing default; json, yaml, csv and text are the machine-readable +// formats meant for automation, piping and (in the future) an MCP server. type Format string const ( @@ -18,12 +18,15 @@ const ( FormatJSON Format = "json" FormatYAML Format = "yaml" FormatCSV Format = "csv" + // FormatText prints raw values (no quotes, tab-separated rows), like + // `aws --output text`; combined with --query it feeds other tools directly. + FormatText Format = "text" ) // IsStructured reports whether the format produces machine-readable, queryable // output. Only structured formats support the --query (JMESPath) flag. func (f Format) IsStructured() bool { - return f == FormatJSON || f == FormatYAML || f == FormatCSV + return f == FormatJSON || f == FormatYAML || f == FormatCSV || f == FormatText } // ParseFormat normalizes a user-supplied format string. ok is false for an @@ -38,6 +41,8 @@ func ParseFormat(s string) (Format, bool) { return FormatYAML, true case "csv": return FormatCSV, true + case "text": + return FormatText, true default: return "", false } @@ -73,7 +78,7 @@ func ValidateOutputSelection() error { // Validate the format unconditionally — even when --json is set, a typo'd // --output should surface early rather than be silently ignored. if _, ok := ParseFormat(viper.GetString("output")); !ok { - return fmt.Errorf("invalid --output %q (valid values: table, json, yaml, csv)", viper.GetString("output")) + return fmt.Errorf("invalid --output %q (valid values: table, json, yaml, csv, text)", viper.GetString("output")) } query := strings.TrimSpace(viper.GetString("query")) @@ -81,7 +86,7 @@ func ValidateOutputSelection() error { return nil } if !ResolveFormat().IsStructured() { - return fmt.Errorf("--query requires a structured output format; add -o json, -o yaml, or -o csv") + return fmt.Errorf("--query requires a structured output format; add -o json, -o yaml, -o csv, or -o text") } // Compile the JMESPath now so an invalid expression fails fast with a // non-zero exit code instead of only printing to stderr at render time — diff --git a/internal/renderer/main.go b/internal/renderer/main.go index 1b14199..358da6a 100644 --- a/internal/renderer/main.go +++ b/internal/renderer/main.go @@ -17,7 +17,7 @@ type Renderer interface { // GetRenderer returns the renderer for the active output format. // -// Structured formats (json/yaml/csv) are honored everywhere, including when +// Structured formats (json/yaml/csv/text) are honored everywhere, including when // stdout is piped — that is the whole point of automation output. The table // format additionally chooses between the classic ASCII writer (for CI / // LSH_CLASSIC_OUTPUT / non-TTY pipes) and the interactive Bubble Tea view. @@ -29,6 +29,8 @@ func GetRenderer() Renderer { return YAMLRenderer{} case FormatCSV: return CSVRenderer{} + case FormatText: + return TextRenderer{} } // Human-facing table path. @@ -52,6 +54,8 @@ func GetStaticRenderer() Renderer { return YAMLRenderer{} case FormatCSV: return CSVRenderer{} + case FormatText: + return TextRenderer{} } return TableRenderer{} // plain ASCII, no full-screen takeover } diff --git a/internal/renderer/text.go b/internal/renderer/text.go new file mode 100644 index 0000000..b436384 --- /dev/null +++ b/internal/renderer/text.go @@ -0,0 +1,132 @@ +package renderer + +import ( + "fmt" + "io" + "os" + "sort" + "strings" +) + +// TextRenderer implements the "text" output format, modelled on +// `aws --output text`: values are printed raw, without quotes or structure, +// so a `--query secret_access_key` result can be piped straight into another +// tool (e.g. `gh secret set`). +// +// Shapes, after --query has been applied: +// - scalar -> the value on one line, without quotes +// - list of scalars -> one value per line +// - list of objects -> one tab-separated row per object, keys sorted +// - object -> one "keyvalue" line per key, keys sorted +// +// Nested values (objects or arrays inside a cell) are JSON-encoded so a row +// never spans multiple lines. +type TextRenderer struct{} + +func (tr TextRenderer) Render(data []ResponseData) { + if err := renderText(data); err != nil { + reportRenderError(err) + } +} + +func renderText(data []ResponseData) error { + generic, err := structuredData(data) + if err != nil { + return err + } + return writeText(os.Stdout, generic) +} + +// writeText renders the generic (JSON-shaped) value as plain text. +func writeText(w io.Writer, v interface{}) error { + switch t := v.(type) { + case nil: + return nil + case []interface{}: + if len(t) == 0 { + return nil + } + if textHasObjects(t) { + cols := textColumns(t) + for _, it := range t { + if _, err := fmt.Fprintln(w, textRow(it, cols)); err != nil { + return err + } + } + return nil + } + for _, it := range t { + if _, err := fmt.Fprintln(w, scalarToString(it)); err != nil { + return err + } + } + return nil + case map[string]interface{}: + keys := sortedKeys(t) + for _, k := range keys { + if _, err := fmt.Fprintf(w, "%s\t%s\n", k, scalarToString(t[k])); err != nil { + return err + } + } + return nil + default: + _, err := fmt.Fprintln(w, scalarToString(v)) + return err + } +} + +// textHasObjects reports whether any item in the list is an object; a list +// of objects is rendered as rows while a list of scalars is one per line. +func textHasObjects(items []interface{}) bool { + for _, it := range items { + if _, ok := it.(map[string]interface{}); ok { + return true + } + } + return false +} + +// textColumns returns the sorted union of keys across every object in the +// list. Sorted (rather than preference-ordered) keys keep the layout +// predictable for callers that cut columns with awk/cut, matching aws. +func textColumns(items []interface{}) []string { + seen := make(map[string]struct{}) + for _, it := range items { + obj, ok := it.(map[string]interface{}) + if !ok { + continue + } + for k := range obj { + seen[k] = struct{}{} + } + } + cols := make([]string, 0, len(seen)) + for k := range seen { + cols = append(cols, k) + } + sort.Strings(cols) + return cols +} + +// textRow renders one list item as a tab-separated row. A scalar mixed into +// a list of objects is printed as-is. +func textRow(it interface{}, cols []string) string { + obj, ok := it.(map[string]interface{}) + if !ok { + return scalarToString(it) + } + cells := make([]string, len(cols)) + for i, c := range cols { + cells[i] = scalarToString(obj[c]) + } + return strings.Join(cells, "\t") +} + +func sortedKeys(m map[string]interface{}) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/renderer/text_test.go b/internal/renderer/text_test.go new file mode 100644 index 0000000..1904082 --- /dev/null +++ b/internal/renderer/text_test.go @@ -0,0 +1,146 @@ +package renderer + +import ( + "bytes" + "testing" + + outputTable "github.com/latitudesh/lsh/internal/output/table" + "github.com/spf13/viper" +) + +func TestParseFormatText(t *testing.T) { + f, ok := ParseFormat("text") + if !ok || f != FormatText { + t.Fatalf("ParseFormat(text) = %q, %v", f, ok) + } + if !FormatText.IsStructured() { + t.Fatal("text must be structured so --query works") + } +} + +func TestTextRendererScalar(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + viper.Set("query", "[0].name") + + out := captureStdout(t, func() { TextRenderer{}.Render(sampleData()) }) + if out != "alpha\n" { + t.Fatalf("scalar: got %q, want %q", out, "alpha\n") + } +} + +func TestTextRendererScalarList(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + viper.Set("query", "[].id") + + out := captureStdout(t, func() { TextRenderer{}.Render(sampleData()) }) + if out != "srv_1\nsrv_2\n" { + t.Fatalf("scalar list: got %q", out) + } +} + +func TestTextRendererObjectList(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + + out := captureStdout(t, func() { TextRenderer{}.Render(sampleData()) }) + // Keys are sorted: id, name, status. + want := "srv_1\talpha\ton\nsrv_2\tbeta\toff\n" + if out != want { + t.Fatalf("object list: got %q, want %q", out, want) + } +} + +func TestTextRendererObject(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + viper.Set("query", "[0]") + + out := captureStdout(t, func() { TextRenderer{}.Render(sampleData()) }) + want := "id\tsrv_1\nname\talpha\nstatus\ton\n" + if out != want { + t.Fatalf("object: got %q, want %q", out, want) + } +} + +func TestWriteTextNestedValuesAreJSONEncoded(t *testing.T) { + var buf bytes.Buffer + v := []interface{}{ + map[string]interface{}{ + "name": "ci", + "buckets": []interface{}{map[string]interface{}{"bucket_name": "b-1", "permission": "rw"}}, + "size": float64(1024), + "missing": nil, + }, + } + if err := writeText(&buf, v); err != nil { + t.Fatal(err) + } + // Sorted columns: buckets, missing, name, size. + want := `[{"bucket_name":"b-1","permission":"rw"}]` + "\t\tci\t1024\n" + if buf.String() != want { + t.Fatalf("nested: got %q, want %q", buf.String(), want) + } +} + +func TestWriteTextEmptyAndNil(t *testing.T) { + var buf bytes.Buffer + if err := writeText(&buf, []interface{}{}); err != nil { + t.Fatal(err) + } + if err := writeText(&buf, nil); err != nil { + t.Fatal(err) + } + if buf.Len() != 0 { + t.Fatalf("expected no output, got %q", buf.String()) + } +} + +func TestGetStaticRendererText(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + viper.Set("output_explicit", true) + if _, ok := GetStaticRenderer().(TextRenderer); !ok { + t.Fatalf("GetStaticRenderer() = %T, want TextRenderer", GetStaticRenderer()) + } + if _, ok := GetRenderer().(TextRenderer); !ok { + t.Fatalf("GetRenderer() = %T, want TextRenderer", GetRenderer()) + } +} + +// zzKeyRow stands in for the access-key create result. +type zzKeyRow struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` +} + +func (r zzKeyRow) TableRow() outputTable.Row { return outputTable.Row{} } + +// TestQueryOnSingleResultNeedsIndex documents the shape --query operates on: a +// command's rows are always a list, so a bare identifier yields null and the +// documented pipes must index the element. The `access-keys create` examples in +// the README and the automation help topic depend on this. +func TestQueryOnSingleResultNeedsIndex(t *testing.T) { + defer viper.Set("query", "") + row := []ResponseData{zzKeyRow{AccessKeyID: "AK", SecretAccessKey: "s3cr3t"}} + cases := map[string]string{ + "secret_access_key": "", // a JMESPath identifier on a list + "[0].secret_access_key": "s3cr3t\n", // the form the docs must use + "[].secret_access_key": "s3cr3t\n", // projection, also fine + } + for expr, want := range cases { + viper.Set("query", expr) + generic, err := structuredData(row) + if err != nil { + t.Fatalf("%s: %v", expr, err) + } + var buf bytes.Buffer + if err := writeText(&buf, generic); err != nil { + t.Fatalf("%s: %v", expr, err) + } + if buf.String() != want { + t.Errorf("--query %q rendered %q, want %q", expr, buf.String(), want) + } + } +} diff --git a/internal/tui/confirm.go b/internal/tui/confirm.go index 1077029..9ce5f65 100644 --- a/internal/tui/confirm.go +++ b/internal/tui/confirm.go @@ -2,6 +2,7 @@ package tui import ( "fmt" + "os" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" @@ -64,7 +65,9 @@ func (m ConfirmModel) Result() bool { // RunConfirm é uma função helper func RunConfirm(message string) (bool, error) { - p := tea.NewProgram(NewConfirm(message)) + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewConfirm(message), tea.WithOutput(os.Stderr)) m, err := p.Run() if err != nil { return false, err diff --git a/internal/tui/list.go b/internal/tui/list.go index 92dc736..c0d799c 100644 --- a/internal/tui/list.go +++ b/internal/tui/list.go @@ -3,6 +3,7 @@ package tui import ( "fmt" "io" + "os" "github.com/charmbracelet/bubbles/list" tea "github.com/charmbracelet/bubbletea" @@ -134,7 +135,9 @@ func (m ListModel) Choice() string { // RunList é uma função helper para executar a lista func RunList(title string, items []string, descriptions []string) (string, error) { - p := tea.NewProgram(NewList(title, items, descriptions)) + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewList(title, items, descriptions), tea.WithOutput(os.Stderr)) m, err := p.Run() if err != nil { return "", err @@ -142,7 +145,7 @@ func RunList(title string, items []string, descriptions []string) (string, error if model, ok := m.(ListModel); ok { if model.Choice() == "" { - return "", fmt.Errorf("selection cancelled") + return "", ErrCanceled } return model.Choice(), nil } diff --git a/internal/tui/multiselect.go b/internal/tui/multiselect.go new file mode 100644 index 0000000..e6f517f --- /dev/null +++ b/internal/tui/multiselect.go @@ -0,0 +1,145 @@ +package tui + +import ( + "errors" + "fmt" + "os" + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +// MultiSelectModel is a compact checkbox list: ↑/↓ (or k/j) move, space toggles, +// enter confirms, a toggles all, esc/ctrl+c cancels. It mirrors the look of the +// single-select list (TitleStyle, SelectedStyle, HelpStyle) so multi-selection +// feels like the rest of the CLI. +type MultiSelectModel struct { + title string + items []string + descs []string + cursor int + selected map[int]bool + done bool + canceled bool +} + +// NewMultiSelect builds the model. Descriptions are optional (may be shorter +// than items). +func NewMultiSelect(title string, items, descriptions []string) MultiSelectModel { + return MultiSelectModel{ + title: title, + items: items, + descs: descriptions, + selected: make(map[int]bool), + } +} + +func (m MultiSelectModel) Init() tea.Cmd { return nil } + +func (m MultiSelectModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + if key, ok := msg.(tea.KeyMsg); ok { + switch key.String() { + case "ctrl+c", "esc", "q": + m.canceled = true + return m, tea.Quit + case "up", "k": + if m.cursor > 0 { + m.cursor-- + } + case "down", "j": + if m.cursor < len(m.items)-1 { + m.cursor++ + } + case " ", "x": + m.selected[m.cursor] = !m.selected[m.cursor] + case "a": + // Count the entries that are actually on: toggling an item off + // leaves a false value behind, so len(m.selected) overcounts. + on := 0 + for i := range m.items { + if m.selected[i] { + on++ + } + } + all := on < len(m.items) + m.selected = make(map[int]bool, len(m.items)) + if all { + for i := range m.items { + m.selected[i] = true + } + } + case "enter": + m.done = true + return m, tea.Quit + } + } + return m, nil +} + +func (m MultiSelectModel) View() string { + if m.done || m.canceled { + return "" + } + var b strings.Builder + b.WriteString(TitleStyle.Render(m.title) + "\n\n") + for i, it := range m.items { + cursor := " " + if i == m.cursor { + cursor = SelectedStyle.Render("> ") + } + box := "[ ]" + if m.selected[i] { + box = SelectedStyle.Render("[x]") + } + line := fmt.Sprintf("%s%s %s", cursor, box, it) + if i == m.cursor { + line = SelectedStyle.Render(line) + } + b.WriteString(line + "\n") + if i == m.cursor && i < len(m.descs) && m.descs[i] != "" { + b.WriteString(" " + lipgloss.NewStyle().Foreground(MutedColor).Render(m.descs[i]) + "\n") + } + } + b.WriteString("\n" + HelpStyle.Render("↑/↓: move • space: toggle • a: all • enter: confirm • esc: cancel") + "\n") + return b.String() +} + +// Selected returns the chosen indices in ascending order. +func (m MultiSelectModel) Selected() []int { + out := make([]int, 0, len(m.selected)) + for i := 0; i < len(m.items); i++ { + if m.selected[i] { + out = append(out, i) + } + } + return out +} + +// Canceled reports whether the user aborted the selection. +func (m MultiSelectModel) Canceled() bool { return m.canceled } + +// ErrCanceled is returned when the user aborts a prompt (esc/ctrl+c), so +// callers can exit "refused" instead of reporting a usage error. +var ErrCanceled = errors.New("selection cancelled") + +// RunMultiSelect shows the checkbox list and returns the selected indices, or +// ErrCanceled when the user aborts. An empty (but confirmed) selection returns +// no indices and no error. +func RunMultiSelect(title string, items, descriptions []string) ([]int, error) { + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewMultiSelect(title, items, descriptions), tea.WithOutput(os.Stderr)) + m, err := p.Run() + if err != nil { + return nil, err + } + model, ok := m.(MultiSelectModel) + if !ok { + return nil, fmt.Errorf("unexpected model type") + } + if model.Canceled() { + return nil, ErrCanceled + } + return model.Selected(), nil +} diff --git a/internal/tui/multiselect_test.go b/internal/tui/multiselect_test.go new file mode 100644 index 0000000..ee7a5e5 --- /dev/null +++ b/internal/tui/multiselect_test.go @@ -0,0 +1,100 @@ +package tui + +import ( + "reflect" + "testing" + + tea "github.com/charmbracelet/bubbletea" +) + +func key(s string) tea.KeyMsg { + switch s { + case " ": + return tea.KeyMsg{Type: tea.KeySpace} + case "enter": + return tea.KeyMsg{Type: tea.KeyEnter} + case "up": + return tea.KeyMsg{Type: tea.KeyUp} + case "down": + return tea.KeyMsg{Type: tea.KeyDown} + case "esc": + return tea.KeyMsg{Type: tea.KeyEsc} + } + return tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(s)} +} + +func drive(m MultiSelectModel, keys ...string) MultiSelectModel { + for _, k := range keys { + nm, _ := m.Update(key(k)) + m = nm.(MultiSelectModel) + } + return m +} + +func TestMultiSelectToggleAndConfirm(t *testing.T) { + m := NewMultiSelect("Pick", []string{"a", "b", "c"}, nil) + // select index 0, move to index 2 and select it, confirm. + m = drive(m, " ", "down", "down", " ", "enter") + if !m.done || m.Canceled() { + t.Fatalf("enter should confirm, not cancel") + } + if got := m.Selected(); !reflect.DeepEqual(got, []int{0, 2}) { + t.Fatalf("selected = %v, want [0 2]", got) + } +} + +func TestMultiSelectToggleAll(t *testing.T) { + m := NewMultiSelect("Pick", []string{"a", "b"}, nil) + m = drive(m, "a") // select all + if got := m.Selected(); !reflect.DeepEqual(got, []int{0, 1}) { + t.Fatalf("a should select all, got %v", got) + } + m = drive(m, "a") // clear all + if got := m.Selected(); len(got) != 0 { + t.Fatalf("a again should clear, got %v", got) + } +} + +func TestMultiSelectCancel(t *testing.T) { + m := NewMultiSelect("Pick", []string{"a", "b"}, nil) + m = drive(m, " ", "esc") + if !m.Canceled() { + t.Fatalf("esc should cancel") + } +} + +func TestMultiSelectCursorBounds(t *testing.T) { + m := NewMultiSelect("Pick", []string{"a", "b"}, nil) + m = drive(m, "up", "up") // cannot go below 0 + m = drive(m, "down", "down", "down") // cannot go past last + m = drive(m, " ", "enter") + if got := m.Selected(); !reflect.DeepEqual(got, []int{1}) { + t.Fatalf("cursor should clamp to last item, got %v", got) + } +} + +// TestMultiSelectToggleAllAfterDeselect covers the stale-entry bug: toggling an +// item off leaves a false value in the map, so counting entries instead of +// selections made "a" clear the list when it should have selected everything. +func TestMultiSelectToggleAllAfterDeselect(t *testing.T) { + var m tea.Model = NewMultiSelect("pick", []string{"a", "b", "c"}, nil) + // Turn every item on, then off again: three map entries, none selected. + for _, k := range []string{" ", "down", " ", "down", " "} { + m, _ = m.Update(key(k)) + } + for _, k := range []string{" ", "up", " ", "up", " "} { + m, _ = m.Update(key(k)) + } + if got := m.(MultiSelectModel).Selected(); len(got) != 0 { + t.Fatalf("precondition failed: %v still selected", got) + } + m, _ = m.Update(key("a")) + if got := m.(MultiSelectModel).Selected(); !reflect.DeepEqual(got, []int{0, 1, 2}) { + t.Errorf("'a' selected %v, want every item", got) + } + // Pressing it again with everything on clears the selection. + m, _ = m.Update(key("a")) + if got := m.(MultiSelectModel).Selected(); len(got) != 0 { + t.Errorf("'a' on a full selection left %v", got) + } +} diff --git a/internal/tui/numberinput.go b/internal/tui/numberinput.go index 3c5688d..62fccaa 100644 --- a/internal/tui/numberinput.go +++ b/internal/tui/numberinput.go @@ -2,6 +2,7 @@ package tui import ( "fmt" + "os" "strconv" "github.com/charmbracelet/bubbles/textinput" @@ -99,7 +100,9 @@ func (m NumberInputModel) Submitted() bool { // RunNumberInput is a helper function to run the number input func RunNumberInput(label, placeholder string) (int64, error) { - p := tea.NewProgram(NewNumberInput(label, placeholder)) + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewNumberInput(label, placeholder), tea.WithOutput(os.Stderr)) m, err := p.Run() if err != nil { return 0, err diff --git a/internal/tui/textinput.go b/internal/tui/textinput.go index 68c3b7d..73c6efe 100644 --- a/internal/tui/textinput.go +++ b/internal/tui/textinput.go @@ -2,6 +2,7 @@ package tui import ( "fmt" + "os" "github.com/charmbracelet/bubbles/textinput" tea "github.com/charmbracelet/bubbletea" @@ -72,7 +73,9 @@ func (m TextInputModel) Value() string { // RunTextInput is a helper function to run the input func RunTextInput(label, placeholder string) (string, error) { - p := tea.NewProgram(NewTextInput(label, placeholder)) + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewTextInput(label, placeholder), tea.WithOutput(os.Stderr)) m, err := p.Run() if err != nil { return "", err diff --git a/internal/wait/server.go b/internal/wait/server.go index 7bc5b94..eef2255 100644 --- a/internal/wait/server.go +++ b/internal/wait/server.go @@ -32,12 +32,12 @@ func ForServerState( ctx context.Context, client *sdk.Latitudesh, serverID string, - want, fail []components.ServerDataStatus, + want, fail []ServerStatus, requireTransition bool, o Options, - onStatus func(components.ServerDataStatus), + onStatus func(ServerStatus), opts ...operations.Option, -) (components.ServerDataStatus, error) { +) (ServerStatus, error) { if o.Timeout > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, o.Timeout) @@ -45,7 +45,7 @@ func ForServerState( } var ( - last components.ServerDataStatus + last ServerStatus lastErr error seenTransition bool ) @@ -95,8 +95,8 @@ func ForServerState( // operation acting on a server already in a target state (e.g. a reinstall on a // powered-on server) does not return before it actually begins. func decideServerState( - status components.ServerDataStatus, - want, fail []components.ServerDataStatus, + status ServerStatus, + want, fail []ServerStatus, requireTransition, seenTransition bool, ) (done bool, transitioned bool, err error) { inWant := containsStatus(want, status) @@ -131,14 +131,18 @@ func isTerminalAPIError(err error) bool { // serverStatus extracts the status from a GetServer response, tolerating any // nil link in the data → attributes → status chain. -func serverStatus(resp *operations.GetServerResponse) *components.ServerDataStatus { +func serverStatus(resp *operations.GetServerResponse) *ServerStatus { if resp == nil || resp.Server == nil || resp.Server.Data == nil || resp.Server.Data.Attributes == nil { return nil } - return resp.Server.Data.Attributes.Status + if resp.Server.Data.Attributes.Status == nil { + return nil + } + st := ServerStatus(*resp.Server.Data.Attributes.Status) + return &st } -func containsStatus(set []components.ServerDataStatus, s components.ServerDataStatus) bool { +func containsStatus(set []ServerStatus, s ServerStatus) bool { for _, v := range set { if v == s { return true diff --git a/internal/wait/status.go b/internal/wait/status.go new file mode 100644 index 0000000..888e5ca --- /dev/null +++ b/internal/wait/status.go @@ -0,0 +1,29 @@ +package wait + +// ServerStatus is the `status` attribute of a server as reported by the API. +// The SDK stopped generating a dedicated enum for it (v1.19.x exposes a plain +// string), so the known values live here to keep callers type-safe. +type ServerStatus string + +const ( + ServerStatusOn ServerStatus = "on" + ServerStatusOff ServerStatus = "off" + ServerStatusUnknown ServerStatus = "unknown" + ServerStatusDiskErasing ServerStatus = "disk_erasing" + ServerStatusDeploying ServerStatus = "deploying" + ServerStatusFailedDeployment ServerStatus = "failed_deployment" + ServerStatusRescueMode ServerStatus = "rescue_mode" +) + +// VirtualMachineStatus is the `status` attribute of a virtual machine as +// reported by the API (plain string in the SDK since v1.19.x). +type VirtualMachineStatus string + +const ( + VirtualMachineStatusRunning VirtualMachineStatus = "Running" + VirtualMachineStatusConfiguringNetwork VirtualMachineStatus = "Configuring network" + VirtualMachineStatusStarting VirtualMachineStatus = "Starting" + VirtualMachineStatusScheduling VirtualMachineStatus = "Scheduling" + VirtualMachineStatusScheduled VirtualMachineStatus = "Scheduled" + VirtualMachineStatusDestroying VirtualMachineStatus = "Destroying" +) diff --git a/internal/wait/vm.go b/internal/wait/vm.go index 49f9134..f907ecd 100644 --- a/internal/wait/vm.go +++ b/internal/wait/vm.go @@ -6,7 +6,6 @@ import ( "fmt" sdk "github.com/latitudesh/latitudesh-go-sdk" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" ) @@ -22,10 +21,10 @@ func ForVirtualMachineState( ctx context.Context, client *sdk.Latitudesh, vmID string, - want []components.VirtualMachineAttributesStatus, + want []VirtualMachineStatus, o Options, opts ...operations.Option, -) (components.VirtualMachineAttributesStatus, error) { +) (VirtualMachineStatus, error) { if o.Timeout > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, o.Timeout) @@ -33,12 +32,12 @@ func ForVirtualMachineState( } var ( - last components.VirtualMachineAttributesStatus + last VirtualMachineStatus lastErr error ) err := Poll(ctx, DefaultBackoff(), func(ctx context.Context) (bool, error) { - resp, err := client.VirtualMachines.Get(ctx, vmID, opts...) + resp, err := client.VirtualMachines.Get(ctx, vmID, nil, opts...) if err != nil { if isTerminalAPIError(err) { return false, err @@ -63,14 +62,18 @@ func ForVirtualMachineState( // virtualMachineStatus extracts the status from a ShowVirtualMachine response, // tolerating any nil link in the data → attributes → status chain. -func virtualMachineStatus(resp *operations.ShowVirtualMachineResponse) *components.VirtualMachineAttributesStatus { +func virtualMachineStatus(resp *operations.ShowVirtualMachineResponse) *VirtualMachineStatus { if resp == nil || resp.VirtualMachine == nil || resp.VirtualMachine.Data == nil || resp.VirtualMachine.Data.Attributes == nil { return nil } - return resp.VirtualMachine.Data.Attributes.Status + if resp.VirtualMachine.Data.Attributes.Status == nil { + return nil + } + st := VirtualMachineStatus(*resp.VirtualMachine.Data.Attributes.Status) + return &st } -func containsVMStatus(set []components.VirtualMachineAttributesStatus, s components.VirtualMachineAttributesStatus) bool { +func containsVMStatus(set []VirtualMachineStatus, s VirtualMachineStatus) bool { for _, v := range set { if v == s { return true diff --git a/internal/wait/wait_test.go b/internal/wait/wait_test.go index f3bdb7b..b2c52fb 100644 --- a/internal/wait/wait_test.go +++ b/internal/wait/wait_test.go @@ -108,7 +108,7 @@ func TestServerStatusNilSafe(t *testing.T) { if serverStatus(&operations.GetServerResponse{}) != nil { t.Error("serverStatus with nil Server should be nil") } - on := components.ServerDataStatusOn + on := string(ServerStatusOn) resp := &operations.GetServerResponse{ Server: &components.Server{ Data: &components.ServerData{ @@ -117,37 +117,37 @@ func TestServerStatusNilSafe(t *testing.T) { }, } got := serverStatus(resp) - if got == nil || *got != components.ServerDataStatusOn { + if got == nil || *got != ServerStatusOn { t.Errorf("serverStatus = %v, want on", got) } } func TestContainsStatus(t *testing.T) { - set := []components.ServerDataStatus{ - components.ServerDataStatusOn, - components.ServerDataStatusDeploying, + set := []ServerStatus{ + ServerStatusOn, + ServerStatusDeploying, } - if !containsStatus(set, components.ServerDataStatusOn) { + if !containsStatus(set, ServerStatusOn) { t.Error("expected on to be in set") } - if containsStatus(set, components.ServerDataStatusOff) { + if containsStatus(set, ServerStatusOff) { t.Error("did not expect off to be in set") } } func TestDecideServerState(t *testing.T) { - want := []components.ServerDataStatus{components.ServerDataStatusOn, components.ServerDataStatusOff} - fail := []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} + want := []ServerStatus{ServerStatusOn, ServerStatusOff} + fail := []ServerStatus{ServerStatusFailedDeployment} t.Run("want hit without requireTransition is done", func(t *testing.T) { - done, _, err := decideServerState(components.ServerDataStatusOn, want, fail, false, false) + done, _, err := decideServerState(ServerStatusOn, want, fail, false, false) if err != nil || !done { t.Fatalf("done=%v err=%v, want done=true err=nil", done, err) } }) t.Run("want hit is gated until a transition is seen", func(t *testing.T) { - done, transitioned, err := decideServerState(components.ServerDataStatusOn, want, fail, true, false) + done, transitioned, err := decideServerState(ServerStatusOn, want, fail, true, false) if err != nil || done { t.Fatalf("done=%v err=%v, want done=false err=nil (gated)", done, err) } @@ -157,18 +157,18 @@ func TestDecideServerState(t *testing.T) { }) t.Run("transition state flips seenTransition then want succeeds", func(t *testing.T) { - _, transitioned, _ := decideServerState(components.ServerDataStatusDeploying, want, fail, true, false) + _, transitioned, _ := decideServerState(ServerStatusDeploying, want, fail, true, false) if !transitioned { t.Fatal("deploying should set seenTransition") } - done, _, err := decideServerState(components.ServerDataStatusOn, want, fail, true, transitioned) + done, _, err := decideServerState(ServerStatusOn, want, fail, true, transitioned) if err != nil || !done { t.Fatalf("done=%v err=%v, want done=true after transition", done, err) } }) t.Run("fail state reported immediately even without transition (H1 regression)", func(t *testing.T) { - done, _, err := decideServerState(components.ServerDataStatusFailedDeployment, want, fail, true, false) + done, _, err := decideServerState(ServerStatusFailedDeployment, want, fail, true, false) if done { t.Error("fail state must not be 'done'") } diff --git a/main.go b/main.go index f5506e7..4030f82 100644 --- a/main.go +++ b/main.go @@ -4,13 +4,15 @@ import ( "os" "github.com/latitudesh/lsh/cmd" + "github.com/latitudesh/lsh/internal/exitcode" ) func main() { // Propagate command failures as a non-zero exit code. Cobra already prints // the error (commands set SilenceUsage), so we only need to set the status - // — scripts, CI and AI agents rely on it. + // — scripts, CI and AI agents rely on it. Commands that attach an explicit + // exit code (see internal/exitcode) get it; everything else exits with 1. if _, err := cmd.Execute(); err != nil { - os.Exit(1) + os.Exit(exitcode.Of(err)) } } From bb79a499ed56d43cb7a08a75ae78a2327c13287e Mon Sep 17 00:00:00 2001 From: Lanusse Morais Date: Wed, 9 Sep 2026 17:20:29 -0300 Subject: [PATCH 2/3] fix(s3): validate explicit access keys and serialize profile writes --- cmd/s3/s3.go | 19 +++- cmd/s3/s3_test.go | 23 +++++ internal/config/lock.go | 45 ++++++++++ internal/config/lock_test.go | 96 ++++++++++++++++++++ internal/config/lock_unix.go | 27 ++++++ internal/config/lock_windows.go | 31 +++++++ internal/objectstorage/creds.go | 129 ++++++++++++++++++++------- internal/objectstorage/creds_test.go | 45 ++++++++++ 8 files changed, 383 insertions(+), 32 deletions(-) create mode 100644 cmd/s3/s3_test.go create mode 100644 internal/config/lock.go create mode 100644 internal/config/lock_test.go create mode 100644 internal/config/lock_unix.go create mode 100644 internal/config/lock_windows.go diff --git a/cmd/s3/s3.go b/cmd/s3/s3.go index ca58dcd..b3889d0 100644 --- a/cmd/s3/s3.go +++ b/cmd/s3/s3.go @@ -303,14 +303,29 @@ func openBucket(ctx context.Context, cmd *cobra.Command, token string, write boo // A high_performance key is bound to one site, but the SDK model drops the // bucket's site, so without this the site check in keyMatchesBucket is a // no-op on the data plane and a key from another site can be selected. - if b.StorageClass == objectstorage.ClassHighPerformance && b.Site == "" && !b.EndpointOverride { + // The site is only a selection constraint, so the lookup is skipped when + // the credential does not come from the profile; when it does, a failed + // lookup has to fail the command — continuing with an empty site turns the + // constraint into a wildcard and lets a key from another site win. + if b.StorageClass == objectstorage.ClassHighPerformance && b.Site == "" && !b.EndpointOverride && usesSavedCredential() { if fillErr := newResolver(cmd).FillSite(ctx, b); fillErr != nil { - lsh.LogDebugf("[s3] could not determine the site of %s: %v", b.Display(), fillErr) + return nil, exitcode.Errorf(exitcode.Of(fillErr), + "could not determine which site bucket %s is in, and a high_performance access key is only valid in its own site: %v\n retry, or name the key explicitly with --access-key ", + b.Display(), fillErr) } } return openResolved(cmd, b, write) } +// usesSavedCredential reports whether the credential will come from the active +// profile, which is the only case where the bucket's site changes the outcome. +// LSH_S3_* credentials bypass the profile entirely; an explicit --access-key +// does not, because the named key is still checked against the bucket's site. +func usesSavedCredential() bool { + _, fromEnv, _ := objectstorage.EnvCredential() + return !fromEnv +} + // openResolved is openBucket for an already-resolved bucket. func openResolved(cmd *cobra.Command, b *objectstorage.Bucket, write bool) (*Session, error) { if err := b.Validate(); err != nil { diff --git a/cmd/s3/s3_test.go b/cmd/s3/s3_test.go new file mode 100644 index 0000000..f7754d6 --- /dev/null +++ b/cmd/s3/s3_test.go @@ -0,0 +1,23 @@ +package s3 + +import ( + "testing" + + "github.com/latitudesh/lsh/internal/objectstorage" +) + +// TestUsesSavedCredential guards the condition that decides whether a bucket's +// site has to be resolved: it only matters when the credential comes from the +// profile, since LSH_S3_* bypasses selection entirely. +func TestUsesSavedCredential(t *testing.T) { + t.Setenv(objectstorage.EnvAccessKeyID, "") + t.Setenv(objectstorage.EnvSecretAccessKey, "") + if !usesSavedCredential() { + t.Error("without environment credentials the profile is used") + } + t.Setenv(objectstorage.EnvAccessKeyID, "AK") + t.Setenv(objectstorage.EnvSecretAccessKey, "SK") + if usesSavedCredential() { + t.Error("environment credentials bypass the profile, so the site is irrelevant") + } +} diff --git a/internal/config/lock.go b/internal/config/lock.go new file mode 100644 index 0000000..6efeeda --- /dev/null +++ b/internal/config/lock.go @@ -0,0 +1,45 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" +) + +// Update applies mutate to the configuration file as one transaction: the file +// is locked, loaded, mutated and written back before the lock is released. +// +// Load/Save on their own are not enough. Save writes a temp file and renames it +// atomically, so a single write can never be torn — but two processes that both +// load, mutate and save (two `lsh s3 access-keys create --save` runs, say) each +// hold a full snapshot, and the second rename silently drops the first one's +// change. For an S3 access key that means losing a secret the API only ever +// returns once, while the credential stays live on the server. Serialising the +// whole read-modify-write is the only way to avoid it. +// +// The lock is advisory and per-file; readers do not take it, so a concurrent +// reader may still observe the pre-update file (which is consistent, thanks to +// the atomic rename). +func Update(mutate func(*File) error) error { + path, err := Path() + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), dirPerm); err != nil { + return fmt.Errorf("config: mkdir: %w", err) + } + unlock, err := lockConfig(path + ".lock") + if err != nil { + return err + } + defer unlock() + + f, err := Load() + if err != nil { + return err + } + if err := mutate(f); err != nil { + return err + } + return Save(f) +} diff --git a/internal/config/lock_test.go b/internal/config/lock_test.go new file mode 100644 index 0000000..ecec146 --- /dev/null +++ b/internal/config/lock_test.go @@ -0,0 +1,96 @@ +package config + +import ( + "path/filepath" + "sync" + "testing" +) + +// TestUpdateSerializesConcurrentWriters covers the transaction guarantee: two +// processes saving different keys at the same time must both survive. Without +// the lock each writer holds its own snapshot and the last rename drops the +// other's entry — losing a secret the API only returns once. +func TestUpdateSerializesConcurrentWriters(t *testing.T) { + t.Setenv("LSH_CONFIG_PATH", filepath.Join(t.TempDir(), "config.json")) + if err := Update(func(f *File) error { + f.SetProfile("p", Profile{Authorization: "token"}) + return nil + }); err != nil { + t.Fatal(err) + } + + const writers = 8 + var wg sync.WaitGroup + errs := make(chan error, writers) + for i := 0; i < writers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + errs <- Update(func(f *File) error { + _, p, err := f.Resolve("p") + if err != nil { + return err + } + p.SetObjectStorageKey(string(rune('a'+i)), StoredAccessKey{AccessKeyID: "AK", Scope: ScopeFullAccess}) + f.SetProfile("p", p) + return nil + }) + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("Update: %v", err) + } + } + + f, err := Load() + if err != nil { + t.Fatal(err) + } + _, p, err := f.Resolve("p") + if err != nil { + t.Fatal(err) + } + keys := p.ObjectStorageKeys() + if len(keys) != writers { + t.Errorf("%d keys survived, want %d — a concurrent save overwrote another: %v", len(keys), writers, keys) + } + if p.Authorization != "token" { + t.Errorf("the rest of the profile must be preserved, got %+v", p) + } +} + +// TestUpdateMutateErrorLeavesFileAlone keeps a failed transaction from writing. +func TestUpdateMutateErrorLeavesFileAlone(t *testing.T) { + t.Setenv("LSH_CONFIG_PATH", filepath.Join(t.TempDir(), "config.json")) + if err := Update(func(f *File) error { + f.SetProfile("p", Profile{Authorization: "first"}) + return nil + }); err != nil { + t.Fatal(err) + } + sentinel := errNoWrite{} + if err := Update(func(f *File) error { + f.SetProfile("p", Profile{Authorization: "second"}) + return sentinel + }); err != sentinel { + t.Fatalf("Update returned %v, want the mutate error", err) + } + f, err := Load() + if err != nil { + t.Fatal(err) + } + _, p, err := f.Resolve("p") + if err != nil { + t.Fatal(err) + } + if p.Authorization != "first" { + t.Errorf("a failed transaction must not be written, got %q", p.Authorization) + } +} + +type errNoWrite struct{} + +func (errNoWrite) Error() string { return "no write" } diff --git a/internal/config/lock_unix.go b/internal/config/lock_unix.go new file mode 100644 index 0000000..a8c1a9d --- /dev/null +++ b/internal/config/lock_unix.go @@ -0,0 +1,27 @@ +//go:build !windows + +package config + +import ( + "fmt" + "os" + "syscall" +) + +// lockConfig takes an exclusive advisory lock on path, blocking until it is +// available. The kernel releases flock locks when the file descriptor closes, +// so a process that dies mid-update never leaves the lock behind. +func lockConfig(path string) (func(), error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, filePerm) + if err != nil { + return nil, fmt.Errorf("config: open lock: %w", err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + f.Close() + return nil, fmt.Errorf("config: lock: %w", err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + }, nil +} diff --git a/internal/config/lock_windows.go b/internal/config/lock_windows.go new file mode 100644 index 0000000..3035b19 --- /dev/null +++ b/internal/config/lock_windows.go @@ -0,0 +1,31 @@ +//go:build windows + +package config + +import ( + "fmt" + "os" + + "golang.org/x/sys/windows" +) + +// lockConfig takes an exclusive lock on path, blocking until it is available. +// Windows releases the range lock when the handle closes, matching the flock +// behaviour used on the other platforms. +func lockConfig(path string) (func(), error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, filePerm) + if err != nil { + return nil, fmt.Errorf("config: open lock: %w", err) + } + handle := windows.Handle(f.Fd()) + var overlapped windows.Overlapped + if err := windows.LockFileEx(handle, windows.LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &overlapped); err != nil { + f.Close() + return nil, fmt.Errorf("config: lock: %w", err) + } + return func() { + var release windows.Overlapped + _ = windows.UnlockFileEx(handle, 0, 1, 0, &release) + _ = f.Close() + }, nil +} diff --git a/internal/objectstorage/creds.go b/internal/objectstorage/creds.go index 8511652..cea0622 100644 --- a/internal/objectstorage/creds.go +++ b/internal/objectstorage/creds.go @@ -162,6 +162,13 @@ func ResolveCredential(b *Bucket, o CredentialOptions) (Credential, error) { } c := fromStored(o.AccessKeyName, k, profileName) if b != nil && !b.EndpointOverride && b.ID != "" { + // Covers() is true for every bucket ID on a fullaccess key, so + // without this the class/site/project constraints that automatic + // selection enforces would never be checked for an explicit key: + // the request would go out and fail at the backend instead. + if err := explainIncompatibleKey(o.AccessKeyName, k, b); err != nil { + return Credential{}, err + } if k.Scope != config.ScopeUnknown && !k.Covers(b.ID, false) { return Credential{}, exitcode.Errorf(exitcode.Permission, "saved key %q does not cover bucket %s", o.AccessKeyName, b.Display()) } @@ -193,6 +200,30 @@ func fromStored(name string, k config.StoredAccessKey, profile string) Credentia } } +// explainIncompatibleKey reports why a saved key cannot serve b, or nil when +// nothing rules it out. It applies the constraints keyMatchesBucket uses for +// automatic selection — with the same "empty means unknown, so do not judge" +// rule, which keeps imported keys with partial metadata usable — but names the +// mismatch instead of silently skipping the key, because here the user picked +// it explicitly. +func explainIncompatibleKey(name string, k config.StoredAccessKey, b *Bucket) error { + switch { + case k.StorageClass != "" && b.StorageClass != "" && k.StorageClass != b.StorageClass: + return exitcode.Errorf(exitcode.Usage, + "saved key %q is a %s key and bucket %s is %s; the two backends do not share credentials — run 'lsh s3 access-keys list --saved' to pick another, or drop --access-key to let the CLI choose", + name, k.StorageClass, b.Display(), b.StorageClass) + case k.Site != "" && b.Site != "" && !strings.EqualFold(k.Site, b.Site): + return exitcode.Errorf(exitcode.Usage, + "saved key %q belongs to site %s and bucket %s is in %s; a %s key only works in its own site", + name, strings.ToUpper(k.Site), b.Display(), strings.ToUpper(b.Site), ClassHighPerformance) + case k.ProjectID != "" && b.ProjectID != "" && k.ProjectID != b.ProjectID: + return exitcode.Errorf(exitcode.Usage, + "saved key %q belongs to project %s and bucket %s to project %s", + name, k.ProjectID, b.Display(), b.ProjectID) + } + return nil +} + // SelectKey applies the automatic selection rules over the saved keys: // same storage class, compatible site and project, covering the bucket with // the needed permission. Ranking: limited rw > limited readonly (reads only) @@ -277,53 +308,91 @@ func NoCredentialError(b *Bucket, profile string, write bool, keys map[string]co } // SaveKey stores k under name in the active profile and persists the file. +// +// The read-modify-write runs inside config.Update so two processes saving keys +// at the same time cannot drop each other's entry: a lost entry would strand a +// live credential whose secret the API never returns again. func SaveKey(profileOverride, name string, k config.StoredAccessKey) (string, error) { - f, profileName, p, err := ActiveProfile(profileOverride) + var profileName string + err := config.Update(func(f *config.File) error { + var p config.Profile + var resolveErr error + profileName, p, resolveErr = resolveProfileIn(f, profileOverride) + if resolveErr != nil { + return resolveErr + } + p.SetObjectStorageKey(name, k) + f.SetProfile(profileName, p) + return nil + }) + return profileName, err +} + +// resolveProfileIn resolves the active profile inside an already-loaded file, +// mapping the same errors ActiveProfile reports. It exists so a config.Update +// transaction works on the file it locked instead of loading a second copy. +func resolveProfileIn(f *config.File, override string) (string, config.Profile, error) { + name, p, err := f.Resolve(override) if err != nil { - return "", err - } - p.SetObjectStorageKey(name, k) - f.SetProfile(profileName, p) - if err := config.Save(f); err != nil { - return profileName, err + if errors.Is(err, config.ErrProfileNotFound) { + if override != "" { + return name, p, exitcode.Errorf(exitcode.Usage, "profile %q not found — run 'lsh profile list'", override) + } + return "", config.Profile{}, exitcode.Errorf(exitcode.Credentials, "no active profile — run 'lsh login' (or use LSH_S3_ACCESS_KEY_ID/LSH_S3_SECRET_ACCESS_KEY)") + } + return name, p, err } - return profileName, nil + return name, p, nil } // ForgetKey removes a saved key by name from the active profile. func ForgetKey(profileOverride, name string) (string, bool, error) { - f, profileName, p, err := ActiveProfile(profileOverride) - if err != nil { - return "", false, err - } - removed := p.RemoveObjectStorageKey(name) - if removed { - f.SetProfile(profileName, p) - if err := config.Save(f); err != nil { - return profileName, false, err + var profileName string + var removed bool + err := config.Update(func(f *config.File) error { + var p config.Profile + var resolveErr error + profileName, p, resolveErr = resolveProfileIn(f, profileOverride) + if resolveErr != nil { + return resolveErr } + removed = p.RemoveObjectStorageKey(name) + if removed { + f.SetProfile(profileName, p) + } + return nil + }) + if err != nil { + return profileName, false, err } return profileName, removed, nil } // ForgetKeyByID removes every saved key with the given access key ID. func ForgetKeyByID(profileOverride, accessKeyID string) (string, []string, error) { - f, profileName, p, err := ActiveProfile(profileOverride) - if err != nil { - return "", nil, err - } + var profileName string var removed []string - for name, k := range p.ObjectStorageKeys() { - if k.AccessKeyID == accessKeyID { - p.RemoveObjectStorageKey(name) - removed = append(removed, name) + err := config.Update(func(f *config.File) error { + var p config.Profile + var resolveErr error + profileName, p, resolveErr = resolveProfileIn(f, profileOverride) + if resolveErr != nil { + return resolveErr } - } - if len(removed) > 0 { - f.SetProfile(profileName, p) - if err := config.Save(f); err != nil { - return profileName, nil, err + removed = nil + for name, k := range p.ObjectStorageKeys() { + if k.AccessKeyID == accessKeyID { + p.RemoveObjectStorageKey(name) + removed = append(removed, name) + } } + if len(removed) > 0 { + f.SetProfile(profileName, p) + } + return nil + }) + if err != nil { + return profileName, nil, err } sort.Strings(removed) return profileName, removed, nil diff --git a/internal/objectstorage/creds_test.go b/internal/objectstorage/creds_test.go index ad6b496..b67ce09 100644 --- a/internal/objectstorage/creds_test.go +++ b/internal/objectstorage/creds_test.go @@ -122,3 +122,48 @@ func TestResolveCredentialEndpointOverrideRequiresEnv(t *testing.T) { t.Errorf("error should tell the user which variables to set: %v", err) } } + +// TestExplicitKeyChecksCompatibility covers the --access-key path: Covers() is +// true for every bucket on a fullaccess key, so without an explicit +// compatibility check a key from another backend, site or project would be +// sent to the bucket and fail at the server instead of locally. +func TestExplicitKeyChecksCompatibility(t *testing.T) { + hp := &Bucket{ID: "bkt_1", Name: "fast", StorageClass: ClassHighPerformance, Site: "TYO4", ProjectID: "proj_1"} + full := func(class, site, project string) config.StoredAccessKey { + return config.StoredAccessKey{AccessKeyID: "AK", StorageClass: class, Site: site, ProjectID: project, Scope: config.ScopeFullAccess} + } + cases := []struct { + name string + key config.StoredAccessKey + want string + }{ + {"other storage class", full(ClassStandard, "TYO4", "proj_1"), "do not share credentials"}, + {"other site", full(ClassHighPerformance, "DAL", "proj_1"), "only works in its own site"}, + {"other project", full(ClassHighPerformance, "TYO4", "proj_2"), "belongs to project"}, + } + for _, c := range cases { + err := explainIncompatibleKey("ci-key", c.key, hp) + if err == nil { + t.Errorf("%s: expected the selection to be rejected", c.name) + continue + } + if code := exitcode.Of(err); code != exitcode.Usage { + t.Errorf("%s: exit code = %d, want %d", c.name, code, exitcode.Usage) + } + if !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: error %q should explain the mismatch (%q)", c.name, err, c.want) + } + } + + // A compatible key passes, and partial metadata is not judged: an imported + // key with no class/site/project stays usable. + for _, ok := range []config.StoredAccessKey{ + full(ClassHighPerformance, "tyo4", "proj_1"), + full("", "", ""), + {AccessKeyID: "AK", Scope: config.ScopeUnknown}, + } { + if err := explainIncompatibleKey("ci-key", ok, hp); err != nil { + t.Errorf("key %+v must be accepted, got %v", ok, err) + } + } +} From f7b161b30c60cb73e8ca13da2a4b92a6ae02d04d Mon Sep 17 00:00:00 2001 From: Lanusse Morais Date: Fri, 11 Sep 2026 14:02:22 -0300 Subject: [PATCH 3/3] refactor(s3): lsh verbs as canonical names, grouped help and s3 help topic --- README.md | 18 +++--- cli/cli.go | 1 + cli/help_topics.go | 65 +++++++++++++++++++-- cmd/build_s3.go | 6 +- cmd/s3/accesskeys.go | 9 ++- cmd/s3/accesskeys_create.go | 2 +- cmd/s3/accesskeys_list.go | 2 +- cmd/s3/configure.go | 14 +++-- cmd/s3/cp.go | 27 ++++----- cmd/s3/lifecycle.go | 3 +- cmd/s3/ls.go | 25 ++++---- cmd/s3/mb.go | 27 ++++----- cmd/s3/metrics.go | 5 +- cmd/s3/mv.go | 19 +++--- cmd/s3/presign.go | 5 +- cmd/s3/rb.go | 17 +++--- cmd/s3/rm.go | 23 ++++---- cmd/s3/rm_test.go | 4 +- cmd/s3/s3.go | 94 ++++++++++++++++++++++-------- cmd/s3/stat.go | 19 +++--- cmd/s3/sync.go | 9 +-- cmd/s3/transfer.go | 2 +- cmd/s3/transfer_test.go | 2 +- cmd/s3/usage.go | 5 +- internal/objectstorage/errors.go | 2 +- internal/objectstorage/resolver.go | 2 +- internal/objectstorage/uri.go | 4 +- 27 files changed, 269 insertions(+), 142 deletions(-) diff --git a/README.md b/README.md index 368420d..362809a 100755 --- a/README.md +++ b/README.md @@ -144,17 +144,17 @@ endpoint and signing region never have to be configured by hand. `` accepts the display name, the `bkt_` ID or the backend bucket name. If the same display name exists in more than one place, the command lists the candidates and you narrow it with `--project`, `-c`/`--storage-class`, `--site` -(e.g. `lsh s3 stat s3://backups -c high_performance --site TYO4`), or the `bkt_` ID. +(e.g. `lsh s3 get s3://backups -c high_performance --site TYO4`), or the `bkt_` ID. Create a bucket, upload, list and delete (in a terminal, `mb` offers to create an S3 access key and saves it to your profile): ```bash -lsh s3 mb s3://backups --region DAL --project -lsh s3 cp ./dump.sql s3://backups/2026/09/ -lsh s3 ls s3://backups/2026/09/ --human-readable --summarize -lsh s3 cp s3://backups/2026/09/dump.sql ./restore/ -lsh s3 rm s3://backups/2026/09/dump.sql +lsh s3 create-bucket s3://backups --region DAL --project +lsh s3 copy ./dump.sql s3://backups/2026/09/ +lsh s3 list s3://backups/2026/09/ --human-readable --summarize +lsh s3 copy s3://backups/2026/09/dump.sql ./restore/ +lsh s3 delete s3://backups/2026/09/dump.sql ``` Give an application or CI job its own scoped access key (the secret is shown @@ -185,9 +185,9 @@ Clean up safely (`--dry-run` only reads; multi-object deletes ask for confirmation in a terminal and require `--yes` in CI): ```bash -lsh s3 rm s3://logs/tmp/ --recursive --dry-run -lsh s3 rm s3://logs/tmp/ --recursive --yes -lsh s3 rb s3://logs --force --yes +lsh s3 delete s3://logs/tmp/ --recursive --dry-run +lsh s3 delete s3://logs/tmp/ --recursive --yes +lsh s3 delete-bucket s3://logs --force --yes ``` In CI, object commands authenticate with an S3 access key from the environment diff --git a/cli/cli.go b/cli/cli.go index cf72234..5a973ab 100755 --- a/cli/cli.go +++ b/cli/cli.go @@ -79,6 +79,7 @@ func MakeRootCmd(rootCmd *cobra.Command) (*cobra.Command, error) { rootCmd.AddCommand(makeHelpAutomationCmd()) rootCmd.AddCommand(makeHelpOutputFormatsCmd()) rootCmd.AddCommand(makeHelpExitCodesCmd()) + rootCmd.AddCommand(makeHelpS3Cmd()) // Re-resolve the active profile once flags have been parsed so that // `--profile ` overrides LSH_PROFILE / default_profile for the diff --git a/cli/help_topics.go b/cli/help_topics.go index c323324..62071de 100644 --- a/cli/help_topics.go +++ b/cli/help_topics.go @@ -133,7 +133,7 @@ projects first and retrying: Confirmations (--yes / --no-input) -Destructive object storage commands (lsh s3 rm --recursive, rb --force, +Destructive object storage commands (lsh s3 delete --recursive, rb --force, access-keys delete, lifecycle delete --all) ask for confirmation when run in a terminal. The contract for automation is: @@ -148,7 +148,7 @@ exits 0. See 'lsh help exit-codes' for the full table. Object storage credentials -Object commands (lsh s3 cp/ls/rm/stat/presign) authenticate with an S3 +Object commands (lsh s3 copy/ls/rm/stat/presign) authenticate with an S3 access key, not with the API token. In CI, export one of the following: LSH_S3_ACCESS_KEY_ID S3 access key id (both variables are required; @@ -164,12 +164,12 @@ access key, not with the API token. In CI, export one of the following: # Resolve s3:// through the API, sign with the env key LATITUDESH_TOKEN=ak_xxx LSH_S3_ACCESS_KEY_ID=... LSH_S3_SECRET_ACCESS_KEY=... \ - lsh s3 cp ./dump.sql s3://backups/2026/09/ + lsh s3 copy ./dump.sql s3://backups/2026/09/ # No API at all: endpoint + backend bucket name LSH_S3_ENDPOINT_URL=https://s3.us-central-1.storage.sh \ LSH_S3_ACCESS_KEY_ID=... LSH_S3_SECRET_ACCESS_KEY=... \ - lsh s3 ls s3://backups-7f3a/ + lsh s3 list s3://backups-7f3a/ # Hand a scoped key to another job (the secret is printed once) lsh s3 access-keys create --bucket backups=rw --name ci -o text \ @@ -281,7 +281,7 @@ specific exit code to every failure so scripts can tell "not found" from Errors are always printed to stderr; stdout carries only data, so '-o json' and '-o text' output stays pipeable even when a command fails. - lsh s3 cp ./dump.sql s3://backups/ || case $? in + lsh s3 copy ./dump.sql s3://backups/ || case $? in 4) echo "configure an access key: lsh s3 configure" ;; 7) echo "refused; add --yes in CI" ;; esac @@ -292,3 +292,58 @@ and still exit 1 for every error. Their behaviour is unchanged; only `, ) } + +func makeHelpS3Cmd() *cobra.Command { + return newHelpTopic( + "object-storage", + "How object storage addressing, endpoints and access keys work", + `lsh — Object storage + +The commands live under 'lsh s3' ('lsh s3 --help' lists them). + +Two planes + Buckets, access keys, lifecycle rules, metrics and usage are managed + through the Latitude API with your API token ('lsh login'). + Objects (list, copy, move, delete, get, presign, sync) are read and + written on the bucket's own S3 endpoint with an S3 access key, which is a + separate credential the API returns exactly once when the key is created. + +Addressing + Buckets and objects are written as s3://[/]. may be + the display name, the bkt_ ID or the backend bucket name. When the same + display name exists more than once (another project, storage class or + site), the command lists the candidates; narrow it with --project, + -c/--storage-class or --site, or use the bkt_ ID. + + The endpoint, the SigV4 signing region and path-style addressing are + derived from the bucket, so none of them is configured by hand: + standard https://s3..storage.sh + high_performance https://objects..storage.sh (bound to one site) + +Access keys + A key is either fullaccess (every bucket of a storage class in a project — + and of one site, for high_performance) or limited_access (specific buckets, + rw or readonly). Keys saved in the active profile are picked automatically + per bucket, preferring the least-privileged one that covers it. + + For the machine you are on: lsh s3 configure + For an app, CI job or someone: lsh s3 access-keys create --bucket =rw + Reuse elsewhere: lsh s3 configure export s3:// --format env + + Precedence: LSH_S3_ACCESS_KEY_ID + LSH_S3_SECRET_ACCESS_KEY (environment) + > --access-key > the best saved key of the active profile. + Without any of them the command exits 4 with the commands that fix it. + +Without the API + --endpoint-url (or LSH_S3_ENDPOINT_URL) talks to an S3 endpoint directly: + is then the backend bucket name and credentials come only from + the environment. --signing-region overrides the region when it cannot be + derived from the endpoint. + +Safety + Deleting several objects asks for confirmation in a terminal and needs + --yes in scripts; --dry-run prints the plan without writing anything; + --max-delete caps a recursive deletion. Exit codes: 'lsh help exit-codes'. +`, + ) +} diff --git a/cmd/build_s3.go b/cmd/build_s3.go index 5e93163..a23b075 100644 --- a/cmd/build_s3.go +++ b/cmd/build_s3.go @@ -4,8 +4,9 @@ import ( s3 "github.com/latitudesh/lsh/cmd/s3" ) -// s3Cmd is the object storage group (`lsh s3`, aliases `buckets` and the -// legacy `storage-objects`). It replaces cmd/storage_objects. +// s3Cmd is the object storage group (`lsh s3`, alias `object-storage`). It +// replaces cmd/storage_objects; the old name survives as a hidden, deprecated +// command that re-dispatches here (see s3.NewLegacyAliasCmd). var s3Cmd = s3.NewGroupCmd() func init() { @@ -28,4 +29,5 @@ func init() { s3.Finalize(s3Cmd) rootCmd.AddCommand(s3Cmd) + rootCmd.AddCommand(s3.NewLegacyAliasCmd(rootCmd)) } diff --git a/cmd/s3/accesskeys.go b/cmd/s3/accesskeys.go index 29a6a25..21b6497 100644 --- a/cmd/s3/accesskeys.go +++ b/cmd/s3/accesskeys.go @@ -11,16 +11,21 @@ func NewAccessKeysCmd() *cobra.Command { cmd := &cobra.Command{ Use: "access-keys", Aliases: []string{"keys", "access-key"}, - Short: "Manage S3 access keys", + GroupID: groupCredentials, + Short: "Manage access keys (create, list, rotate, delete)", Long: `Manage the S3 access keys of your object storage buckets. +To set up the machine you are on, 'lsh s3 configure' asks the questions and +saves the key for you; the commands here are for keys handed to applications, +CI jobs or other people, and for auditing what exists. + Access keys are separate from your API token: the API creates them, but the secret is returned once and never again. A key is either fullaccess (every bucket of the project in that storage class, and site for high_performance) or limited_access (specific buckets with rw or readonly permission). Keys can be saved in the active lsh profile (--save, or --save-as ); -'lsh s3 cp/ls/rm' then pick the least-privileged saved key that covers the +'lsh s3 copy/ls/rm' then pick the least-privileged saved key that covers the bucket automatically. Saved keys never leave this machine except towards the bucket's endpoint.`, Example: ` lsh s3 access-keys list diff --git a/cmd/s3/accesskeys_create.go b/cmd/s3/accesskeys_create.go index 8c1aae3..149ea97 100644 --- a/cmd/s3/accesskeys_create.go +++ b/cmd/s3/accesskeys_create.go @@ -414,7 +414,7 @@ func printCreatedHuman(w io.Writer, v createdKeyView, showSecret bool) { if v.Endpoint != "" { fmt.Fprintf(w, " Endpoint: %s\n", v.Endpoint) } else { - fmt.Fprintf(w, " Endpoint: (see 'lsh s3 stat s3://' for the bucket endpoint)\n") + fmt.Fprintf(w, " Endpoint: (see 'lsh s3 get s3://' for the bucket endpoint)\n") } if v.SigningRegion != "" { fmt.Fprintf(w, " Signing region: %s\n", v.SigningRegion) diff --git a/cmd/s3/accesskeys_list.go b/cmd/s3/accesskeys_list.go index b5cd45b..ece7c07 100644 --- a/cmd/s3/accesskeys_list.go +++ b/cmd/s3/accesskeys_list.go @@ -144,7 +144,7 @@ func runAccessKeysList(cmd *cobra.Command, _ []string) error { slugs[b.ProjectRef()] = b.ProjectSlug } } else { - // Pick the project like `lsh s3 ls`: --project / LSH_PROJECT, or + // Pick the project like `lsh s3 list`: --project / LSH_PROJECT, or // --all-projects, otherwise the interactive picker (with "All projects"). project, allProjects, perr := cli.PickProjectForList(cmd) if perr != nil { diff --git a/cmd/s3/configure.go b/cmd/s3/configure.go index d01dc8d..4b38c6a 100644 --- a/cmd/s3/configure.go +++ b/cmd/s3/configure.go @@ -40,9 +40,15 @@ func NewConfigureCmd() *cobra.Command { cmd := newCmd(&cobra.Command{ Use: "configure", Aliases: []string{"credentials"}, - Short: "Set up S3 access keys on this machine", + GroupID: groupCredentials, + Short: "Create an access key and save it for this machine", Long: `Create an S3 access key and save it in the active lsh profile so the object -commands (ls, cp, rm, stat, presign) can use it automatically. +commands (list, copy, move, delete, get, presign) can use it automatically. + +This is the path for the machine you are typing on. For an application, a CI +job or a colleague, create a separate key with 'lsh s3 access-keys create': +it prints the key once (and saves it here too with --save) instead of +walking through the wizard. The wizard asks for the project, the storage class (standard = general purpose; high_performance = low latency on selected sites), the site for @@ -630,7 +636,7 @@ func runConfigure(cmd *cobra.Command, _ []string) error { if savedAs != st.Name { objectstorage.Hintf("(%q already held another key, so this one was saved as %q)", st.Name, savedAs) } - objectstorage.Hintf("Next: lsh s3 ls%s", configureExampleBucket(st, " lsh s3 cp ./file s3://%s/")) + objectstorage.Hintf("Next: lsh s3 list%s", configureExampleBucket(st, " lsh s3 copy ./file s3://%s/")) objectstorage.Hintf("For apps or CI create a separate key: lsh s3 access-keys create --bucket --name ") objectstorage.Hintf("Export it for other tools: lsh s3 configure export --format env%s", configureExampleBucket(st, " s3://%s")) if !isHuman() { @@ -912,7 +918,7 @@ func configureSelectInteractive(ctx context.Context, cmd *cobra.Command, st *con if st.Site != "" { where += " in " + st.Site } - return exitcode.Errorf(exitcode.NotFound, "no %s buckets found in project %s; create one with 'lsh s3 mb s3:// --region --project %s'", where, r.Project, r.Project) + return exitcode.Errorf(exitcode.NotFound, "no %s buckets found in project %s; create one with 'lsh s3 create-bucket s3:// --region --project %s'", where, r.Project, r.Project) } labels := make([]string, len(candidates)) for i, b := range candidates { diff --git a/cmd/s3/cp.go b/cmd/s3/cp.go index 4af24a5..8673bdd 100644 --- a/cmd/s3/cp.go +++ b/cmd/s3/cp.go @@ -4,13 +4,14 @@ import ( "github.com/spf13/cobra" ) -// NewCpCmd builds `lsh s3 cp `. +// NewCpCmd builds `lsh s3 copy `. func NewCpCmd() *cobra.Command { var f transferFlags cmd := newCmd(&cobra.Command{ - Use: "cp ", - Aliases: []string{"copy", "upload", "download"}, - Short: "Copy files and objects", + Use: "copy ", + Aliases: []string{"cp", "upload", "download"}, + GroupID: groupObjects, + Short: "Copy files and objects (alias: cp)", Long: `Copy a local file to a bucket, an object to the local disk, or an object to another object (server-side, same endpoint; when the two buckets resolve different access keys the object is streamed through this machine instead, @@ -34,15 +35,15 @@ progress meter, and the upload succeeds whatever the real length turns out to be The Content-Type is taken from --content-type, then the file extension, then the first bytes of the file (--no-guess-mime-type stores binary/octet-stream). Output lines (upload:/download:/copy:) go to stdout; progress, hints and -errors to stderr. --dry-run (or --dryrun) prints the plan without writing.`, - Example: ` lsh s3 cp ./dump.sql s3://backups/2026/09/ - lsh s3 cp s3://backups/2026/09/dump.sql ./restore/ - lsh s3 cp ./site s3://www --recursive --exclude "*" --include "*.html" --content-type text/html - lsh s3 cp s3://backups/2026/ ./backups/ --recursive --dryrun - pg_dump mydb | lsh s3 cp - s3://backups/mydb.sql - tar cz /data | lsh s3 cp - s3://backups/data.tgz --expected-size 500000000000 # approximate; sizes parts for streams over 160 GiB - lsh s3 cp s3://backups/report.pdf - > report.pdf - lsh s3 cp s3://backups/a.txt s3://archive/2026/a.txt --metadata owner=ops,team=infra`, +errors to stderr. --dry-run (or --dry-run) prints the plan without writing.`, + Example: ` lsh s3 copy ./dump.sql s3://backups/2026/09/ + lsh s3 copy s3://backups/2026/09/dump.sql ./restore/ + lsh s3 copy ./site s3://www --recursive --exclude "*" --include "*.html" --content-type text/html + lsh s3 copy s3://backups/2026/ ./backups/ --recursive --dry-run + pg_dump mydb | lsh s3 copy - s3://backups/mydb.sql + tar cz /data | lsh s3 copy - s3://backups/data.tgz --expected-size 500000000000 # approximate; sizes parts for streams over 160 GiB + lsh s3 copy s3://backups/report.pdf - > report.pdf + lsh s3 copy s3://backups/a.txt s3://archive/2026/a.txt --metadata owner=ops,team=infra`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { return runCopyCommand(cmd, args, &f, false) diff --git a/cmd/s3/lifecycle.go b/cmd/s3/lifecycle.go index fa27bc0..4ab999e 100644 --- a/cmd/s3/lifecycle.go +++ b/cmd/s3/lifecycle.go @@ -37,7 +37,8 @@ func NewLifecycleCmd() *cobra.Command { cmd := &cobra.Command{ Use: "lifecycle", Aliases: []string{"lifecycle-rules"}, - Short: "Manage bucket lifecycle rules", + GroupID: groupReports, + Short: "Manage lifecycle rules (expire objects automatically)", Long: `Manage the lifecycle rules of a bucket (automatic expiration of objects, noncurrent versions and incomplete multipart uploads). diff --git a/cmd/s3/ls.go b/cmd/s3/ls.go index c7c093e..b6e0323 100644 --- a/cmd/s3/ls.go +++ b/cmd/s3/ls.go @@ -33,12 +33,13 @@ const ( // maxListPageSize is the largest max-keys S3 accepts per request. const maxListPageSize = 1000 -// NewLsCmd builds `lsh s3 ls [s3://bucket[/prefix]]`. +// NewLsCmd builds `lsh s3 list [s3://bucket[/prefix]]`. func NewLsCmd() *cobra.Command { cmd := newCmd(&cobra.Command{ - Use: "ls [s3://bucket[/prefix]]", - Aliases: []string{"list"}, - Short: "List buckets, or the objects under a prefix", + Use: "list [s3://bucket[/prefix]]", + Aliases: []string{"ls"}, + GroupID: groupBuckets, + Short: "List buckets, or the objects under a prefix (alias: ls)", Long: `List buckets, or the objects under a prefix. Without an argument, lists buckets through the Latitude API. Pick a project @@ -59,14 +60,14 @@ stderr and accepted by --starting-token. Structured output (-o json|yaml|csv) emits one row per object or prefix; --stream with -o json prints one JSON object per line as the listing arrives instead of buffering everything.`, - Example: ` lsh s3 ls - lsh s3 ls --project my-project --storage-class high_performance - lsh s3 ls s3://backups/2026/09/ - lsh s3 ls s3://backups --recursive --human-readable --summarize - lsh s3 ls s3://backups/logs/ --versions - lsh s3 ls s3://backups --recursive -o json --stream | jq -r .key - lsh s3 ls s3://backups --no-paginate --page-size 100 - lsh s3 ls s3://backups --starting-token `, + Example: ` lsh s3 list + lsh s3 list --project my-project --storage-class high_performance + lsh s3 list s3://backups/2026/09/ + lsh s3 list s3://backups --recursive --human-readable --summarize + lsh s3 list s3://backups/logs/ --versions + lsh s3 list s3://backups --recursive -o json --stream | jq -r .key + lsh s3 list s3://backups --no-paginate --page-size 100 + lsh s3 list s3://backups --starting-token `, Args: cobra.MaximumNArgs(1), RunE: runLs, }) diff --git a/cmd/s3/mb.go b/cmd/s3/mb.go index 8ed0a03..f6b8eed 100644 --- a/cmd/s3/mb.go +++ b/cmd/s3/mb.go @@ -48,12 +48,13 @@ type mbOptions struct { RetentionDays int64 } -// NewMbCmd builds `lsh s3 mb s3://bucket`. +// NewMbCmd builds `lsh s3 create-bucket s3://bucket`. func NewMbCmd() *cobra.Command { cmd := newCmd(&cobra.Command{ - Use: "mb s3://bucket", - Aliases: []string{"create"}, - Short: "Create a bucket", + Use: "create-bucket s3://bucket", + Aliases: []string{"mb", "create"}, + GroupID: groupBuckets, + Short: "Create a bucket (alias: mb)", Long: `Create an object storage bucket through the Latitude API. The bucket is created in a Latitude site (--region DAL, NYC, TYO4…). The @@ -63,13 +64,13 @@ retention cannot be changed after creation. In an interactive session missing --project and --region are asked for, and right after the bucket is created the CLI offers to create and save an S3 -access key so cp/ls/rm work immediately; use --create-access-key or +access key so copy/list/delete work immediately; use --create-access-key or --no-access-key in scripts.`, - Example: ` lsh s3 mb s3://backups --region DAL --project my-project - lsh s3 mb s3://fast-cache --region TYO4 --storage-class high_performance - lsh s3 mb s3://audit --region NYC --locking --retention-mode COMPLIANCE --retention-days 30 - lsh s3 mb s3://ci-artifacts --region DAL --create-access-key - lsh s3 mb s3://backups --region DAL -o json`, + Example: ` lsh s3 create-bucket s3://backups --region DAL --project my-project + lsh s3 create-bucket s3://fast-cache --region TYO4 --storage-class high_performance + lsh s3 create-bucket s3://audit --region NYC --locking --retention-mode COMPLIANCE --retention-days 30 + lsh s3 create-bucket s3://ci-artifacts --region DAL --create-access-key + lsh s3 create-bucket s3://backups --region DAL -o json`, Args: cobra.MaximumNArgs(1), RunE: runMb, }) @@ -173,7 +174,7 @@ func runMb(cmd *cobra.Command, args []string) error { offerAccessKey(ctx, cmd, b, o) if isHuman() { - objectstorage.Hintf("Try: lsh s3 cp ./file s3://%s/", b.Name) + objectstorage.Hintf("Try: lsh s3 copy ./file s3://%s/", b.Name) } return nil } @@ -199,7 +200,7 @@ func mbOptionsFromCmd(cmd *cobra.Command, args []string) (mbOptions, error) { } o.Name = ref.Bucket default: - return o, exitcode.Errorf(exitcode.Usage, "missing bucket: usage 'lsh s3 mb s3:// --region '") + return o, exitcode.Errorf(exitcode.Usage, "missing bucket: usage 'lsh s3 create-bucket s3:// --region '") } o.Region, _ = f.GetString(flagMbRegion) o.StorageClass, _ = f.GetString(flagMbStorageClass) @@ -223,7 +224,7 @@ func buildCreateBucketRequest(o mbOptions) (operations.PostStorageBucketsRequest var req operations.PostStorageBucketsRequestBody name := strings.TrimSpace(o.Name) if name == "" { - return req, exitcode.Errorf(exitcode.Usage, "missing bucket name: usage 'lsh s3 mb s3:// --region '") + return req, exitcode.Errorf(exitcode.Usage, "missing bucket name: usage 'lsh s3 create-bucket s3:// --region '") } if strings.ContainsAny(name, " /\\") { return req, exitcode.Errorf(exitcode.Usage, "invalid bucket name %q: it cannot contain spaces or slashes", name) diff --git a/cmd/s3/metrics.go b/cmd/s3/metrics.go index 1cd5f7c..9d37521 100644 --- a/cmd/s3/metrics.go +++ b/cmd/s3/metrics.go @@ -258,8 +258,9 @@ func fetchMetrics(ctx context.Context, api *sdk.Latitudesh, buckets []*objectsto // NewMetricsCmd builds `lsh s3 metrics [s3://bucket]`. func NewMetricsCmd() *cobra.Command { cmd := newCmd(&cobra.Command{ - Use: "metrics [s3://bucket]", - Short: "Storage consumption and estimated cost", + Use: "metrics [s3://bucket]", + GroupID: groupReports, + Short: "Current size and estimated cost of a bucket", Long: `Show the storage consumed in the current billing period and the estimated cost, per bucket. Without a bucket every bucket of the project (or of the team with --all-projects) is listed, sorted by estimated cost, with a TOTAL row in diff --git a/cmd/s3/mv.go b/cmd/s3/mv.go index a5f2e7a..1df96d4 100644 --- a/cmd/s3/mv.go +++ b/cmd/s3/mv.go @@ -4,13 +4,14 @@ import ( "github.com/spf13/cobra" ) -// NewMvCmd builds `lsh s3 mv `. +// NewMvCmd builds `lsh s3 move `. func NewMvCmd() *cobra.Command { var f transferFlags cmd := newCmd(&cobra.Command{ - Use: "mv ", - Aliases: []string{"move"}, - Short: "Move files and objects", + Use: "move ", + Aliases: []string{"mv"}, + GroupID: groupObjects, + Short: "Move files and objects (alias: mv)", Long: `Move a local file into a bucket, an object to the local disk, or an object to another key (server-side, same endpoint; buckets that resolve different access keys are streamed through this machine). Each source is deleted only after @@ -21,11 +22,11 @@ local, or source and destination naming the same object, is refused. All cp flags apply (--recursive, --exclude/--include, --content-type, --metadata, --no-overwrite, --quiet, ...). Output lines use the "move:" -prefix. --dry-run (or --dryrun) prints the plan without copying or deleting.`, - Example: ` lsh s3 mv ./dump.sql s3://backups/2026/09/ - lsh s3 mv s3://backups/tmp/report.pdf ./reports/ - lsh s3 mv s3://backups/tmp/ s3://backups/archive/2026/ --recursive - lsh s3 mv ./logs s3://logs/host-1/ --recursive --exclude "*" --include "*.log" --dryrun`, +prefix. --dry-run (or --dry-run) prints the plan without copying or deleting.`, + Example: ` lsh s3 move ./dump.sql s3://backups/2026/09/ + lsh s3 move s3://backups/tmp/report.pdf ./reports/ + lsh s3 move s3://backups/tmp/ s3://backups/archive/2026/ --recursive + lsh s3 move ./logs s3://logs/host-1/ --recursive --exclude "*" --include "*.log" --dry-run`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { return runCopyCommand(cmd, args, &f, true) diff --git a/cmd/s3/presign.go b/cmd/s3/presign.go index 43d66c3..befddfa 100644 --- a/cmd/s3/presign.go +++ b/cmd/s3/presign.go @@ -34,7 +34,8 @@ func NewPresignCmd() *cobra.Command { cmd := newCmd(&cobra.Command{ Use: "presign s3://bucket/key", Aliases: []string{"presigned-url", "share"}, - Short: "Generate a pre-signed URL", + GroupID: groupObjects, + Short: "Generate a pre-signed URL for an object", Long: `Generate a pre-signed URL that grants temporary access to one object without sharing credentials. @@ -69,7 +70,7 @@ command composes with $(...) and curl.`, // ObjectRef's shared message points at --recursive, a flag only rm and // the transfer commands have; presign signs exactly one object. if r, parseErr := objectstorage.ParseRemote(args[0]); parseErr == nil && strings.HasSuffix(r.Key, "/") { - return printErr(objectstorage.ErrUsagef("key %q ends with '/'; presign signs a single object (use 'lsh s3 ls %s' to list the prefix)", r.Key, r)) + return printErr(objectstorage.ErrUsagef("key %q ends with '/'; presign signs a single object (use 'lsh s3 list %s' to list the prefix)", r.Key, r)) } ref, err := objectstorage.ObjectRef(args[0], false) if err != nil { diff --git a/cmd/s3/rb.go b/cmd/s3/rb.go index 94dad6a..918658a 100644 --- a/cmd/s3/rb.go +++ b/cmd/s3/rb.go @@ -62,12 +62,13 @@ type rbOptions struct { Human bool } -// NewRbCmd builds `lsh s3 rb s3://bucket`. +// NewRbCmd builds `lsh s3 delete-bucket s3://bucket`. func NewRbCmd() *cobra.Command { cmd := newCmd(&cobra.Command{ - Use: "rb s3://bucket", - Aliases: []string{"delete"}, - Short: "Delete a bucket", + Use: "delete-bucket s3://bucket", + Aliases: []string{"rb"}, + GroupID: groupBuckets, + Short: "Delete a bucket (alias: rb)", Long: `Delete a bucket through the Latitude API. Without --force the bucket must be empty; the API refuses otherwise (exit 7). @@ -78,10 +79,10 @@ force-deleted; GOVERNANCE retention needs --bypass-governance-retention. --force never skips the confirmation: pass --yes in scripts. Without a TTY and without --yes the command fails fast with exit 7.`, - Example: ` lsh s3 rb s3://backups - lsh s3 rb s3://backups --force - lsh s3 rb s3://backups --force --versions --yes - lsh s3 rb s3://backups --force --max-delete 100 --dryrun`, + Example: ` lsh s3 delete-bucket s3://backups + lsh s3 delete-bucket s3://backups --force + lsh s3 delete-bucket s3://backups --force --versions --yes + lsh s3 delete-bucket s3://backups --force --max-delete 100 --dry-run`, Args: cobra.ExactArgs(1), RunE: runRb, }) diff --git a/cmd/s3/rm.go b/cmd/s3/rm.go index c054073..f184f49 100644 --- a/cmd/s3/rm.go +++ b/cmd/s3/rm.go @@ -17,7 +17,7 @@ import ( // rmBatchSize is the S3 multi-delete limit. const rmBatchSize = 1000 -// rmOptions are the parsed flags of `lsh s3 rm`. +// rmOptions are the parsed flags of `lsh s3 delete`. type rmOptions struct { Recursive bool All bool @@ -31,13 +31,14 @@ type rmOptions struct { Filters *objectstorage.Filters } -// NewRmCmd builds `lsh s3 rm s3://bucket/key`. +// NewRmCmd builds `lsh s3 delete s3://bucket/key`. func NewRmCmd() *cobra.Command { filters := &objectstorage.Filters{} cmd := newCmd(&cobra.Command{ - Use: "rm s3://bucket/key", - Aliases: []string{"delete-object", "remove"}, - Short: "Delete objects", + Use: "delete s3://bucket/key", + Aliases: []string{"rm", "delete-object", "remove"}, + GroupID: groupObjects, + Short: "Delete objects — the bucket stays (alias: rm)", Long: `Delete one object, or every object under a prefix with --recursive. A single object is deleted without confirmation and the command exits 0 even @@ -52,11 +53,11 @@ relative to the prefix, in the order given; the last matching rule wins. Object lock: buckets in COMPLIANCE mode are refused; GOVERNANCE mode requires --bypass-governance-retention. Use --dry-run to print the plan without deleting anything.`, - Example: ` lsh s3 rm s3://backups/2026/09/dump.sql - lsh s3 rm s3://logs/tmp/ --recursive --dryrun - lsh s3 rm s3://logs/tmp/ --recursive --exclude "*" --include "*.log" --yes - lsh s3 rm s3://logs --recursive --all --max-delete 5000 --yes - lsh s3 rm s3://backups/dump.sql --version-id 3HL4kqtJlcpXroDTDmJ`, + Example: ` lsh s3 delete s3://backups/2026/09/dump.sql + lsh s3 delete s3://logs/tmp/ --recursive --dry-run + lsh s3 delete s3://logs/tmp/ --recursive --exclude "*" --include "*.log" --yes + lsh s3 delete s3://logs --recursive --all --max-delete 5000 --yes + lsh s3 delete s3://backups/dump.sql --version-id 3HL4kqtJlcpXroDTDmJ`, Args: cobra.ExactArgs(1), }) f := cmd.Flags() @@ -135,7 +136,7 @@ func rmParseArgs(arg string, opts rmOptions) (objectstorage.Ref, error) { ref, err := objectstorage.ObjectRef(arg, false) if err != nil { if bucket, bucketErr := objectstorage.ParseBucketOnly(arg); bucketErr == nil { - return ref, objectstorage.ErrUsagef("%s names a bucket, not an object; to delete its objects use 'lsh s3 rm s3://%s --recursive --all', to delete the bucket itself use 'lsh s3 rb s3://%s'", arg, bucket.Bucket, bucket.Bucket) + return ref, objectstorage.ErrUsagef("%s names a bucket, not an object; to delete its objects use 'lsh s3 delete s3://%s --recursive --all', to delete the bucket itself use 'lsh s3 delete-bucket s3://%s'", arg, bucket.Bucket, bucket.Bucket) } } return ref, err diff --git a/cmd/s3/rm_test.go b/cmd/s3/rm_test.go index dc9d029..f57eebc 100644 --- a/cmd/s3/rm_test.go +++ b/cmd/s3/rm_test.go @@ -650,13 +650,13 @@ func TestRmBucketOnlyPointsAtRb(t *testing.T) { if code := exitcode.Of(err); code != exitcode.Usage { t.Errorf("exit code = %d, want %d", code, exitcode.Usage) } - for _, want := range []string{"names a bucket, not an object", "--recursive --all", "lsh s3 rb s3://bkt_abc123"} { + for _, want := range []string{"names a bucket, not an object", "--recursive --all", "lsh s3 delete-bucket s3://bkt_abc123"} { if !strings.Contains(err.Error(), want) { t.Errorf("error %q is missing %q", err, want) } } // A prefix keeps the shorter --recursive hint. - if _, err := rmParseArgs("s3://logs/tmp/", rmOptions{}); err == nil || strings.Contains(err.Error(), "lsh s3 rb") { + if _, err := rmParseArgs("s3://logs/tmp/", rmOptions{}); err == nil || strings.Contains(err.Error(), "lsh s3 delete-bucket") { t.Errorf("a prefix must keep the --recursive hint, got %v", err) } } diff --git a/cmd/s3/s3.go b/cmd/s3/s3.go index b3889d0..c70f413 100644 --- a/cmd/s3/s3.go +++ b/cmd/s3/s3.go @@ -1,7 +1,7 @@ // Package s3 implements `lsh s3`, the object storage command group. It mixes // control-plane calls to the Latitude API (buckets, access keys, lifecycle, // metrics, usage) with data-plane calls straight to the bucket's S3 endpoint -// (ls, cp, rm, stat, presign), using the verbs and flags of `aws s3` so +// (list, copy, move, delete, get, presign), using the verbs and flags of `aws s3` so // existing habits and scripts carry over. package s3 @@ -39,39 +39,54 @@ const ( flagSite = "site" ) +// Help groups of the s3 command list. Grouping the fourteen subcommands the +// way the root groups its command families keeps the list from reading flat. +const ( + groupBuckets = "s3-buckets" + groupObjects = "s3-objects" + groupCredentials = "s3-credentials" + groupReports = "s3-reports" +) + // NewGroupCmd builds the `s3` group. Subcommands are attached by // cmd/build_s3.go so each lives in its own file. +// +// The canonical verbs follow the rest of the CLI (list, get, copy, move, +// delete, create-bucket, delete-bucket); the short S3 spellings (ls, stat, +// cp, mv, rm, mb, rb) are aliases, so both vocabularies work and the help +// reads in the CLI's own terms. func NewGroupCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "s3", - Aliases: []string{"buckets", "storage-objects"}, - // Typing these at the root gets a "did you mean s3" hint. - SuggestFor: []string{"s3api", "upload", "download", "configure", "bucket", "objects", "object-storage"}, + Use: "s3", + // Typing these at the root gets a "did you mean s3" hint. The long + // name is deliberately not an alias: 'lsh help object-storage' is the + // guide, and cobra's help resolves aliases as well as names. + SuggestFor: []string{"s3api", "object-storage", "upload", "download", "configure", "bucket", "buckets", "objects"}, GroupID: cli.StorageGroupID, - Short: "Object storage: buckets, objects, access keys and lifecycle", - Long: `Manage S3-compatible object storage. + Short: "Object storage: buckets, objects, access keys and lifecycle rules", + Long: `Object storage: buckets, objects, access keys and lifecycle rules. -Bucket management, access keys, lifecycle rules and metrics go through the -Latitude API; object operations (ls, cp, rm, stat, presign) talk to the -bucket's S3 endpoint directly. Buckets are addressed as s3://[/], -where is the display name, the bkt_ ID or the backend bucket name. -Endpoint, signing region and path-style addressing are resolved from the API, -so nothing has to be configured by hand. +Buckets and objects are addressed as s3://[/]. Endpoint, region +and credentials are resolved for you. See 'lsh help object-storage' for how +addressing and access keys work. Getting started: - lsh s3 mb s3://backups --region DAL --project my-project - lsh s3 cp ./dump.sql s3://backups/2026/09/ - lsh s3 ls s3://backups/2026/09/ + lsh s3 create-bucket s3://backups --region DAL --project my-project + lsh s3 copy ./dump.sql s3://backups/2026/09/ + lsh s3 list s3://backups/2026/09/ +Short aliases: ls, mb, rb, cp, mv, rm, stat. Exit codes for scripts: 'lsh help exit-codes'.`, - Example: ` lsh s3 ls - lsh s3 ls s3://backups/2026/ --recursive --human-readable --summarize - lsh s3 cp ./dump.sql s3://backups/2026/09/ - lsh s3 cp s3://backups/2026/09/dump.sql ./restore/ - lsh s3 rm s3://backups/tmp/ --recursive --dryrun + Example: ` lsh s3 list + lsh s3 list s3://backups/2026/ --recursive --human-readable --summarize + lsh s3 copy s3://backups/2026/09/dump.sql ./restore/ + lsh s3 delete s3://backups/tmp/ --recursive --dry-run + lsh s3 delete-bucket s3://backups lsh s3 presign s3://backups/report.pdf --expires-in 15m - lsh s3 access-keys create --bucket backups --name ci-deploy + lsh s3 access-keys create --bucket backups=rw --name ci-deploy lsh s3 lifecycle create s3://logs --prefix tmp/ --expiration-days 7`, + // --dryrun is the hidden spelling of the global --dry-run: map it onto + // the same switch so every subcommand sees a single value. PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { if v, _ := cmd.Flags().GetBool(flagDryRunAWS); v { lsh.DryRun = true @@ -79,7 +94,12 @@ Exit codes for scripts: 'lsh help exit-codes'.`, return nil }, } - + cmd.AddGroup( + &cobra.Group{ID: groupBuckets, Title: "Buckets:"}, + &cobra.Group{ID: groupObjects, Title: "Objects:"}, + &cobra.Group{ID: groupCredentials, Title: "Credentials:"}, + &cobra.Group{ID: groupReports, Title: "Settings and reports:"}, + ) pf := cmd.PersistentFlags() pf.String(flagAccessKey, "", "use the saved access key with this name instead of the automatic selection") pf.String(flagEndpointURL, "", "address the bucket on this S3 endpoint without the Latitude API (bucket is the backend name; credentials only from the environment)") @@ -224,7 +244,7 @@ func signingRegionOverride(cmd *cobra.Command) string { return os.Getenv(objectstorage.EnvSigningRegion) } -// dryRun reports whether --dry-run (or --dryrun) is active. +// dryRun reports whether --dry-run (or --dry-run) is active. func dryRun() bool { return lsh.DryRun } // isHuman reports whether output goes to the human (table) format, in which @@ -491,3 +511,29 @@ func saveNewKey(cmd *cobra.Command, name string, k config.StoredAccessKey) (stri } return final, profileName, nil } + +// NewLegacyAliasCmd returns the hidden `storage-objects` command kept for +// scripts written against the group this one replaced. Cobra prints the +// deprecation notice on stderr and leaves the command out of the help; the +// arguments are re-dispatched to `lsh s3` untouched (flag parsing is disabled +// here so nothing is interpreted before it reaches the real command). +// +// Two legacy verbs changed meaning and are called out in the notice: `rm` and +// `delete` remove objects now, and a bucket is removed with `delete-bucket`. +func NewLegacyAliasCmd(root *cobra.Command) *cobra.Command { + return &cobra.Command{ + Use: "storage-objects", + Hidden: true, + SilenceUsage: true, + Deprecated: "use 'lsh s3' instead. Note: 'rm' and 'delete' now remove objects; remove a bucket with 'lsh s3 delete-bucket'", + DisableFlagParsing: true, + RunE: func(cmd *cobra.Command, args []string) error { + root.SetArgs(append([]string{"s3"}, args...)) + err := root.Execute() + // The inner run has already reported the error; keep the outer + // Execute from printing it a second time. + cmd.SilenceErrors = true + return err + }, + } +} diff --git a/cmd/s3/stat.go b/cmd/s3/stat.go index b549e09..5441b53 100644 --- a/cmd/s3/stat.go +++ b/cmd/s3/stat.go @@ -18,13 +18,14 @@ import ( const flagStatVersionID = "version-id" -// NewStatCmd builds `lsh s3 stat s3://bucket[/key]`. +// NewStatCmd builds `lsh s3 get s3://bucket[/key]`. func NewStatCmd() *cobra.Command { cmd := newCmd(&cobra.Command{ - Use: "stat s3://bucket[/key]", - Aliases: []string{"get", "describe", "head"}, + Use: "get s3://bucket[/key]", + Aliases: []string{"stat", "describe", "head"}, + GroupID: groupBuckets, SuggestFor: []string{"show", "info"}, - Short: "Show a bucket or object", + Short: "Show a bucket or an object (alias: stat)", Long: `Show a bucket or an object. For a bucket (s3://bucket) the details come from the Latitude API: id, names, @@ -32,10 +33,10 @@ endpoint, site, signing region, class, versioning, object lock and which saved access keys cover it. For an object (s3://bucket/key) the CLI issues a HEAD request to the S3 endpoint: size, ETag, content type, last modified, version and user metadata.`, - Example: ` lsh s3 stat s3://backups - lsh s3 stat s3://backups/2026/09/dump.sql - lsh s3 stat s3://backups/2026/09/dump.sql --version-id 3HL4kqtJlcpXroDTDmJ - lsh s3 stat s3://backups -o json`, + Example: ` lsh s3 get s3://backups + lsh s3 get s3://backups/2026/09/dump.sql + lsh s3 get s3://backups/2026/09/dump.sql --version-id 3HL4kqtJlcpXroDTDmJ + lsh s3 get s3://backups -o json`, Args: cobra.ExactArgs(1), RunE: runStat, }) @@ -75,7 +76,7 @@ func runStat(cmd *cobra.Command, args []string) error { } if strings.HasSuffix(ref.Key, "/") { - return printErr(objectstorage.ErrUsagef("key %q ends with '/'; stat inspects a single object (use 'lsh s3 ls %s' to list a prefix)", ref.Key, ref)) + return printErr(objectstorage.ErrUsagef("key %q ends with '/'; stat inspects a single object (use 'lsh s3 list %s' to list a prefix)", ref.Key, ref)) } sess, err := openBucket(ctx, cmd, ref.Bucket, false) if err != nil { diff --git a/cmd/s3/sync.go b/cmd/s3/sync.go index ce9d464..30d1654 100644 --- a/cmd/s3/sync.go +++ b/cmd/s3/sync.go @@ -28,8 +28,9 @@ func NewSyncCmd() *cobra.Command { var f transferFlags var so syncOptions cmd := newCmd(&cobra.Command{ - Use: "sync ", - Short: "Synchronize directories and prefixes", + Use: "sync ", + GroupID: groupObjects, + Short: "Synchronize directories and prefixes", Long: `Recursively copy new and changed files between a local directory and a prefix, or between two prefixes on the same endpoint. @@ -44,11 +45,11 @@ a partial run never leaves the destination without both versions; re-run the sync to apply them. They also never leave the destination tree: a candidate reached through a directory symlink is reported and kept. -Output lines use upload:/download:/copy:/delete:. --dry-run (or --dryrun) +Output lines use upload:/download:/copy:/delete:. --dry-run (or --dry-run) prints the plan without writing or deleting anything.`, Example: ` lsh s3 sync ./site s3://www lsh s3 sync s3://backups/2026/ ./backups/2026/ - lsh s3 sync ./logs s3://logs/host-1/ --exclude "*" --include "*.log" --delete --dryrun + lsh s3 sync ./logs s3://logs/host-1/ --exclude "*" --include "*.log" --delete --dry-run lsh s3 sync s3://backups/ s3://archive/backups/ --size-only`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/s3/transfer.go b/cmd/s3/transfer.go index afc383c..84a50b9 100644 --- a/cmd/s3/transfer.go +++ b/cmd/s3/transfer.go @@ -1267,7 +1267,7 @@ func addTransferFlags(cmd *cobra.Command, f *transferFlags, recursive bool) { "checksum-algorithm": "the CLI uses Content-MD5; additional checksums are rejected by S3-compatible backends", "checksum-mode": "checksum validation on download is not available", "request-payer": "there is no requester-pays billing on Latitude", - "storage-class": "storage class is a bucket attribute on Latitude (standard|high_performance); set it with lsh s3 mb --storage-class", + "storage-class": "storage class is a bucket attribute on Latitude (standard|high_performance); set it with lsh s3 create-bucket --storage-class", }) if sse := fs.Lookup("sse"); sse != nil { sse.NoOptDefVal = "AES256" diff --git a/cmd/s3/transfer_test.go b/cmd/s3/transfer_test.go index becb6e7..86b379c 100644 --- a/cmd/s3/transfer_test.go +++ b/cmd/s3/transfer_test.go @@ -865,7 +865,7 @@ func TestCommandsRegisterFlags(t *testing.T) { t.Fatal(err) } err := cp.PreRunE(cp, nil) - if exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "lsh s3 mb --storage-class") { + if exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "lsh s3 create-bucket --storage-class") { t.Errorf("--storage-class: %v", err) } } diff --git a/cmd/s3/usage.go b/cmd/s3/usage.go index bd99761..f2482c4 100644 --- a/cmd/s3/usage.go +++ b/cmd/s3/usage.go @@ -340,8 +340,9 @@ func usageStorageIDNames(ctx context.Context, api *sdk.Latitudesh, buckets []*ob // NewUsageCmd builds `lsh s3 usage`. func NewUsageCmd() *cobra.Command { cmd := newCmd(&cobra.Command{ - Use: "usage", - Short: "Daily storage usage history", + Use: "usage", + GroupID: groupReports, + Short: "Day-by-day storage history for a project", Long: `Show the daily storage usage of a project (bytes stored per day), optionally restricted to one bucket and aggregated per bucket, tier or region. diff --git a/internal/objectstorage/errors.go b/internal/objectstorage/errors.go index f85f3db..fb5e99c 100644 --- a/internal/objectstorage/errors.go +++ b/internal/objectstorage/errors.go @@ -56,7 +56,7 @@ func Humanize(err error, b *Bucket, c *Credential) error { endpoint = b.Endpoint backend = b.BucketName } - return exitcode.Errorf(exitcode.NotFound, "bucket %s (backend name %q) was not found at %s; it may still be provisioning — check 'lsh s3 stat %s'", bucketName, backend, endpoint, displayOrBackend(b)) + return exitcode.Errorf(exitcode.NotFound, "bucket %s (backend name %q) was not found at %s; it may still be provisioning — check 'lsh s3 get %s'", bucketName, backend, endpoint, displayOrBackend(b)) case "NoSuchKey", "NotFound": key := resp.Key if key == "" { diff --git a/internal/objectstorage/resolver.go b/internal/objectstorage/resolver.go index b6b1507..ed35131 100644 --- a/internal/objectstorage/resolver.go +++ b/internal/objectstorage/resolver.go @@ -237,7 +237,7 @@ func (r *Resolver) Resolve(ctx context.Context, token string) (*Bucket, error) { if nameMatches > 0 { // The name existed but the --storage-class/--site filters ruled // every candidate out. - return nil, exitcode.Errorf(exitcode.NotFound, "no bucket named %q matches the given filters (%s); run 'lsh s3 ls' to see the buckets", token, r.filterDesc()) + return nil, exitcode.Errorf(exitcode.NotFound, "no bucket named %q matches the given filters (%s); run 'lsh s3 list' to see the buckets", token, r.filterDesc()) } return nil, bucketNotFound(token, r.Project) case 1: diff --git a/internal/objectstorage/uri.go b/internal/objectstorage/uri.go index 489dee2..a799843 100644 --- a/internal/objectstorage/uri.go +++ b/internal/objectstorage/uri.go @@ -189,9 +189,9 @@ func ErrUsagef(format string, a ...interface{}) error { // bucketNotFound formats the "no such bucket" error shared by the resolver. func bucketNotFound(ref, project string) error { if project != "" { - return exitcode.Errorf(exitcode.NotFound, "bucket %q not found in project %s; run 'lsh s3 ls' to see available buckets", ref, project) + return exitcode.Errorf(exitcode.NotFound, "bucket %q not found in project %s; run 'lsh s3 list' to see available buckets", ref, project) } - return exitcode.Errorf(exitcode.NotFound, "bucket %q not found; run 'lsh s3 ls' to see available buckets", ref) + return exitcode.Errorf(exitcode.NotFound, "bucket %q not found; run 'lsh s3 list' to see available buckets", ref) } var _ = fmt.Sprintf