diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 612d290..b4c90d5 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -21,8 +21,11 @@ builds: - env: - CGO_ENABLED=0 + flags: + - -trimpath ldflags: - - -X "github.com/latitudesh/lsh/internal/version.Version={{ .Tag }}" + # -s -w strips symbol and DWARF tables (~30% smaller binaries). + - -s -w -X "github.com/latitudesh/lsh/internal/version.Version={{ .Tag }}" goos: - linux - windows diff --git a/README.md b/README.md index 7d28f26..362809a 100755 --- a/README.md +++ b/README.md @@ -132,6 +132,77 @@ sudo lsh volume mount --id vol_abc123 - The CLI automatically finds your credentials when you run commands with sudo - Volume mount needs sudo for nvme-cli installation and NVMe operations +`lsh volumes` and `lsh filesystems` are aliases of `lsh volume` and `lsh storage-filesystems`. + +## Object storage + +`lsh s3` manages buckets, objects, access keys and lifecycle rules, addressing +buckets and objects as `s3:///`. Bucket administration goes through the +Latitude API; object operations talk to the bucket's S3 endpoint directly, so +endpoint and signing region never have to be configured by hand. + +`` accepts the display name, the `bkt_` ID or the backend bucket name. If +the same display name exists in more than one place, the command lists the +candidates and you narrow it with `--project`, `-c`/`--storage-class`, `--site` +(e.g. `lsh s3 get s3://backups -c high_performance --site TYO4`), or the `bkt_` ID. + +Create a bucket, upload, list and delete (in a terminal, `mb` offers to create +an S3 access key and saves it to your profile): + +```bash +lsh s3 create-bucket s3://backups --region DAL --project +lsh s3 copy ./dump.sql s3://backups/2026/09/ +lsh s3 list s3://backups/2026/09/ --human-readable --summarize +lsh s3 copy s3://backups/2026/09/dump.sql ./restore/ +lsh s3 delete s3://backups/2026/09/dump.sql +``` + +Give an application or CI job its own scoped access key (the secret is shown +once; `-o text --query` extracts it for a secret store): + +```bash +lsh s3 access-keys create --bucket backups=rw --bucket logs=readonly --name ci-deploy +lsh s3 access-keys create --bucket backups=rw --name ci-deploy -o text --query "[0].secret_access_key" | gh secret set LSH_S3_SECRET_ACCESS_KEY +lsh s3 access-keys list +lsh s3 access-keys rotate ci-deploy --delete-old +``` + +Expire objects automatically with lifecycle rules: + +```bash +lsh s3 lifecycle create s3://logs --prefix tmp/ --expiration-days 7 +lsh s3 lifecycle list s3://logs +lsh s3 lifecycle delete s3://logs expire-7d-tmp +``` + +Use the same buckets from rclone, mc, s3cmd or any other S3 client: + +```bash +lsh s3 configure export s3://backups --format env # also: aws | rclone | mc | s3cmd | process +``` + +Clean up safely (`--dry-run` only reads; multi-object deletes ask for +confirmation in a terminal and require `--yes` in CI): + +```bash +lsh s3 delete s3://logs/tmp/ --recursive --dry-run +lsh s3 delete s3://logs/tmp/ --recursive --yes +lsh s3 delete-bucket s3://logs --force --yes +``` + +In CI, object commands authenticate with an S3 access key from the environment +instead of a saved profile: + +| Variable | Purpose | +| --- | --- | +| `LSH_S3_ACCESS_KEY_ID` / `LSH_S3_SECRET_ACCESS_KEY` | S3 access key used by `cp`, `ls`, `rm`, `stat` and `presign` (both required). | +| `LSH_S3_ENDPOINT_URL` | Talk to this S3 endpoint without the API; buckets are then addressed by their backend `bucket_name`. | +| `LSH_S3_SIGNING_REGION` | SigV4 signing region when it cannot be derived from the endpoint. | +| `LSH_S3_USE_AWS_ENV` | Set to `1` to reuse `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`. | + +`lsh help exit-codes` documents the exit codes (0-7, 130) that `lsh s3` returns +for scripts. + ## Output formats & automation Every `list` command can render its results in different formats, so the output @@ -143,11 +214,12 @@ lsh servers list -o table # human-readable table (default) lsh servers list -o json # JSON lsh servers list -o yaml # YAML lsh servers list -o csv # CSV (header + one row per item) +lsh servers list -o text # raw values, tab-separated lsh servers list --json # shortcut for -o json ``` Filter the structured output with a [JMESPath](https://jmespath.org/) expression -via `--query` (works with json/yaml/csv): +via `--query` (works with json/yaml/csv/text): ```bash lsh servers list --query "[?status=='on'].id" -o json @@ -165,8 +237,8 @@ lsh servers list --no-paginate # first page only; next page printed to std | Variable | Purpose | | --- | --- | -| `LSH_OUTPUT` | Default output format (`table`/`json`/`yaml`/`csv`). Precedence: `--output` flag > `LSH_OUTPUT` > config file > default. | -| `LSH_CLASSIC_OUTPUT` | Set to `true` to force the legacy plain-ASCII table. An explicit `-o json/yaml/csv` still wins over it. | +| `LSH_OUTPUT` | Default output format (`table`/`json`/`yaml`/`csv`/`text`). Precedence: `--output` flag > `LSH_OUTPUT` > config file > default. | +| `LSH_CLASSIC_OUTPUT` | Set to `true` to force the legacy plain-ASCII table. An explicit `-o json/yaml/csv/text` still wins over it. | | `LATITUDESH_TOKEN` | API token; bypasses any stored profile (see `lsh help authentication`). | | `LSH_PROFILE` | Use the named profile for the command. | | `LSH_PROJECT` | Pre-fill `--project` so list commands don't prompt. | diff --git a/cli/cli.go b/cli/cli.go index b9f7c94..5a973ab 100755 --- a/cli/cli.go +++ b/cli/cli.go @@ -8,6 +8,7 @@ import ( "github.com/latitudesh/lsh/client" "github.com/latitudesh/lsh/cmd/lsh" servers "github.com/latitudesh/lsh/cmd/servers" + "github.com/latitudesh/lsh/internal/exitcode" "github.com/latitudesh/lsh/internal/pagination" "github.com/latitudesh/lsh/internal/renderer" "github.com/latitudesh/lsh/internal/version" @@ -70,10 +71,15 @@ func MakeRootCmd(rootCmd *cobra.Command) (*cobra.Command, error) { // Dedicated group so help topics show up clearly in `lsh --help`. rootCmd.AddGroup(&cobra.Group{ID: helpTopicsGroupID, Title: "Help topics:"}) + // Storage products (object storage, filesystems, volumes) share a section + // so they stay discoverable together in `lsh --help`. + rootCmd.AddGroup(&cobra.Group{ID: StorageGroupID, Title: "Storage:"}) rootCmd.AddCommand(makeHelpAuthenticationCmd()) rootCmd.AddCommand(makeHelpProfilesCmd()) rootCmd.AddCommand(makeHelpAutomationCmd()) rootCmd.AddCommand(makeHelpOutputFormatsCmd()) + rootCmd.AddCommand(makeHelpExitCodesCmd()) + rootCmd.AddCommand(makeHelpS3Cmd()) // Re-resolve the active profile once flags have been parsed so that // `--profile ` overrides LSH_PROFILE / default_profile for the @@ -88,11 +94,20 @@ func MakeRootCmd(rootCmd *cobra.Command) (*cobra.Command, error) { // Validate output/query/pagination selection up front so commands fail // fast with an actionable message (and a non-zero exit) instead of // silently falling back or clamping. + // These are command-line errors. Groups that opted into the documented + // exit codes report them as usage (2); the older groups keep exiting 1 + // for every failure, so their scripts are unaffected. + usageCode := func(err error) error { + if err == nil || !usesExitCodes(cmd) { + return err + } + return exitcode.New(exitcode.Usage, err) + } if err := renderer.ValidateOutputSelection(); err != nil { - return err + return usageCode(err) } if err := pagination.Validate(); err != nil { - return err + return usageCode(err) } // Hydrate the active profile into viper for commands that authenticate // against the API. Skip the login/auth/profile subtree: there --profile @@ -119,7 +134,7 @@ func MakeRootCmd(rootCmd *cobra.Command) (*cobra.Command, error) { viper.BindPFlag("base_path", rootCmd.PersistentFlags().Lookup("base-path")) var outputFlag string - rootCmd.PersistentFlags().StringVarP(&outputFlag, "output", "o", "table", "output format: table | json | yaml | csv") + rootCmd.PersistentFlags().StringVarP(&outputFlag, "output", "o", "table", "output format: table | json | yaml | csv | text") viper.BindPFlag("output", rootCmd.PersistentFlags().Lookup("output")) // LSH_OUTPUT sets a per-user default format. viper precedence is // flag > env > config > default, which is exactly what PD-6072 requires. @@ -131,7 +146,7 @@ func MakeRootCmd(rootCmd *cobra.Command) (*cobra.Command, error) { // Global automation controls. --query post-processes structured output with // a JMESPath expression; the pagination flags govern every `list` command. - rootCmd.PersistentFlags().String("query", "", "filter json/yaml/csv output with a JMESPath expression (see 'lsh help output-formats')") + rootCmd.PersistentFlags().String("query", "", "filter json/yaml/csv/text output with a JMESPath expression (see 'lsh help output-formats')") viper.BindPFlag("query", rootCmd.PersistentFlags().Lookup("query")) rootCmd.PersistentFlags().Int64("page-size", pagination.DefaultPageSize, "items to request per API page") @@ -561,9 +576,11 @@ func makeOperationGroupVirtualNetworksCmd() (*cobra.Command, error) { func makeOperationGroupVolumeCmd() (*cobra.Command, error) { operationGroupVolumeCmd := &cobra.Command{ - Use: "volume", - Short: "Manage volumes", - Long: `Commands to manage volume operations such as listing, mounting, creating, and deleting volumes`, + Use: "volume", + Aliases: []string{"volumes"}, + GroupID: StorageGroupID, + Short: "Manage volumes", + Long: `Commands to manage volume operations such as listing, mounting, creating, and deleting volumes`, } operationVolumeListCmd, err := makeOperationVolumeListCmd() @@ -598,3 +615,16 @@ func makeOperationGroupVolumeCmd() (*cobra.Command, error) { return operationGroupVolumeCmd, nil } + +// usesExitCodes reports whether cmd belongs to a subtree that opted into the +// documented exit codes (see exitcode.OptInAnnotation). The annotation is set +// on every command of such a group, but the walk keeps it working if only the +// group root carries it. +func usesExitCodes(cmd *cobra.Command) bool { + for c := cmd; c != nil; c = c.Parent() { + if c.Annotations[exitcode.OptInAnnotation] == "true" { + return true + } + } + return false +} diff --git a/cli/help_topics.go b/cli/help_topics.go index 7e511c5..62071de 100644 --- a/cli/help_topics.go +++ b/cli/help_topics.go @@ -6,6 +6,12 @@ import "github.com/spf13/cobra" // commands under a dedicated section of `lsh --help`. const helpTopicsGroupID = "help-topics" +// StorageGroupID groups the storage command groups (s3, storage-filesystems, +// volume) under a "Storage:" heading in `lsh --help`. Commands are added in +// init() before MakeRootCmd registers the group, which Cobra allows as long as +// the group exists by the time Execute runs. +const StorageGroupID = "storage" + // newHelpTopic returns a Cobra command that exists purely to host // long-form documentation. Running it (with or without `help`) prints // the topic content; it has no subcommands and no side effects. @@ -124,6 +130,54 @@ error: "--project is required ...". The caller can recover by listing projects first and retrying: lsh --no-input projects list -o json + +Confirmations (--yes / --no-input) + +Destructive object storage commands (lsh s3 delete --recursive, rb --force, +access-keys delete, lifecycle delete --all) ask for confirmation when run +in a terminal. The contract for automation is: + + --yes skip the confirmation and proceed + --no-input never prompt; a command that would have asked fails with + exit 7 instead of hanging (stdin not being a TTY has the + same effect) + +Prompts and progress go to stderr; stdout carries only data, so '-o json' +and '-o text' remain pipeable. A declined or refused confirmation never +exits 0. See 'lsh help exit-codes' for the full table. + +Object storage credentials + +Object commands (lsh s3 copy/ls/rm/stat/presign) authenticate with an S3 +access key, not with the API token. In CI, export one of the following: + + LSH_S3_ACCESS_KEY_ID S3 access key id (both variables are required; + LSH_S3_SECRET_ACCESS_KEY setting only one fails with exit 4) + LSH_S3_ENDPOINT_URL talk to this S3 endpoint without the API; + buckets are then addressed by their backend + bucket_name (see 'lsh s3 configure export') + LSH_S3_SIGNING_REGION SigV4 signing region when it cannot be derived + from the endpoint + LSH_S3_USE_AWS_ENV=1 opt in to reuse AWS_ACCESS_KEY_ID and + AWS_SECRET_ACCESS_KEY (ignored when + AWS_SESSION_TOKEN is set) + + # Resolve s3:// through the API, sign with the env key + LATITUDESH_TOKEN=ak_xxx LSH_S3_ACCESS_KEY_ID=... LSH_S3_SECRET_ACCESS_KEY=... \ + lsh s3 copy ./dump.sql s3://backups/2026/09/ + + # No API at all: endpoint + backend bucket name + LSH_S3_ENDPOINT_URL=https://s3.us-central-1.storage.sh \ + LSH_S3_ACCESS_KEY_ID=... LSH_S3_SECRET_ACCESS_KEY=... \ + lsh s3 list s3://backups-7f3a/ + + # Hand a scoped key to another job (the secret is printed once) + lsh s3 access-keys create --bucket backups=rw --name ci -o text \ + --query "[0].secret_access_key" | gh secret set LSH_S3_SECRET_ACCESS_KEY + +Precedence: LSH_S3_* environment > --access-key > the best +saved key of the active profile ('lsh s3 configure'). Without any of them +the command exits 4 with the commands that fix it. `, ) } @@ -131,7 +185,7 @@ projects first and retrying: func makeHelpOutputFormatsCmd() *cobra.Command { return newHelpTopic( "output-formats", - "Render results as table, JSON, YAML or CSV (with JMESPath queries)", + "Render results as table, JSON, YAML, CSV or text (with JMESPath queries)", `lsh — Output formats By default, lsh prints a human-readable table. Use --output (or -o) to switch @@ -141,6 +195,7 @@ to a machine-readable format for scripts, pipelines and AI agents: lsh servers list -o json # raw JSON lsh servers list -o yaml # YAML lsh servers list -o csv # CSV (header + one row per item) + lsh servers list -o text # raw values, tab-separated lsh servers list --json # shortcut for -o json # Set a per-user default without passing -o every time. @@ -149,13 +204,13 @@ to a machine-readable format for scripts, pipelines and AI agents: lsh servers list # prints JSON # Force the legacy plain-ASCII table (e.g. CI that parses fixed columns). - # An explicit -o json/yaml/csv still wins over this. + # An explicit -o json/yaml/csv/text still wins over this. LSH_CLASSIC_OUTPUT=true lsh servers list Filtering with --query (JMESPath) -The --query flag post-processes structured output (json/yaml/csv) with a -JMESPath expression — no jq or extra tooling required: +The --query flag post-processes structured output (json/yaml/csv/text) +with a JMESPath expression — no jq or extra tooling required: # Only the IDs of servers that are powered on lsh servers list --query '[?status==`+"`on`"+`].id' -o json @@ -163,7 +218,21 @@ JMESPath expression — no jq or extra tooling required: # A projection of selected fields lsh servers list --query '[].{id: id, host: hostname}' -o yaml ---query requires a structured format; combine it with -o json, yaml or csv. + # A single raw value, ready for another tool (no quotes, no JSON) + lsh s3 access-keys create --bucket backups --name ci -o text \ + --query "[0].secret_access_key" | gh secret set LSH_S3_SECRET_ACCESS_KEY + +--query requires a structured format; combine it with -o json, yaml, csv or +text. + +The text format + +-o text prints values without quotes or structure: +a scalar on one line; a list of scalars one per line; a list of objects as +one tab-separated row per item with keys in sorted order; a single object +as keyvalue lines. Nested values are JSON-encoded so a row never spans +several lines. Use it with --query to extract exactly one field for a shell +variable or a pipe. Pagination @@ -179,3 +248,102 @@ List commands fetch every page by default. These flags give you control: `, ) } + +func makeHelpExitCodesCmd() *cobra.Command { + return newHelpTopic( + "exit-codes", + "Process exit codes for scripts and CI", + `lsh — Exit codes + +The object storage commands ('lsh s3' and its subcommands) attach a +specific exit code to every failure so scripts can tell "not found" from +"no credentials" from "refused for safety" without parsing stderr. + + Code Meaning + ---- ----------------------------------------------------------- + 0 success (including an empty listing) + 1 generic error, or one or more transfers failed + 2 invalid usage: bad URI or flag, ambiguous bucket, + --recursive on a whole bucket without --all + 3 not found: bucket, object, access key or lifecycle rule + 4 credentials missing or invalid (no S3 access key, + InvalidAccessKeyId, SignatureDoesNotMatch) + 5 permission denied (403 from the API or the S3 endpoint) + 6 partial success: some objects failed in rm --recursive + or rb --force; the remaining ones are listed on stderr + 7 refused for safety: non-empty bucket without --force, + --max-delete exceeded, object lock retention, prompt + declined, or a confirmation needed without a TTY and + without --yes + 130 interrupted (Ctrl-C / SIGINT) after in-flight work was + aborted + +Errors are always printed to stderr; stdout carries only data, so +'-o json' and '-o text' output stays pipeable even when a command fails. + + lsh s3 copy ./dump.sql s3://backups/ || case $? in + 4) echo "configure an access key: lsh s3 configure" ;; + 7) echo "refused; add --yes in CI" ;; + esac + +Older command groups (servers, projects, plans, ...) predate this table +and still exit 1 for every error. Their behaviour is unchanged; only +'lsh s3' uses the codes above. +`, + ) +} + +func makeHelpS3Cmd() *cobra.Command { + return newHelpTopic( + "object-storage", + "How object storage addressing, endpoints and access keys work", + `lsh — Object storage + +The commands live under 'lsh s3' ('lsh s3 --help' lists them). + +Two planes + Buckets, access keys, lifecycle rules, metrics and usage are managed + through the Latitude API with your API token ('lsh login'). + Objects (list, copy, move, delete, get, presign, sync) are read and + written on the bucket's own S3 endpoint with an S3 access key, which is a + separate credential the API returns exactly once when the key is created. + +Addressing + Buckets and objects are written as s3://[/]. may be + the display name, the bkt_ ID or the backend bucket name. When the same + display name exists more than once (another project, storage class or + site), the command lists the candidates; narrow it with --project, + -c/--storage-class or --site, or use the bkt_ ID. + + The endpoint, the SigV4 signing region and path-style addressing are + derived from the bucket, so none of them is configured by hand: + standard https://s3..storage.sh + high_performance https://objects..storage.sh (bound to one site) + +Access keys + A key is either fullaccess (every bucket of a storage class in a project — + and of one site, for high_performance) or limited_access (specific buckets, + rw or readonly). Keys saved in the active profile are picked automatically + per bucket, preferring the least-privileged one that covers it. + + For the machine you are on: lsh s3 configure + For an app, CI job or someone: lsh s3 access-keys create --bucket =rw + Reuse elsewhere: lsh s3 configure export s3:// --format env + + Precedence: LSH_S3_ACCESS_KEY_ID + LSH_S3_SECRET_ACCESS_KEY (environment) + > --access-key > the best saved key of the active profile. + Without any of them the command exits 4 with the commands that fix it. + +Without the API + --endpoint-url (or LSH_S3_ENDPOINT_URL) talks to an S3 endpoint directly: + is then the backend bucket name and credentials come only from + the environment. --signing-region overrides the region when it cannot be + derived from the endpoint. + +Safety + Deleting several objects asks for confirmation in a terminal and needs + --yes in scripts; --dry-run prints the plan without writing anything; + --max-delete caps a recursive deletion. Exit codes: 'lsh help exit-codes'. +`, + ) +} diff --git a/cli/operating_systems.go b/cli/operating_systems.go index c9247c1..8ecf5e2 100644 --- a/cli/operating_systems.go +++ b/cli/operating_systems.go @@ -78,7 +78,7 @@ func runOperatingSystemsList(_ *cobra.Command, _ []string) error { pageSize := page.PageSize pageNumber := int64(1) - resp, err := client.OperatingSystems.ListPlans(ctx, &pageSize, &pageNumber) + resp, err := client.OperatingSystems.ListPlans(ctx, &pageSize, &pageNumber, nil) if err != nil { stopSpinner() utils.PrintError(err) diff --git a/cli/project_flag.go b/cli/project_flag.go index 8604391..d5d4ed7 100644 --- a/cli/project_flag.go +++ b/cli/project_flag.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/latitudesh/lsh/internal/exitcode" "github.com/latitudesh/lsh/internal/prompt" "github.com/latitudesh/lsh/internal/util" "github.com/spf13/cobra" @@ -86,3 +87,44 @@ func resolveProjectFlag(cmd *cobra.Command) error { } return cmd.Flags().Set("project", selected) } + +// PickProjectForList resolves the project scope for a "list" command that can +// also run across every project. Precedence: an explicit --project flag, +// LSH_PROJECT, --all-projects, then — in an interactive terminal — a project +// picker that includes an "All projects" entry. +// +// A non-interactive session (or --no-input) has no picker to show, so it keeps +// the behaviour these listings always had and covers every project. Failing +// there instead would break existing scripts (including the ones calling the +// legacy `storage-objects list`) for a prompt they could never have answered. +// +// It returns the chosen project (id or slug, empty when "all") and whether the +// user opted into all projects. The command owns the flags "project", +// "all-projects" and "no-input". +func PickProjectForList(cmd *cobra.Command) (project string, allProjects bool, err error) { + if v, _ := cmd.Flags().GetString("project"); v != "" { + return v, false, nil + } + if env := os.Getenv("LSH_PROJECT"); env != "" { + return env, false, nil + } + if all, _ := cmd.Flags().GetBool("all-projects"); all { + return "", true, nil + } + noInput, _ := cmd.Flags().GetBool("no-input") + if noInput || !isInteractive() { + return "", true, nil + } + token := viper.GetString("Authorization") + if token == "" { + return "", false, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login' first") + } + selected, err := prompt.SelectProject(cmd.Context(), newAuthClient(), token, true) + if err != nil { + return "", false, err + } + if selected == prompt.AllProjectsSentinel { + return "", true, nil + } + return selected, false, nil +} diff --git a/cli/regions.go b/cli/regions.go index b8924f5..4d5921b 100644 --- a/cli/regions.go +++ b/cli/regions.go @@ -77,7 +77,7 @@ func runRegionsList(_ *cobra.Command, _ []string) error { pageSize := page.PageSize pageNumber := int64(1) - resp, err := client.Regions.Get(ctx, &pageSize, &pageNumber) + resp, err := client.Regions.Get(ctx, operations.GetRegionsRequest{PageSize: &pageSize, PageNumber: &pageNumber}) if err != nil { stopSpinner() utils.PrintError(err) diff --git a/cli/server_wait.go b/cli/server_wait.go index 6b8dfcf..0a9b690 100644 --- a/cli/server_wait.go +++ b/cli/server_wait.go @@ -6,7 +6,6 @@ import ( "fmt" "os" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" "github.com/latitudesh/lsh/client/servers" "github.com/latitudesh/lsh/cmd/lsh" @@ -21,7 +20,7 @@ import ( // The create/reinstall calls still go through the legacy client; the wait loop // polls via the SDK (Servers.Get). Progress and outcome are written to stderr // so they never corrupt structured (-o json) output on stdout. -func waitForServerState(cmd *cobra.Command, serverID string, want, fail []components.ServerDataStatus) error { +func waitForServerState(cmd *cobra.Command, serverID string, want, fail []wait.ServerStatus) error { o := wait.OptionsFrom(cmd) if !o.Enabled { if cmd.Flags().Changed("timeout") { @@ -85,10 +84,10 @@ func renderServerState(cmd *cobra.Command, serverID string) { // Provisioning is "done" once the server settles into a stable power state — // it may finish either powered on or off — and "failed" on a failed deployment. // The in-progress states (deploying, disk_erasing) keep the wait polling. -func serverProvisionTargets() (want, fail []components.ServerDataStatus) { - return []components.ServerDataStatus{ - components.ServerDataStatusOn, - components.ServerDataStatusOff, +func serverProvisionTargets() (want, fail []wait.ServerStatus) { + return []wait.ServerStatus{ + wait.ServerStatusOn, + wait.ServerStatusOff, }, - []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} + []wait.ServerStatus{wait.ServerStatusFailedDeployment} } diff --git a/cli/teams_members.go b/cli/teams_members.go index 53f9139..b8d2af6 100644 --- a/cli/teams_members.go +++ b/cli/teams_members.go @@ -128,7 +128,7 @@ func runTeamMembersList(_ *cobra.Command, _ []string) error { pageSize := page.PageSize pageNumber := int64(1) - resp, err := client.Teams.Members.GetTeamMembers(ctx, &pageSize, &pageNumber) + resp, err := client.Teams.Members.GetTeamMembers(ctx, &pageSize, &pageNumber, nil) if err != nil { stopSpinner() utils.PrintError(err) @@ -339,7 +339,7 @@ func selectTeamMember(ctx context.Context, client *latitudeshgosdk.Latitudesh) ( // output-pagination flags (--max-items / --no-paginate), so it walks all // pages itself using the default page size. pageSize := pagination.DefaultPageSize - resp, err := client.Teams.Members.GetTeamMembers(ctx, &pageSize, nil) + resp, err := client.Teams.Members.GetTeamMembers(ctx, &pageSize, nil, nil) if err != nil { return teamMemberChoice{}, err } diff --git a/cmd/build_s3.go b/cmd/build_s3.go new file mode 100644 index 0000000..a23b075 --- /dev/null +++ b/cmd/build_s3.go @@ -0,0 +1,33 @@ +package cmd + +import ( + s3 "github.com/latitudesh/lsh/cmd/s3" +) + +// s3Cmd is the object storage group (`lsh s3`, alias `object-storage`). It +// replaces cmd/storage_objects; the old name survives as a hidden, deprecated +// command that re-dispatches here (see s3.NewLegacyAliasCmd). +var s3Cmd = s3.NewGroupCmd() + +func init() { + s3Cmd.AddCommand(s3.NewLsCmd()) + s3Cmd.AddCommand(s3.NewMbCmd()) + s3Cmd.AddCommand(s3.NewRbCmd()) + s3Cmd.AddCommand(s3.NewStatCmd()) + s3Cmd.AddCommand(s3.NewCpCmd()) + s3Cmd.AddCommand(s3.NewMvCmd()) + s3Cmd.AddCommand(s3.NewRmCmd()) + s3Cmd.AddCommand(s3.NewSyncCmd()) + s3Cmd.AddCommand(s3.NewPresignCmd()) + s3Cmd.AddCommand(s3.NewConfigureCmd()) + s3Cmd.AddCommand(s3.NewMetricsCmd()) + s3Cmd.AddCommand(s3.NewUsageCmd()) + s3Cmd.AddCommand(s3.NewAccessKeysCmd()) + s3Cmd.AddCommand(s3.NewLifecycleCmd()) + // Install the shared error/exit-code contract on the whole tree now that + // every subcommand (and its late-assigned RunE/PreRunE) is in place. + s3.Finalize(s3Cmd) + + rootCmd.AddCommand(s3Cmd) + rootCmd.AddCommand(s3.NewLegacyAliasCmd(rootCmd)) +} diff --git a/cmd/build_storage_filesystems.go b/cmd/build_storage_filesystems.go index 0e64617..078c66b 100644 --- a/cmd/build_storage_filesystems.go +++ b/cmd/build_storage_filesystems.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/latitudesh/lsh/cli" storage_filesystems "github.com/latitudesh/lsh/cmd/storage_filesystems" cobra "github.com/spf13/cobra" ) @@ -14,9 +15,11 @@ func init() { } var storageFilesystemsCmd = &cobra.Command{ - Use: "storage-filesystems", - Short: "Manage filesystem storage", - Long: "Manage filesystem storage: list, resize and delete filesystems.", + Use: "storage-filesystems", + Aliases: []string{"filesystems"}, + GroupID: cli.StorageGroupID, + Short: "Manage filesystem storage", + Long: "Manage filesystem storage: list, resize and delete filesystems.", Example: ` lsh storage-filesystems list --project my-project lsh storage-filesystems update fs_xxxxxxxx --size 3000 lsh storage-filesystems delete fs_xxxxxxxx`, diff --git a/cmd/build_storage_objects.go b/cmd/build_storage_objects.go deleted file mode 100644 index ae1f2cb..0000000 --- a/cmd/build_storage_objects.go +++ /dev/null @@ -1,25 +0,0 @@ -package cmd - -import ( - storage_objects "github.com/latitudesh/lsh/cmd/storage_objects" - cobra "github.com/spf13/cobra" -) - -func init() { - storageObjectsCmd.AddCommand(storage_objects.NewListCmd()) - storageObjectsCmd.AddCommand(storage_objects.NewGetCmd()) - storageObjectsCmd.AddCommand(storage_objects.NewCreateCmd()) - storageObjectsCmd.AddCommand(storage_objects.NewDeleteCmd()) - - rootCmd.AddCommand(storageObjectsCmd) -} - -var storageObjectsCmd = &cobra.Command{ - Use: "storage-objects", - Short: "Manage object storage buckets", - Long: "Manage object storage buckets: list, get, create and delete S3-compatible buckets.", - Example: ` lsh storage-objects list --project my-project - lsh storage-objects create --project my-project --name my-bucket --region SAO2 - lsh storage-objects get bucket_xxxxxxxx - lsh storage-objects delete bucket_xxxxxxxx`, -} diff --git a/cmd/elasticips/create.go b/cmd/elasticips/create.go index ded49c7..e191216 100644 --- a/cmd/elasticips/create.go +++ b/cmd/elasticips/create.go @@ -49,9 +49,9 @@ func buildCreateRequest(cmd *cobra.Command) (components.CreateElasticIP, error) request := components.CreateElasticIP{ Data: components.CreateElasticIPData{ Type: components.CreateElasticIPTypeElasticIps, - Attributes: components.CreateElasticIPAttributes{ + Attributes: &components.CreateElasticIPAttributes{ ProjectID: project, - ServerID: server, + ServerID: &server, }, }, } diff --git a/cmd/elasticips/elasticips_test.go b/cmd/elasticips/elasticips_test.go index 09ea094..e748165 100644 --- a/cmd/elasticips/elasticips_test.go +++ b/cmd/elasticips/elasticips_test.go @@ -71,8 +71,8 @@ func TestBuildCreateRequest(t *testing.T) { if request.Data.Attributes.ProjectID != "my-project" { t.Errorf("project_id = %q, want my-project", request.Data.Attributes.ProjectID) } - if request.Data.Attributes.ServerID != "sv_x" { - t.Errorf("server_id = %q, want sv_x", request.Data.Attributes.ServerID) + if request.Data.Attributes.ServerID == nil || *request.Data.Attributes.ServerID != "sv_x" { + t.Errorf("server_id = %v, want sv_x", request.Data.Attributes.ServerID) } } diff --git a/cmd/firewalls/assignments.go b/cmd/firewalls/assignments.go index bf0349d..f133bb4 100644 --- a/cmd/firewalls/assignments.go +++ b/cmd/firewalls/assignments.go @@ -91,7 +91,7 @@ func (o *ListAssignmentsOperation) run(cmd *cobra.Command, args []string) error value, _ := cmd.Flags().GetString("server") filterServer = &value } - response, err := client.Firewalls.GetAllFirewallAssignments(ctx, filterServer, nil, nil, operations.WithRetries(lsh.RetryConfig())) + response, err := client.Firewalls.GetAllFirewallAssignments(ctx, filterServer, nil, nil, nil, operations.WithRetries(lsh.RetryConfig())) if err != nil { utils.PrintError(err) return err @@ -149,7 +149,7 @@ func buildCreateAssignmentRequest(cmd *cobra.Command) (firewallID string, body o Data: operations.CreateFirewallAssignmentFirewallsAssignmentsData{ Type: operations.CreateFirewallAssignmentFirewallsAssignmentsTypeFirewallAssignments, Attributes: &operations.CreateFirewallAssignmentFirewallsAssignmentsAttributes{ - ServerID: serverID, + ServerID: &serverID, }, }, } @@ -189,11 +189,13 @@ func (o *CreateAssignmentOperation) run(cmd *cobra.Command, args []string) error // current SDK model does not map (its fields come back empty). Fall back // to the request inputs so the confirmation shows the association that // was created instead of a blank row. - if fs.Attributes == nil || getStr(fs.Attributes.FirewallID) == "" { + if fs.Data == nil || fs.Data.Attributes == nil || getStr(fs.Data.Attributes.FirewallID) == "" { serverID, _ := cmd.Flags().GetString("server") - fs.Attributes = &components.FirewallServerAttributes{ - FirewallID: &firewallID, - ServerID: &serverID, + fs.Data = &components.FirewallServerData{ + Attributes: &components.FirewallServerAttributes{ + FirewallID: &firewallID, + Server: &components.FirewallServerServer{ID: &serverID}, + }, } } assignment := FirewallServerAssignment{FirewallServer: fs} diff --git a/cmd/firewalls/create.go b/cmd/firewalls/create.go index 2371875..eae91e0 100644 --- a/cmd/firewalls/create.go +++ b/cmd/firewalls/create.go @@ -119,9 +119,9 @@ func buildCreateFirewallRequest(cmd *cobra.Command) (*operations.CreateFirewallF } request := operations.CreateFirewallFirewallsRequestBody{ - Data: operations.CreateFirewallData{ - Type: operations.CreateFirewallTypeFirewalls, - Attributes: &operations.CreateFirewallAttributes{ + Data: operations.CreateFirewallFirewallsData{ + Type: operations.CreateFirewallFirewallsTypeFirewalls, + Attributes: &operations.CreateFirewallFirewallsAttributes{ Name: name, Project: project, Rules: rules, diff --git a/cmd/firewalls/firewall.go b/cmd/firewalls/firewall.go index 87dc59f..2792bae 100644 --- a/cmd/firewalls/firewall.go +++ b/cmd/firewalls/firewall.go @@ -171,16 +171,21 @@ func (m *FirewallServerAssignment) GetData() []renderer.ResponseData { } func (m *FirewallServerAssignment) TableRow() table.Row { - var firewallID, serverID string - if attr := m.Attributes; attr != nil { - firewallID = getStr(attr.FirewallID) - serverID = getStr(attr.ServerID) + var id, firewallID, serverID string + if d := m.Data; d != nil { + id = getStr(d.ID) + if attr := d.Attributes; attr != nil { + firewallID = getStr(attr.FirewallID) + if attr.Server != nil { + serverID = getStr(attr.Server.ID) + } + } } return table.Row{ "id": table.Cell{ Label: "ID", - Value: table.String(getStr(m.ID)), + Value: table.String(id), }, "firewall_id": table.Cell{ Label: "Firewall ID", diff --git a/cmd/firewalls/firewalls_test.go b/cmd/firewalls/firewalls_test.go index 0501899..03086c3 100644 --- a/cmd/firewalls/firewalls_test.go +++ b/cmd/firewalls/firewalls_test.go @@ -165,7 +165,7 @@ func TestBuildCreateFirewallRequest(t *testing.T) { t.Fatalf("buildCreateFirewallRequest returned error: %v", err) } - if request.Data.Type != operations.CreateFirewallTypeFirewalls { + if request.Data.Type != operations.CreateFirewallFirewallsTypeFirewalls { t.Errorf("type = %v, want firewalls", request.Data.Type) } if request.Data.Attributes.Name != "web" { @@ -262,7 +262,7 @@ func TestBuildCreateAssignmentRequest(t *testing.T) { if body.Data.Type != operations.CreateFirewallAssignmentFirewallsAssignmentsTypeFirewallAssignments { t.Errorf("type = %v, want firewall_assignments", body.Data.Type) } - if body.Data.Attributes == nil || body.Data.Attributes.ServerID != "sv_x" { + if body.Data.Attributes == nil || body.Data.Attributes.ServerID == nil || *body.Data.Attributes.ServerID != "sv_x" { t.Errorf("server_id = %v, want sv_x", body.Data.Attributes) } } diff --git a/cmd/firewalls/list.go b/cmd/firewalls/list.go index ce86902..e59f328 100644 --- a/cmd/firewalls/list.go +++ b/cmd/firewalls/list.go @@ -46,7 +46,7 @@ func (o *ListFirewallsOperation) run(cmd *cobra.Command, args []string) error { filterProject = &value } - response, err := client.Firewalls.List(ctx, filterProject, nil, nil, operations.WithRetries(lsh.RetryConfig())) + response, err := client.Firewalls.List(ctx, filterProject, nil, nil, nil, operations.WithRetries(lsh.RetryConfig())) if err != nil { utils.PrintError(err) return err diff --git a/cmd/projectsshkeys/create.go b/cmd/projectsshkeys/create.go index 05f0f30..9254c46 100644 --- a/cmd/projectsshkeys/create.go +++ b/cmd/projectsshkeys/create.go @@ -54,8 +54,8 @@ func (o *CreateProjectSSHKeyOperation) run(cmd *cobra.Command, args []string) er Data: operations.PostProjectSSHKeyProjectsSSHKeysData{ Type: operations.PostProjectSSHKeyProjectsSSHKeysTypeSSHKeys, Attributes: &operations.PostProjectSSHKeyProjectsSSHKeysAttributes{ - Name: &name, - PublicKey: &publicKey, + Name: name, + PublicKey: publicKey, }, }, } diff --git a/cmd/projectsshkeys/projectsshkeys_test.go b/cmd/projectsshkeys/projectsshkeys_test.go index 823237e..b37656c 100644 --- a/cmd/projectsshkeys/projectsshkeys_test.go +++ b/cmd/projectsshkeys/projectsshkeys_test.go @@ -66,15 +66,15 @@ func TestCreateBuildsRequestBody(t *testing.T) { Data: operations.PostProjectSSHKeyProjectsSSHKeysData{ Type: operations.PostProjectSSHKeyProjectsSSHKeysTypeSSHKeys, Attributes: &operations.PostProjectSSHKeyProjectsSSHKeysAttributes{ - Name: &name, - PublicKey: &publicKey, + Name: name, + PublicKey: publicKey, }, }, } if request.Data.Type != "ssh_keys" { t.Errorf("type = %q, want ssh_keys", request.Data.Type) } - if *request.Data.Attributes.Name != name { + if request.Data.Attributes.Name != name { t.Error("name not wired through") } } diff --git a/cmd/s3/accesskeys.go b/cmd/s3/accesskeys.go new file mode 100644 index 0000000..21b6497 --- /dev/null +++ b/cmd/s3/accesskeys.go @@ -0,0 +1,47 @@ +package s3 + +import ( + cobra "github.com/spf13/cobra" +) + +// NewAccessKeysCmd builds `lsh s3 access-keys` (alias `keys`): the S3 +// credentials of a project, managed through the Latitude API and optionally +// saved in the local profile so the data-plane commands pick them up. +func NewAccessKeysCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "access-keys", + Aliases: []string{"keys", "access-key"}, + GroupID: groupCredentials, + Short: "Manage access keys (create, list, rotate, delete)", + Long: `Manage the S3 access keys of your object storage buckets. + +To set up the machine you are on, 'lsh s3 configure' asks the questions and +saves the key for you; the commands here are for keys handed to applications, +CI jobs or other people, and for auditing what exists. + +Access keys are separate from your API token: the API creates them, but the +secret is returned once and never again. A key is either fullaccess (every +bucket of the project in that storage class, and site for high_performance) +or limited_access (specific buckets with rw or readonly permission). + +Keys can be saved in the active lsh profile (--save, or --save-as ); +'lsh s3 copy/ls/rm' then pick the least-privileged saved key that covers the +bucket automatically. +Saved keys never leave this machine except towards the bucket's endpoint.`, + Example: ` lsh s3 access-keys list + lsh s3 access-keys list --saved + lsh s3 access-keys create --bucket backups=rw --bucket logs=readonly --name ci-deploy + lsh s3 access-keys create --all-buckets --storage-class standard --project my-project --save + lsh s3 access-keys rotate ci-deploy --delete-old + lsh s3 access-keys delete ci-deploy --yes + echo "$SECRET" | lsh s3 access-keys import --name legacy --access-key-id AKIA... --project my-project`, + } + cmd.AddCommand(newAccessKeysListCmd()) + cmd.AddCommand(newAccessKeysGetCmd()) + cmd.AddCommand(newAccessKeysCreateCmd()) + cmd.AddCommand(newAccessKeysDeleteCmd()) + cmd.AddCommand(newAccessKeysRotateCmd()) + cmd.AddCommand(newAccessKeysImportCmd()) + cmd.AddCommand(newAccessKeysForgetCmd()) + return cmd +} diff --git a/cmd/s3/accesskeys_create.go b/cmd/s3/accesskeys_create.go new file mode 100644 index 0000000..149ea97 --- /dev/null +++ b/cmd/s3/accesskeys_create.go @@ -0,0 +1,554 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysCreateCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "create", + Aliases: []string{"new", "add"}, + Short: "Create an access key for specific buckets or for all buckets of a class", + Long: `Create an S3 access key through the Latitude API. + +Scope is either specific buckets (--bucket, see its help for the grammar) or +every bucket of the project in one storage class (--all-buckets). All --bucket +values must share storage class, project and, for high_performance, site: one +key cannot span backends. Project, class and site are inferred from the buckets +when possible. + +The secret is returned once by the API and cannot be retrieved again: + - human output without --save prints it in clear, once; + - --save stores the key in the active profile (under the key name, or the + --save-as name) and does not print the secret (add --show-secret to print + it too); a name already holding another key gets a -2, -3… suffix; + - -o json|yaml|csv includes secret_access_key only when -o was passed + explicitly or with --show-secret (never because of LSH_OUTPUT/config). + +--like copies class, site, project and scope from a saved key so a +new bucket can be added to an existing scope (the API has no key update).`, + Example: ` lsh s3 access-keys create --bucket backups=rw --bucket logs=readonly --name ci-deploy + lsh s3 access-keys create --bucket backups --save + lsh s3 access-keys create --all-buckets --storage-class standard --project my-project --name ops --save-as ops + lsh s3 access-keys create --all-buckets --storage-class high_performance --region TYO4 --project my-project + lsh s3 access-keys create --bucket backups=rw --name ci-deploy -o json | jq -r .secret_access_key + lsh s3 access-keys create --like ci-deploy --bucket new-bucket=rw --name ci-deploy-v2`, + Args: cobra.NoArgs, + RunE: runAccessKeysCreate, + }) + addProjectFlag(cmd, true, "project of the key (ID or slug); inferred from --bucket when omitted") + cmd.Flags().StringArray("bucket", nil, bucketSpecUsage) + cmd.Flags().Bool("all-buckets", false, "grant access to every bucket of the project in the storage class (fullaccess)") + cmd.Flags().StringP("storage-class", "c", "", "storage class of the key: standard or high_performance (inferred from --bucket)") + cmd.Flags().String("region", "", "site of the key (e.g. TYO4); required for high_performance when it cannot be inferred") + cmd.Flags().String("name", "", "key name (default lsh--); normalized server-side") + cmd.Flags().Bool("save", false, "save the key in the active profile under the key name (the secret is then not printed)") + cmd.Flags().String("save-as", "", "save the key in the active profile under this name (implies --save)") + cmd.Flags().Bool("show-secret", false, "print the secret even when saving or when -o comes from the environment") + cmd.Flags().String("like", "", "copy storage class, site, project and scope from this saved key") + return cmd +} + +// createOptions are the parsed flags of create (also used by rotate/mb). +type createOptions struct { + BucketSpecs []bucketSpec + AllBuckets bool + StorageClass string + Site string + Project string + Name string + Save bool + SaveName string + ShowSecret bool + Like string +} + +// createPlan is everything needed to call the API and print the result. +type createPlan struct { + Request accessKeyRequest + Buckets []scopedBucket // resolved --bucket values (empty for fullaccess) + Endpoint string + Signing string + // ProjectID is the bkt_ project ID to store with the key when known. + ProjectID string +} + +// parseCreateOptions reads the flags. +func parseCreateOptions(cmd *cobra.Command) (createOptions, error) { + var o createOptions + values, _ := cmd.Flags().GetStringArray("bucket") + specs, err := parseBucketSpecs(values) + if err != nil { + return o, err + } + o.BucketSpecs = specs + o.AllBuckets, _ = cmd.Flags().GetBool("all-buckets") + if o.StorageClass, err = storageClassFlag(cmd); err != nil { + return o, err + } + if o.Site, err = regionFlag(cmd); err != nil { + return o, err + } + o.Project = projectFlag(cmd) + o.Name, _ = cmd.Flags().GetString("name") + o.Name = strings.TrimSpace(o.Name) + if err := validateAccessKeyName(o.Name); err != nil { + return o, err + } + // --save is a plain bool (so --save=false never saves); --save-as + // picks the saved name and implies --save. + o.Save, _ = cmd.Flags().GetBool("save") + o.SaveName, _ = cmd.Flags().GetString("save-as") + o.SaveName = strings.TrimSpace(o.SaveName) + if o.SaveName != "" { + o.Save = true + } + o.ShowSecret, _ = cmd.Flags().GetBool("show-secret") + o.Like, _ = cmd.Flags().GetString("like") + return o, nil +} + +// applyLike fills the options from a saved key template. Explicit flags win. +func applyLike(o *createOptions, like config.StoredAccessKey) error { + if like.Scope == config.ScopeUnknown { + return exitcode.Errorf(exitcode.Usage, "saved key %q has unknown scope and cannot be used as a template", o.Like) + } + if o.StorageClass == "" { + o.StorageClass = like.StorageClass + } + if o.Site == "" { + o.Site = strings.ToUpper(like.Site) + } + if o.Project == "" { + o.Project = like.ProjectID + } + if !o.AllBuckets && len(o.BucketSpecs) == 0 && like.Scope == config.ScopeFullAccess { + o.AllBuckets = true + } + if like.Scope == config.ScopeLimitedAccess && !o.AllBuckets { + seen := map[string]bool{} + for _, s := range o.BucketSpecs { + seen[s.Token] = true + } + ids := make([]string, 0, len(like.Buckets)) + for id := range like.Buckets { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + if !seen[id] { + o.BucketSpecs = append(o.BucketSpecs, bucketSpec{Token: id, Permission: like.Buckets[id]}) + } + } + } + return nil +} + +// buildCreatePlan resolves the buckets, validates that they belong to one +// backend and infers project, class and site. +func buildCreatePlan(ctx context.Context, r *objectstorage.Resolver, o createOptions) (*createPlan, error) { + if o.AllBuckets && len(o.BucketSpecs) > 0 { + return nil, exitcode.Errorf(exitcode.Usage, "--all-buckets and --bucket are mutually exclusive") + } + if !o.AllBuckets && len(o.BucketSpecs) == 0 { + return nil, exitcode.Errorf(exitcode.Usage, "choose the scope: --bucket [=rw|readonly] (repeatable) or --all-buckets --storage-class ") + } + plan := &createPlan{Request: accessKeyRequest{Project: o.Project, StorageClass: o.StorageClass, Site: o.Site, Name: o.Name}} + + if len(o.BucketSpecs) > 0 { + for _, spec := range o.BucketSpecs { + b, err := r.Resolve(ctx, spec.Token) + if err != nil { + return nil, err + } + if b.EndpointOverride { + return nil, exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url") + } + if err := r.FillSite(ctx, b); err != nil { + if b.StorageClass == objectstorage.ClassHighPerformance && o.Site == "" { + return nil, err + } + lsh.LogDebugf("[s3] site lookup failed for %s: %v", b.ID, err) + } + plan.Buckets = append(plan.Buckets, scopedBucket{Bucket: b, Permission: spec.Permission}) + } + if err := validateSameGroup(plan.Buckets); err != nil { + return nil, err + } + first := plan.Buckets[0].Bucket + plan.ProjectID = first.ProjectID + if plan.Request.Project == "" { + plan.Request.Project = first.ProjectRef() + } + if plan.Request.StorageClass == "" { + plan.Request.StorageClass = first.StorageClass + } else if plan.Request.StorageClass != first.StorageClass { + return nil, exitcode.Errorf(exitcode.Usage, "--storage-class %s does not match the buckets (%s)", plan.Request.StorageClass, first.StorageClass) + } + if plan.Request.Site == "" { + plan.Request.Site = strings.ToUpper(first.Site) + } + plan.Request.Scope = config.ScopeLimitedAccess + plan.Request.Buckets = map[string]string{} + for _, sb := range plan.Buckets { + plan.Request.Buckets[sb.Bucket.ID] = sb.Permission + } + plan.Endpoint, plan.Signing = first.Endpoint, first.SigningRegion + } else { + plan.Request.Scope = config.ScopeFullAccess + if plan.Request.Project == "" { + return nil, exitcode.Errorf(exitcode.Usage, "--project is required with --all-buckets (the key covers every bucket of one project)") + } + if err := inferFromProject(ctx, r, plan); err != nil { + return nil, err + } + } + + if plan.Request.Project == "" { + return nil, exitcode.Errorf(exitcode.Usage, "could not infer the project; pass --project ") + } + if plan.Request.StorageClass == objectstorage.ClassHighPerformance && plan.Request.Site == "" { + return nil, exitcode.Errorf(exitcode.Usage, "--region is required for high_performance keys and could not be inferred; pass the site slug of the VAST cluster (e.g. TYO4)") + } + if plan.Request.Site == "" { + return nil, exitcode.Errorf(exitcode.Usage, "--region is required and could not be inferred from the buckets; pass a Latitude site slug (e.g. DAL)") + } + return plan, nil +} + +// inferFromProject fills class, site, endpoint and project ID for a +// fullaccess key from the project's buckets. +func inferFromProject(ctx context.Context, r *objectstorage.Resolver, plan *createPlan) error { + scoped := *r + scoped.Project = plan.Request.Project + list, err := scoped.ListBuckets(ctx) + if err != nil { + return err + } + var buckets []*objectstorage.Bucket + classes := map[string]bool{} + for _, d := range list { + b := objectstorage.BucketFromData(d) + if plan.ProjectID == "" { + plan.ProjectID = b.ProjectID + } + classes[b.StorageClass] = true + if plan.Request.StorageClass == "" || b.StorageClass == plan.Request.StorageClass { + buckets = append(buckets, b) + } + } + if plan.Request.StorageClass == "" { + switch len(classes) { + case 1: + for c := range classes { + plan.Request.StorageClass = c + } + case 0: + return exitcode.Errorf(exitcode.Usage, "project %s has no buckets yet; pass --storage-class standard|high_performance (and --region )", plan.Request.Project) + default: + return exitcode.Errorf(exitcode.Usage, "project %s has buckets of several storage classes; pass --storage-class standard or high_performance", plan.Request.Project) + } + buckets = buckets[:0] + for _, d := range list { + b := objectstorage.BucketFromData(d) + if b.StorageClass == plan.Request.StorageClass { + buckets = append(buckets, b) + } + } + } + // Sites come from the raw API (the SDK model drops them). + if len(buckets) > 0 { + sites, err := objectstorage.RawBucketSites(ctx, "") + if err != nil { + lsh.LogDebugf("[s3] site lookup failed: %v", err) + } + distinct := map[string]bool{} + for _, b := range buckets { + if b.Site == "" { + b.Site = sites[b.ID] + } + if b.Site != "" { + distinct[strings.ToUpper(b.Site)] = true + } + } + if plan.Request.Site == "" { + if plan.Request.StorageClass == objectstorage.ClassHighPerformance && len(distinct) > 1 { + names := make([]string, 0, len(distinct)) + for s := range distinct { + names = append(names, s) + } + sort.Strings(names) + return exitcode.Errorf(exitcode.Usage, "project %s has high_performance buckets in several sites (%s); pass --region ", plan.Request.Project, strings.Join(names, ", ")) + } + for _, b := range buckets { + if b.Site != "" { + plan.Request.Site = strings.ToUpper(b.Site) + break + } + } + } + for _, b := range buckets { + if plan.Request.StorageClass == objectstorage.ClassHighPerformance && !strings.EqualFold(b.Site, plan.Request.Site) { + continue + } + if b.Endpoint != "" { + plan.Endpoint, plan.Signing = b.Endpoint, b.SigningRegion + break + } + } + } + return nil +} + +// defaultCreateName generates a pet name for the plan's storage class, the +// same style the dashboard suggests (key---). +func defaultCreateName(cmd *cobra.Command, plan *createPlan) string { + _ = cmd + return generateKeyName(plan.Request.StorageClass) +} + +// bucketPermView is one bucket in the JSON output. +type bucketPermView struct { + BucketName string `json:"bucket_name"` + Name string `json:"name,omitempty"` + ID string `json:"id,omitempty"` + Permission string `json:"permission"` +} + +// createdKeyView is the normalized JSON shape of a created key. +type createdKeyView struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key,omitempty"` + Username string `json:"username,omitempty"` + Status string `json:"status,omitempty"` + Scope string `json:"scope"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Endpoint string `json:"endpoint,omitempty"` + SigningRegion string `json:"signing_region,omitempty"` + Buckets []bucketPermView `json:"buckets"` + SavedAs string `json:"saved_as,omitempty"` + Profile string `json:"profile,omitempty"` + DryRun bool `json:"dry_run,omitempty"` +} + +func newCreatedKeyView(k *createdAccessKey, plan *createPlan, showSecret bool) createdKeyView { + v := createdKeyView{ + Name: k.Name, AccessKeyID: k.AccessKeyID, Username: k.Username, Status: k.Status, Scope: k.Scope, + StorageClass: k.StorageClass, Site: k.Site, Project: k.Project, Endpoint: k.Endpoint, SigningRegion: k.SigningRegion, + Buckets: []bucketPermView{}, + } + if showSecret { + v.SecretAccessKey = k.SecretAccessKey + } + for _, sb := range plan.Buckets { + v.Buckets = append(v.Buckets, bucketPermView{BucketName: sb.Bucket.BucketName, Name: sb.Bucket.Name, ID: sb.Bucket.ID, Permission: sb.Permission}) + } + return v +} + +func (v createdKeyView) TableRow() table.Row { + status := "created" + if v.DryRun { + status = "dryrun" + } + return table.Row{ + "name": {Label: "Name", Value: v.Name}, + "access_key_id": {Label: "Access Key ID", Value: v.AccessKeyID}, + "scope": {Label: "Scope", Value: v.Scope}, + "storage_class": {Label: "Class", Value: v.StorageClass}, + "region": {Label: "Site", Value: dash(v.Site)}, + "project": {Label: "Project", Value: v.Project}, + "status": {Label: "Status", Value: status}, + } +} + +// bucketsLine renders "backups-7f3a (rw), logs-91aa (readonly)" or the +// fullaccess description. +func (v createdKeyView) bucketsLine() string { + if v.Scope == config.ScopeFullAccess { + s := fmt.Sprintf("all %s buckets of project %s", v.StorageClass, v.Project) + if v.Site != "" { + s += " in " + v.Site + } + return s + } + parts := make([]string, 0, len(v.Buckets)) + for _, b := range v.Buckets { + parts = append(parts, fmt.Sprintf("%s (%s)", firstNonEmptyStr(b.BucketName, b.Name, b.ID), b.Permission)) + } + return strings.Join(parts, ", ") +} + +// printCreatedHuman prints the J3 block. The secret line is only present +// when showSecret is true; it is never masked. +func printCreatedHuman(w io.Writer, v createdKeyView, showSecret bool) { + if v.DryRun { + fmt.Fprintf(w, "(dryrun) create access key %q (no API call made):\n", v.Name) + } else if showSecret { + fmt.Fprintf(w, "Access key %q created. The secret is shown once and cannot be retrieved again:\n", v.Name) + } else { + fmt.Fprintf(w, "Access key %q created.\n", v.Name) + } + if v.AccessKeyID != "" { + fmt.Fprintf(w, " Access Key ID: %s\n", v.AccessKeyID) + } + if showSecret && v.SecretAccessKey != "" { + fmt.Fprintf(w, " Secret Access Key: %s\n", v.SecretAccessKey) + } + if v.Endpoint != "" { + fmt.Fprintf(w, " Endpoint: %s\n", v.Endpoint) + } else { + fmt.Fprintf(w, " Endpoint: (see 'lsh s3 get s3://' for the bucket endpoint)\n") + } + if v.SigningRegion != "" { + fmt.Fprintf(w, " Signing region: %s\n", v.SigningRegion) + } + fmt.Fprintf(w, " Class: %s\n", v.StorageClass) + if v.Site != "" { + fmt.Fprintf(w, " Site: %s\n", v.Site) + } + fmt.Fprintf(w, " Project: %s\n", v.Project) + fmt.Fprintf(w, " Scope: %s\n", v.Scope) + fmt.Fprintf(w, " Buckets: %s\n", v.bucketsLine()) + if v.SavedAs != "" { + fmt.Fprintf(w, " Saved as: %q in profile %s\n", v.SavedAs, v.Profile) + } +} + +func runAccessKeysCreate(cmd *cobra.Command, _ []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + o, err := parseCreateOptions(cmd) + if err != nil { + return printErr(err) + } + if o.Like != "" { + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + like, ok := p.ObjectStorageKeys()[o.Like] + if !ok { + return printErr(exitcode.Errorf(exitcode.NotFound, "no saved access key named %q in profile %s; run 'lsh s3 access-keys list --saved'", o.Like, profileName)) + } + if err := applyLike(&o, like); err != nil { + return printErr(err) + } + } + // Fail before creating anything if the key could not be saved afterwards: + // the secret would be lost. + profileName := "" + if o.Save { + _, name, _, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(exitcode.Errorf(exitcode.Of(err), "--save needs an active profile: %v", err)) + } + profileName = name + } + + r := newResolver(cmd) + if r.EndpointURL != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url")) + } + plan, err := buildCreatePlan(ctx, r, o) + if err != nil { + return printErr(err) + } + if plan.Request.Name == "" { + plan.Request.Name = defaultCreateName(cmd, plan) + } + saveName := o.SaveName + if o.Save && saveName == "" { + saveName = plan.Request.Name + } + + decision := decideSecretDisplay(isHuman(), o.Save, o.ShowSecret, outputExplicit(cmd)) + + if dryRun() { + k := &createdAccessKey{ + Name: plan.Request.Name, StorageClass: plan.Request.StorageClass, Site: plan.Request.Site, + Project: plan.Request.Project, Scope: plan.Request.Scope, Endpoint: plan.Endpoint, SigningRegion: plan.Signing, + } + v := newCreatedKeyView(k, plan, false) + v.DryRun = true + if o.Save { + v.SavedAs, v.Profile = saveName, profileName + } + if isHuman() { + printCreatedHuman(os.Stdout, v, false) + } else { + render([]renderer.ResponseData{v}) + } + return nil + } + + // The API name is generated whenever --name was not given; --save-as only + // names the local alias, so it must not disable the collision re-roll. + created, err := createAccessKeyRetrying(ctx, cmd, plan.Request, o.Name == "") + if err != nil { + return printErr(err) + } + // A re-roll changed the API name: the alias has to follow it, or it would + // keep pointing at the name of another live key. + if o.SaveName == "" && created.Name != "" && created.Name != saveName { + saveName = created.Name + } + created.Endpoint, created.SigningRegion = plan.Endpoint, plan.Signing + if len(plan.Buckets) > 0 { + created.Buckets = map[string]string{} + for _, sb := range plan.Buckets { + created.Buckets[sb.Bucket.Name] = sb.Permission + } + } + v := newCreatedKeyView(created, plan, decision.Show) + + if o.Save { + stored := created.stored(plan.ProjectID, plan.Request.Buckets, config.KeySourceCreate) + // saveNewKey never overwrites another key: a taken name gets -2, -3… + finalName, savedProfile, err := saveNewKey(cmd, saveName, stored) + if err != nil { + // "it was saved in the profile" would be a lie now. + decision.Warning = "" + // Removing the key is preferred over printing its secret; the + // helper only prints when it cannot remove it (or --show-secret + // asked for it), and always on stderr, never on stdout. + reportUnsavedKey(ctx, os.Stderr, plan.Request, created, plan, saveName, o.ShowSecret, err) + // The command's contract with --save is create *and* persist, and + // the key may have been removed again: exiting 0 here would tell + // automation the credential is in the profile. + return printErr(exitcode.Errorf(exitcode.Generic, "access key %q was created but not saved in the profile", saveName)) + } else { + v.SavedAs, v.Profile = finalName, savedProfile + if finalName != saveName { + objectstorage.Hintf("a saved key named %q already exists; saved as %q instead", saveName, finalName) + } + objectstorage.Hintf("Saved as %q in profile %s. The CLI uses it automatically for the buckets it covers.", finalName, savedProfile) + } + } + if decision.Warning != "" { + objectstorage.Warnf("%s", decision.Warning) + } + if isHuman() { + printCreatedHuman(os.Stdout, v, decision.Show) + return nil + } + render([]renderer.ResponseData{v}) + return nil +} diff --git a/cmd/s3/accesskeys_create_test.go b/cmd/s3/accesskeys_create_test.go new file mode 100644 index 0000000..c29d3d0 --- /dev/null +++ b/cmd/s3/accesskeys_create_test.go @@ -0,0 +1,257 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "regexp" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func TestCreateStoredRoundTrip(t *testing.T) { + home := withTempProfile(t, nil) + f := newFakeAPI(t) + api := f.client() + + req := accessKeyRequest{ + Project: "proj_1", StorageClass: objectstorage.ClassStandard, Site: "DAL", Name: "CI Deploy", + Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw", "bkt_2": "readonly"}, + } + created, err := api.create(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if created.AccessKeyID != "WASABIKEYID" || created.SecretAccessKey != "wasabi-secret-value" || created.Name != "ci deploy" { + t.Fatalf("unexpected created key: id=%s name=%s", created.AccessKeyID, created.Name) + } + + // The request body carries the expected JSON:API shape. + reqs := f.Requests() + if len(reqs) != 1 || reqs[0].Method != "POST" || reqs[0].Path != "/storage/access_keys" { + t.Fatalf("unexpected requests: %+v", reqs) + } + var body map[string]interface{} + if err := json.Unmarshal([]byte(reqs[0].Body), &body); err != nil { + t.Fatal(err) + } + attrs := body["data"].(map[string]interface{})["attributes"].(map[string]interface{}) + if attrs["access_scope"] != "limited_access" || attrs["region"] != "DAL" || attrs["storage_class"] != "standard" || attrs["project"] != "proj_1" { + t.Fatalf("unexpected attributes: %v", attrs) + } + perms := attrs["bucket_permissions"].([]interface{}) + if len(perms) != 2 || perms[0].(map[string]interface{})["bucket_id"] != "bkt_1" { + t.Fatalf("unexpected bucket_permissions: %v", perms) + } + + // Save into the temp profile and read it back through the same helpers. + stored := created.stored("proj_1", req.Buckets, config.KeySourceCreate) + profileName, err := objectstorage.SaveKey("", "ci-deploy", stored) + if err != nil || profileName != "test" { + t.Fatalf("SaveKey: profile=%q err=%v", profileName, err) + } + _, _, p, err := objectstorage.ActiveProfile("") + if err != nil { + t.Fatal(err) + } + got, ok := p.ObjectStorageKeys()["ci-deploy"] + if !ok { + t.Fatalf("key not saved: %v", p.ObjectStorageKeys()) + } + if got.AccessKeyID != "WASABIKEYID" || got.SecretAccessKey != "wasabi-secret-value" || got.Scope != config.ScopeLimitedAccess || + got.Buckets["bkt_2"] != "readonly" || got.Username != "someone+ci deploy@example.com" || got.Source != config.KeySourceCreate || + got.StorageClass != "standard" || got.ProjectID != "proj_1" || got.CreatedAt.IsZero() { + t.Fatalf("stored key mismatch: %s scope=%s buckets=%v", got, got.Scope, got.Buckets) + } + raw := readConfigRaw(t, home) + if !strings.Contains(raw, `"secret_access_key": "wasabi-secret-value"`) || !strings.Contains(raw, `"scope": "limited_access"`) { + t.Fatalf("config.json content unexpected:\n%s", raw) + } + + // The credential resolver now picks the key for the bucket it covers. + cred, err := objectstorage.ResolveCredential(&objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a", Endpoint: "https://s3.us-central-1.storage.sh", StorageClass: "standard", ProjectID: "proj_1"}, objectstorage.CredentialOptions{Write: true}) + if err != nil || cred.Name != "ci-deploy" || cred.Secret() != "wasabi-secret-value" { + t.Fatalf("saved key not selected: %v err=%v", cred, err) + } + + // ForgetKeyByID removes it again. + _, removed, err := objectstorage.ForgetKeyByID("", "WASABIKEYID") + if err != nil || len(removed) != 1 || removed[0] != "ci-deploy" { + t.Fatalf("ForgetKeyByID: %v err=%v", removed, err) + } + if strings.Contains(readConfigRaw(t, home), "wasabi-secret-value") { + t.Fatal("secret still on disk after forget") + } +} + +func TestCreateVASTShapeAndAPIErrors(t *testing.T) { + f := newFakeAPI(t) + f.createShape = "vast" + api := f.client() + req := accessKeyRequest{Project: "proj_1", StorageClass: objectstorage.ClassHighPerformance, Site: "TYO4", Name: "fast", Scope: config.ScopeFullAccess} + created, err := api.create(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if created.AccessKeyID != "VASTKEYID" || created.SecretAccessKey != "vast-secret-value" || created.Site != "TYO4" || created.Buckets != nil { + t.Fatalf("unexpected created key: %+v", created.stored("", nil, "")) + } + + // Validation errors never reach the API. + before := len(f.Requests()) + bad := req + bad.Site = "" + if _, err := api.create(context.Background(), bad); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("missing site must be a usage error, got %v", err) + } + if len(f.Requests()) != before { + t.Fatal("invalid request must not hit the API") + } +} + +func TestCreatedKeyViewSecretHandling(t *testing.T) { + k := &createdAccessKey{ + Name: "ci-deploy", AccessKeyID: "XL68DDURVGUUOULWPCAE", SecretAccessKey: "the-secret-value", StorageClass: "standard", Site: "DAL", + Project: "my-project", Scope: config.ScopeLimitedAccess, Endpoint: "https://s3.us-central-1.storage.sh", SigningRegion: "us-central-1", + } + plan := &createPlan{Buckets: []scopedBucket{ + {Bucket: &objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a"}, Permission: "rw"}, + {Bucket: &objectstorage.Bucket{ID: "bkt_2", Name: "logs", BucketName: "logs-91aa"}, Permission: "readonly"}, + }} + + // Shown once, in clear, with the J3 layout. + var buf bytes.Buffer + printCreatedHuman(&buf, newCreatedKeyView(k, plan, true), true) + out := buf.String() + for _, want := range []string{ + `Access key "ci-deploy" created. The secret is shown once and cannot be retrieved again:`, + "Access Key ID: XL68DDURVGUUOULWPCAE", + "Secret Access Key: the-secret-value", + "Endpoint: https://s3.us-central-1.storage.sh", + "Signing region: us-central-1", + "Site: DAL", + "Buckets: backups-7f3a (rw), logs-91aa (readonly)", + } { + if !strings.Contains(out, want) { + t.Errorf("human output missing %q:\n%s", want, out) + } + } + + // Omitted: no secret at all, not even a fragment. + buf.Reset() + v := newCreatedKeyView(k, plan, false) + printCreatedHuman(&buf, v, false) + if strings.Contains(buf.String(), "secret") || strings.Contains(buf.String(), "the-") || strings.Contains(buf.String(), "value") { + t.Fatalf("secret (or a fragment) leaked:\n%s", buf.String()) + } + b, _ := json.Marshal(v) + if strings.Contains(string(b), "secret") { + t.Fatalf("JSON must omit secret_access_key entirely: %s", b) + } + if !strings.Contains(string(b), `"buckets":[{"bucket_name":"backups-7f3a","name":"backups","id":"bkt_1","permission":"rw"}`) { + t.Fatalf("JSON buckets shape unexpected: %s", b) + } + b, _ = json.Marshal(newCreatedKeyView(k, plan, true)) + if !strings.Contains(string(b), `"secret_access_key":"the-secret-value"`) || !strings.Contains(string(b), `"signing_region":"us-central-1"`) { + t.Fatalf("explicit JSON must include the secret and endpoint data: %s", b) + } + + // Fullaccess keys describe their coverage instead of listing buckets. + full := *k + full.Scope = config.ScopeFullAccess + fv := newCreatedKeyView(&full, &createPlan{}, false) + if got := fv.bucketsLine(); got != "all standard buckets of project my-project in DAL" { + t.Fatalf("fullaccess buckets line = %q", got) + } +} + +func TestApplyLike(t *testing.T) { + like := config.StoredAccessKey{StorageClass: "standard", Site: "", ProjectID: "proj_1", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw", "bkt_2": "readonly"}} + o := createOptions{Like: "ci", BucketSpecs: []bucketSpec{{Token: "bkt_3", Permission: "rw"}}} + if err := applyLike(&o, like); err != nil { + t.Fatal(err) + } + if o.Project != "proj_1" || o.StorageClass != "standard" || len(o.BucketSpecs) != 3 || o.BucketSpecs[1].Token != "bkt_1" || o.BucketSpecs[2].Permission != "readonly" { + t.Fatalf("applyLike merged wrongly: %+v", o) + } + full := config.StoredAccessKey{StorageClass: "high_performance", Site: "tyo4", ProjectID: "proj_1", Scope: config.ScopeFullAccess} + o = createOptions{Like: "hp"} + if err := applyLike(&o, full); err != nil || !o.AllBuckets || o.Site != "TYO4" { + t.Fatalf("fullaccess template must set --all-buckets and the site: %+v err=%v", o, err) + } + if err := applyLike(&createOptions{Like: "u"}, config.StoredAccessKey{Scope: config.ScopeUnknown}); err == nil { + t.Fatal("unknown scope template must fail") + } +} + +func TestBuildCreatePlanScopeErrors(t *testing.T) { + r := &objectstorage.Resolver{} + if _, err := buildCreatePlan(context.Background(), r, createOptions{}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("no scope must be a usage error, got %v", err) + } + if _, err := buildCreatePlan(context.Background(), r, createOptions{AllBuckets: true, BucketSpecs: []bucketSpec{{Token: "a"}}}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("both scopes must be a usage error, got %v", err) + } + if _, err := buildCreatePlan(context.Background(), r, createOptions{AllBuckets: true, StorageClass: "standard"}); err == nil || !strings.Contains(err.Error(), "--project") { + t.Fatalf("--all-buckets without project must ask for --project, got %v", err) + } +} + +// TestParseCreateOptionsSaveFlags covers F27: --save is a plain bool, +// --save-as implies --save, and --save=false never saves. +func TestParseCreateOptionsSaveFlags(t *testing.T) { + parse := func(args ...string) createOptions { + t.Helper() + cmd := newAccessKeysCreateCmd() + if err := cmd.ParseFlags(args); err != nil { + t.Fatalf("ParseFlags(%v): %v", args, err) + } + o, err := parseCreateOptions(cmd) + if err != nil { + t.Fatalf("parseCreateOptions(%v): %v", args, err) + } + return o + } + if o := parse("--bucket", "backups"); o.Save || o.SaveName != "" { + t.Fatalf("default must not save: %+v", o) + } + if o := parse("--bucket", "backups", "--save"); !o.Save || o.SaveName != "" { + t.Fatalf("--save must save under the key name: %+v", o) + } + if o := parse("--bucket", "backups", "--save=false"); o.Save { + t.Fatalf("--save=false must not save: %+v", o) + } + if o := parse("--bucket", "backups", "--save-as", "ops"); !o.Save || o.SaveName != "ops" { + t.Fatalf("--save-as must imply --save with that name: %+v", o) + } + if o := parse("--bucket", "backups", "--save", "--save-as", " ops "); !o.Save || o.SaveName != "ops" { + t.Fatalf("--save --save-as must trim the name: %+v", o) + } + cmd := newAccessKeysCreateCmd() + if err := cmd.ParseFlags([]string{"--save=ops"}); err == nil { + t.Fatal("--save= is no longer accepted; --save-as replaces it") + } +} + +func TestGenerateKeyName(t *testing.T) { + re := regexp.MustCompile(`^key-[a-z]+-[a-z]+-(std|hp)$`) + for _, class := range []string{"standard", "high_performance", ""} { + got := generateKeyName(class) + if !re.MatchString(got) { + t.Errorf("generateKeyName(%q) = %q, want key---", class, got) + } + if len(got) < 3 || len(got) > maxAccessKeyNameLen { + t.Errorf("generateKeyName(%q) = %q, length %d out of 3..%d", class, got, len(got), maxAccessKeyNameLen) + } + if err := validateAccessKeyName(got); err != nil { + t.Errorf("generated name %q rejected by validator: %v", got, err) + } + } + if tierAbbr("high_performance") != "hp" || tierAbbr("standard") != "std" || tierAbbr("") != "std" { + t.Error("tierAbbr mapping wrong") + } +} diff --git a/cmd/s3/accesskeys_delete.go b/cmd/s3/accesskeys_delete.go new file mode 100644 index 0000000..c265b5b --- /dev/null +++ b/cmd/s3/accesskeys_delete.go @@ -0,0 +1,188 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysDeleteCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "delete ", + Aliases: []string{"rm", "revoke"}, + Short: "Delete an access key on the API and forget it locally", + Long: `Delete an access key. The key is looked up by name, access key ID or username +in the API (every project unless --project is given) and among the saved keys. +When several keys match (same name in two classes, sites or projects) the +command lists them and asks for --storage-class, --region or --project. + +The key is revoked on the API and removed from the active profile. Anything +still using it stops working immediately.`, + Example: ` lsh s3 access-keys delete ci-deploy + lsh s3 access-keys delete XL68DDURVGUUOULWPCAE --yes + lsh s3 access-keys delete backup --storage-class high_performance --region TYO4 --project my-project`, + Args: cobra.ExactArgs(1), + RunE: runAccessKeysDelete, + }) + addProjectFlag(cmd, true, "project of the key (ID or slug); every project is searched when omitted") + cmd.Flags().StringP("storage-class", "c", "", "disambiguate by storage class: standard or high_performance") + cmd.Flags().String("region", "", "disambiguate by site (e.g. TYO4); required for high_performance keys when unknown") + addYesFlag(cmd) + return cmd +} + +// deleteResult is the structured output of delete. +type deleteKeyResult struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + Username string `json:"username,omitempty"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Deleted bool `json:"deleted"` + Forgotten []string `json:"forgotten,omitempty"` + DryRun bool `json:"dry_run,omitempty"` +} + +func (d deleteKeyResult) TableRow() table.Row { + status := "deleted" + if d.DryRun { + status = "dryrun" + } + return table.Row{ + "name": {Label: "Name", Value: d.Name}, + "access_key_id": {Label: "Access Key ID", Value: d.AccessKeyID}, + "storage_class": {Label: "Class", Value: d.StorageClass}, + "region": {Label: "Site", Value: dash(d.Site)}, + "project": {Label: "Project", Value: d.Project}, + "status": {Label: "Status", Value: status}, + } +} + +// resolveDeleteTarget picks the single key token refers to among the API +// keys and the saved keys. Saved keys resolve a name to an access key ID; +// when the API does not list the key, the saved metadata is used as target +// (the API still needs the username). +func resolveDeleteTarget(token string, keys []apiKey, saved map[string]config.StoredAccessKey, filter listFilter) (apiKey, error) { + matches := findAPIKeys(token, keys, filter) + savedName, savedKey, hasSaved := findSavedKey(token, saved) + if len(matches) == 0 && hasSaved && savedKey.AccessKeyID != "" { + matches = findAPIKeys(savedKey.AccessKeyID, keys, filter) + } + switch len(matches) { + case 1: + return filter.withSite(matches[0]), nil + case 0: + if hasSaved && filter.matchSaved(savedKey) { + k := apiKey{ + Name: savedName, Username: savedKey.Username, AccessKeyID: savedKey.AccessKeyID, + StorageClass: savedKey.StorageClass, Site: savedKey.Site, Project: firstNonEmptyStr(filter.Project, savedKey.ProjectID), + Access: savedKey.Scope, + } + if filter.Site != "" { + k.Site = filter.Site + } + return k, nil + } + return apiKey{}, exitcode.Errorf(exitcode.NotFound, "access key %q not found; run 'lsh s3 access-keys list' (add --project to search one project)", token) + } + return apiKey{}, ambiguousKeys(token, matches) +} + +func runAccessKeysDelete(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + token := strings.TrimSpace(args[0]) + + class, err := storageClassFlag(cmd) + if err != nil { + return printErr(err) + } + site, err := regionFlag(cmd) + if err != nil { + return printErr(err) + } + yes, _ := cmd.Flags().GetBool(flagYes) + filter := listFilter{StorageClass: class, Site: site, Project: projectFlag(cmd)} + _, _, savedKeys := savedKeysByID(cmd) + _, savedKey, hasSaved := findSavedKey(token, savedKeys) + + r := newResolver(cmd) + if r.EndpointURL != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url")) + } + api := newKeysAPI() + projects := []string{filter.Project} + if filter.Project == "" { + if hasSaved && savedKey.ProjectID != "" { + projects = []string{savedKey.ProjectID} + } else { + projects, _, err = teamProjects(ctx, r) + if err != nil { + return printErr(err) + } + } + } + keys, err := api.listProjects(ctx, projects) + if err != nil { + return printErr(err) + } + target, err := resolveDeleteTarget(token, keys, savedKeys, filter) + if err != nil { + return printErr(err) + } + if target.StorageClass == "" { + return printErr(exitcode.Errorf(exitcode.Usage, "the storage class of %q is unknown; pass --storage-class standard|high_performance", token)) + } + + res := deleteKeyResult{ + Name: target.Name, AccessKeyID: target.AccessKeyID, Username: target.Username, + StorageClass: target.StorageClass, Site: strings.ToUpper(target.Site), Project: target.Project, DryRun: dryRun(), + } + objectstorage.Hintf("access key %q id=%s class=%s site=%s project=%s", dash(res.Name), res.AccessKeyID, res.StorageClass, dash(res.Site), res.Project) + + if !dryRun() { + if err := objectstorage.ConfirmOrRefuse(cmd, yes, fmt.Sprintf("Delete access key %q (%s)? Anything using it stops working", dash(res.Name), res.AccessKeyID)); err != nil { + return printErr(err) + } + if err := api.delete(ctx, target.Username, target.StorageClass, target.Project, target.Site); err != nil { + return printErr(err) + } + res.Deleted = true + profileName, removed, err := objectstorage.ForgetKeyByID(profileFlag(cmd), target.AccessKeyID) + if err != nil { + lsh.LogDebugf("[s3] could not update the profile after deleting %s: %v", target.AccessKeyID, err) + } + res.Forgotten = removed + if len(removed) > 0 { + objectstorage.Hintf("removed saved key(s) %s from profile %s", strings.Join(quoteAll(removed), ", "), profileName) + } + } + + if isHuman() { + prefix := "" + if res.DryRun { + prefix = "(dryrun) " + } + fmt.Printf("%sdelete_access_key: %s (%s)\n", prefix, dash(res.Name), res.AccessKeyID) + return nil + } + render([]renderer.ResponseData{res}) + return nil +} + +func quoteAll(values []string) []string { + out := make([]string, 0, len(values)) + for _, v := range values { + out = append(out, fmt.Sprintf("%q", v)) + } + return out +} diff --git a/cmd/s3/accesskeys_forget.go b/cmd/s3/accesskeys_forget.go new file mode 100644 index 0000000..b6fd273 --- /dev/null +++ b/cmd/s3/accesskeys_forget.go @@ -0,0 +1,74 @@ +package s3 + +import ( + "fmt" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysForgetCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "forget ", + Aliases: []string{"unsave"}, + Short: "Remove a saved access key from this profile (the key stays valid on the API)", + Long: `Remove an access key from the active profile only. The key keeps working for +whoever else has it; use 'lsh s3 access-keys delete' to revoke it on the API.`, + Example: ` lsh s3 access-keys forget ci-deploy + lsh s3 access-keys forget ci-deploy --profile acme`, + Args: cobra.ExactArgs(1), + RunE: runAccessKeysForget, + }) + return cmd +} + +// forgetResult is the structured output of forget. +type forgetResult struct { + Name string `json:"name"` + Profile string `json:"profile"` + Removed bool `json:"removed"` + DryRun bool `json:"dry_run,omitempty"` +} + +func (f forgetResult) TableRow() table.Row { + status := "removed" + if f.DryRun { + status = "dryrun" + } + return table.Row{ + "name": {Label: "Name", Value: f.Name}, + "profile": {Label: "Profile", Value: f.Profile}, + "status": {Label: "Status", Value: status}, + } +} + +func runAccessKeysForget(cmd *cobra.Command, args []string) error { + name := strings.TrimSpace(args[0]) + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + if _, ok := p.ObjectStorageKeys()[name]; !ok { + return printErr(exitcode.Errorf(exitcode.NotFound, "no saved access key named %q in profile %s; run 'lsh s3 access-keys list --saved'", name, profileName)) + } + res := forgetResult{Name: name, Profile: profileName, Removed: true, DryRun: dryRun()} + if !dryRun() { + if _, _, err := objectstorage.ForgetKey(profileFlag(cmd), name); err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + } + if isHuman() { + prefix := "" + if res.DryRun { + prefix = "(dryrun) " + } + fmt.Printf("%sforget: access key %q removed from profile %s\n", prefix, name, profileName) + return nil + } + render([]renderer.ResponseData{res}) + return nil +} diff --git a/cmd/s3/accesskeys_get.go b/cmd/s3/accesskeys_get.go new file mode 100644 index 0000000..c58f50a --- /dev/null +++ b/cmd/s3/accesskeys_get.go @@ -0,0 +1,197 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysGetCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "get ", + Aliases: []string{"show", "describe"}, + Short: "Show one access key", + Long: `Show one access key by name, access key ID or username. The API is searched +first (every project unless --project is given), then the keys saved in the +active profile. The secret is never shown: it only exists in the create output.`, + Example: ` lsh s3 access-keys get ci-deploy + lsh s3 access-keys get XL68DDURVGUUOULWPCAE -o json`, + Args: cobra.ExactArgs(1), + RunE: runAccessKeysGet, + }) + addProjectFlag(cmd, true, "search this project only (ID or slug)") + cmd.Flags().StringP("storage-class", "c", "", "disambiguate by storage class: standard or high_performance") + cmd.Flags().String("region", "", "disambiguate by site (e.g. DAL, TYO4)") + return cmd +} + +// findAPIKeys returns the API keys matching token by name, access key ID or +// username, narrowed by the filter. +func findAPIKeys(token string, keys []apiKey, filter listFilter) []apiKey { + var out []apiKey + for _, k := range keys { + if !filter.matchAPI(k) { + continue + } + if k.AccessKeyID == token || k.Username == token || strings.EqualFold(k.Name, token) { + out = append(out, k) + } + } + return out +} + +// findSavedKey looks a saved key up by name or access key ID. +func findSavedKey(token string, keys map[string]config.StoredAccessKey) (string, config.StoredAccessKey, bool) { + if k, ok := keys[token]; ok { + return token, k, true + } + for name, k := range keys { + if k.AccessKeyID == token { + return name, k, true + } + } + return "", config.StoredAccessKey{}, false +} + +// ambiguousKeys formats the exit-2 error for several matching keys. +func ambiguousKeys(token string, keys []apiKey) error { + lines := make([]string, 0, len(keys)) + for _, k := range keys { + lines = append(lines, fmt.Sprintf(" %-22s class=%s site=%s project=%s", k.AccessKeyID, k.StorageClass, dash(strings.ToUpper(k.Site)), k.Project)) + } + return exitcode.Errorf(exitcode.Usage, "%q matches %d access keys:\n%s\ndisambiguate with the access key ID or --storage-class / --region / --project", token, len(keys), strings.Join(lines, "\n")) +} + +func runAccessKeysGet(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + token := strings.TrimSpace(args[0]) + + class, err := storageClassFlag(cmd) + if err != nil { + return printErr(err) + } + site, err := regionFlag(cmd) + if err != nil { + return printErr(err) + } + filter := listFilter{StorageClass: class, Site: site, Project: projectFlag(cmd)} + savedByID, profileName, savedKeys := savedKeysByID(cmd) + + // Saved keys let a name or ID resolve to a project even without --project. + savedName, savedKey, hasSaved := findSavedKey(token, savedKeys) + + r := newResolver(cmd) + api := newKeysAPI() + projects := []string{filter.Project} + slugs := map[string]string{} + if filter.Project == "" { + if hasSaved && savedKey.ProjectID != "" { + projects = []string{savedKey.ProjectID} + } else { + projects, slugs, err = teamProjects(ctx, r) + if err != nil { + return printErr(err) + } + } + } + keys, listErr := api.listProjects(ctx, projects) + if listErr != nil && !hasSaved { + return printErr(listErr) + } + if listErr != nil { + lsh.LogDebugf("[s3] API listing failed, falling back to the saved key: %v", listErr) + } + + matches := findAPIKeys(token, keys, filter) + if len(matches) == 0 && hasSaved && savedKey.AccessKeyID != "" { + matches = findAPIKeys(savedKey.AccessKeyID, keys, filter) + } + switch { + case len(matches) > 1: + return printErr(ambiguousKeys(token, matches)) + case len(matches) == 1: + row := newKeyRow(filter.withSite(matches[0]), savedByID, slugs) + if isHuman() { + printKeyDetails(os.Stdout, row) + return nil + } + render([]renderer.ResponseData{row}) + return nil + } + + if !hasSaved { + return printErr(exitcode.Errorf(exitcode.NotFound, "access key %q not found in the API nor in profile %s; run 'lsh s3 access-keys list'", token, dash(profileName))) + } + row := newSavedKeyRow(savedName, savedKey, profileName) + if listErr == nil { + row.API = "missing on API" + objectstorage.Warnf("access key %q (%s) is saved in profile %s but the API does not list it; if it was deleted run 'lsh s3 access-keys forget %s'", savedName, savedKey.AccessKeyID, profileName, savedName) + } + if isHuman() { + printSavedKeyDetails(os.Stdout, row) + return nil + } + render([]renderer.ResponseData{row}) + return nil +} + +// printKeyDetails prints the human view of one API key as Label: value lines. +func printKeyDetails(w io.Writer, r keyRow) { + fmt.Fprintf(w, "Name: %s\n", r.Name) + fmt.Fprintf(w, "Access Key ID: %s\n", r.AccessKeyID) + fmt.Fprintf(w, "Scope: %s\n", r.scopeCell()) + buckets := r.bucketsCell() + if r.Access != config.ScopeFullAccess && len(r.Buckets) > 0 { + buckets = strings.Join(r.Buckets, ", ") + } + fmt.Fprintf(w, "Buckets: %s\n", buckets) + fmt.Fprintf(w, "Class: %s\n", r.StorageClass) + fmt.Fprintf(w, "Site: %s\n", dash(r.Site)) + fmt.Fprintf(w, "Project: %s\n", dash(r.Project)) + fmt.Fprintf(w, "Status: %s\n", dash(r.Status)) + if r.Username != "" { + fmt.Fprintf(w, "Username: %s\n", r.Username) + } + if r.CreatedAt != "" { + fmt.Fprintf(w, "Created: %s\n", r.CreatedAt) + } + saved := "no" + if r.Saved { + saved = fmt.Sprintf("yes (as %q)", r.SavedAs) + } + fmt.Fprintf(w, "Saved: %s\n", saved) +} + +// printSavedKeyDetails prints the human view of one saved key (no secret). +func printSavedKeyDetails(w io.Writer, r savedKeyRow) { + fmt.Fprintf(w, "Name: %s\n", r.Name) + fmt.Fprintf(w, "Access Key ID: %s\n", r.AccessKeyID) + fmt.Fprintf(w, "Scope: %s\n", dash(r.Scope)) + buckets := r.bucketsCell() + if r.Scope != config.ScopeFullAccess && len(r.Buckets) > 0 { + buckets = formatPerms(r.Buckets) + } + fmt.Fprintf(w, "Buckets: %s\n", buckets) + fmt.Fprintf(w, "Class: %s\n", dash(r.StorageClass)) + fmt.Fprintf(w, "Site: %s\n", dash(r.Site)) + fmt.Fprintf(w, "Project: %s\n", dash(r.ProjectID)) + if r.Username != "" { + fmt.Fprintf(w, "Username: %s\n", r.Username) + } + fmt.Fprintf(w, "Source: %s\n", dash(r.Source)) + fmt.Fprintf(w, "Created: %s\n", dash(r.CreatedAt)) + fmt.Fprintf(w, "Profile: %s\n", dash(r.Profile)) + if r.API != "" { + fmt.Fprintf(w, "API: %s\n", r.API) + } +} diff --git a/cmd/s3/accesskeys_import.go b/cmd/s3/accesskeys_import.go new file mode 100644 index 0000000..501ea68 --- /dev/null +++ b/cmd/s3/accesskeys_import.go @@ -0,0 +1,224 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysImportCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "import --name --access-key-id ", + Short: "Save an existing access key in this profile (secret from stdin or a prompt)", + Long: `Save an access key that already exists (created in the dashboard, by another +tool or on another machine) in the active profile. + +The secret is never passed as a flag: it is read from a prompt without echo on +a terminal, or from a single line on stdin otherwise, so +'echo "$SECRET" | lsh s3 access-keys import ...' works in scripts. + +When --project is known the key is matched against the API to record its scope +(class, site, buckets); otherwise --storage-class/--region/--bucket/--all-buckets +describe it. Keys with unknown scope are never selected automatically: they are +only used with --access-key .`, + Example: ` lsh s3 access-keys import --name legacy --access-key-id AKIA... --project my-project + echo "$SECRET" | lsh s3 access-keys import --name ci --access-key-id AKIA... --bucket backups=rw + lsh s3 access-keys import --name ops --access-key-id AKIA... --all-buckets --storage-class standard --project my-project`, + Args: cobra.NoArgs, + RunE: runAccessKeysImport, + }) + addProjectFlag(cmd, true, "project of the key (ID or slug); lets the CLI look its scope up on the API") + cmd.Flags().String("name", "", "name to save the key under (required)") + cmd.Flags().String("access-key-id", "", "the access key ID (required)") + cmd.Flags().StringP("storage-class", "c", "", "storage class of the key: standard or high_performance") + cmd.Flags().String("region", "", "site of the key (e.g. TYO4) for high_performance keys") + cmd.Flags().StringArray("bucket", nil, bucketSpecUsage) + cmd.Flags().Bool("all-buckets", false, "the key covers every bucket of the project in its storage class (fullaccess)") + _ = cmd.MarkFlagRequired("name") + _ = cmd.MarkFlagRequired("access-key-id") + return cmd +} + +// importScope resolves the metadata of an imported key: first from the API +// (when the project is known and the ID is listed), then from the flags. +func importScope(ctx context.Context, r *objectstorage.Resolver, accessKeyID string, o createOptions) (config.StoredAccessKey, error) { + k := config.StoredAccessKey{ + AccessKeyID: accessKeyID, + StorageClass: o.StorageClass, + Site: o.Site, + ProjectID: o.Project, + Scope: config.ScopeUnknown, + Source: config.KeySourceImport, + CreatedAt: time.Now().UTC(), + } + + if o.Project != "" && r.API != nil { + scopes, err := objectstorage.RawAccessKeyScopes(ctx, o.Project) + if err != nil { + lsh.LogDebugf("[s3] could not look the key up on the API: %v", err) + } else if s, ok := scopes[accessKeyID]; ok { + if k.StorageClass == "" { + k.StorageClass = s.StorageClass + } + if k.Site == "" { + k.Site = strings.ToUpper(s.Site) + } + k.Username = s.Username + switch s.Access { + case config.ScopeFullAccess: + k.Scope = config.ScopeFullAccess + case config.PermissionRW, config.PermissionReadOnly: + k.Scope = config.ScopeLimitedAccess + k.Buckets = map[string]string{} + scoped := *r + scoped.Project = o.Project + list, err := scoped.ListBuckets(ctx) + if err != nil { + return k, err + } + for _, d := range list { + b := objectstorage.BucketFromData(d) + for _, name := range s.Buckets { + if b.BucketName == name || b.Name == name { + k.Buckets[b.ID] = s.Access + if k.ProjectID == "" || !strings.HasPrefix(k.ProjectID, "proj_") { + k.ProjectID = b.ProjectID + } + } + } + } + } + if k.Scope != config.ScopeUnknown { + return k, nil + } + } + } + + switch { + case o.AllBuckets && len(o.BucketSpecs) > 0: + return k, exitcode.Errorf(exitcode.Usage, "--all-buckets and --bucket are mutually exclusive") + case o.AllBuckets: + k.Scope = config.ScopeFullAccess + if k.StorageClass == "" { + return k, exitcode.Errorf(exitcode.Usage, "--storage-class standard|high_performance is required with --all-buckets") + } + if k.StorageClass == objectstorage.ClassHighPerformance && k.Site == "" { + return k, exitcode.Errorf(exitcode.Usage, "--region is required for a high_performance key (e.g. TYO4)") + } + case len(o.BucketSpecs) > 0: + var scoped []scopedBucket + for _, spec := range o.BucketSpecs { + b, err := r.Resolve(ctx, spec.Token) + if err != nil { + return k, err + } + if b.EndpointOverride { + return k, exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url") + } + if err := r.FillSite(ctx, b); err != nil { + lsh.LogDebugf("[s3] site lookup failed for %s: %v", b.ID, err) + } + scoped = append(scoped, scopedBucket{Bucket: b, Permission: spec.Permission}) + } + if err := validateSameGroup(scoped); err != nil { + return k, err + } + first := scoped[0].Bucket + k.Scope = config.ScopeLimitedAccess + k.Buckets = map[string]string{} + for _, sb := range scoped { + k.Buckets[sb.Bucket.ID] = sb.Permission + } + if k.StorageClass == "" { + k.StorageClass = first.StorageClass + } + if k.Site == "" && first.StorageClass == objectstorage.ClassHighPerformance { + k.Site = strings.ToUpper(first.Site) + } + if k.ProjectID == "" || !strings.HasPrefix(k.ProjectID, "proj_") { + k.ProjectID = firstNonEmptyStr(first.ProjectID, k.ProjectID) + } + } + return k, nil +} + +func runAccessKeysImport(cmd *cobra.Command, _ []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + o, err := parseCreateOptions(cmd) + if err != nil { + return printErr(err) + } + accessKeyID, _ := cmd.Flags().GetString("access-key-id") + accessKeyID = strings.TrimSpace(accessKeyID) + if o.Name == "" || accessKeyID == "" { + return printErr(exitcode.Errorf(exitcode.Usage, "--name and --access-key-id are required")) + } + // Fail before reading the secret when there is no profile to save into. + _, profileName, _, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + + r := newResolver(cmd) + if r.EndpointURL != "" { + r.API = nil + } + stored, err := importScope(ctx, r, accessKeyID, o) + // keyMatchesBucket compares ProjectID with the bucket's proj_ ID, so a slug + // passed on --project (accepted everywhere else) would leave the imported + // key permanently unselectable. The --bucket paths already resolve it. + if err == nil && stored.ProjectID != "" && !strings.HasPrefix(stored.ProjectID, "proj_") { + stored.ProjectID = resolveProjectID(ctx, r, stored.ProjectID) + } + if err != nil { + return printErr(err) + } + if stored.Scope == config.ScopeUnknown { + objectstorage.Warnf("the scope of %s is unknown (not found on the API and no --bucket/--all-buckets given); the key is only used when selected explicitly with --access-key %s", accessKeyID, o.Name) + } + row := newSavedKeyRow(o.Name, stored, profileName) + + if dryRun() { + if isHuman() { + fmt.Printf("(dryrun) import: access key %s as %q in profile %s (class=%s site=%s scope=%s)\n", accessKeyID, o.Name, profileName, dash(stored.StorageClass), dash(stored.Site), stored.Scope) + return nil + } + render([]renderer.ResponseData{row}) + return nil + } + + secret, err := objectstorage.ReadSecret(cmd, fmt.Sprintf("Secret access key for %s: ", accessKeyID)) + if err != nil { + return printErr(exitcode.Errorf(exitcode.Usage, "could not read the secret: %v", err)) + } + if secret == "" { + return printErr(exitcode.Errorf(exitcode.Usage, "no secret provided; type it at the prompt or pipe it on stdin: echo \"$SECRET\" | lsh s3 access-keys import ...")) + } + stored.SecretAccessKey = secret + // Same clash policy as create/configure/mb: a name holding another key + // is never overwritten; the import is saved with a -2, -3… suffix. + finalName, profileName, err := saveNewKey(cmd, o.Name, stored) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + if finalName != o.Name { + objectstorage.Hintf("a saved key named %q already exists; saved as %q instead", o.Name, finalName) + row = newSavedKeyRow(finalName, stored, profileName) + } + if isHuman() { + fmt.Printf("import: access key %s saved as %q in profile %s (scope %s)\n", accessKeyID, finalName, profileName, stored.Scope) + return nil + } + render([]renderer.ResponseData{row}) + return nil +} diff --git a/cmd/s3/accesskeys_list.go b/cmd/s3/accesskeys_list.go new file mode 100644 index 0000000..ece7c07 --- /dev/null +++ b/cmd/s3/accesskeys_list.go @@ -0,0 +1,316 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + "text/tabwriter" + + "github.com/latitudesh/lsh/cli" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysListCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List access keys (API), or the keys saved in this profile (--saved)", + Long: `List the access keys of a project. Pick a project with --project (or +LSH_PROJECT), list every project with --all-projects, or — in a terminal — +choose from the project picker (which includes "All projects"). + +SCOPE is fullaccess, rw or readonly as the API reports it; BUCKETS are the +backend bucket names the key covers; SAVED tells whether the key (by access key +ID) is stored in the active profile. Secrets are never listed. + +--saved lists only the local profile, marking keys the API no longer knows as +"missing on API".`, + Example: ` lsh s3 access-keys list + lsh s3 access-keys list --project my-project -o json + lsh s3 access-keys list --bucket backups + lsh s3 access-keys list --saved`, + Args: cobra.NoArgs, + RunE: runAccessKeysList, + }) + addProjectFlag(cmd, true, "list the keys of this project only (ID or slug)") + cmd.Flags().Bool("all-projects", false, "list the keys of every project (skip the project picker)") + cmd.Flags().StringP("storage-class", "c", "", "only keys of this storage class: standard or high_performance") + cmd.Flags().String("region", "", "only keys of this site (e.g. DAL, TYO4)") + cmd.Flags().String("bucket", "", "only the keys covering this bucket (name or bkt_ ID)") + cmd.Flags().Bool("saved", false, "list the keys saved in the active profile instead of the API") + return cmd +} + +// listFilter narrows a key listing. +type listFilter struct { + StorageClass string + Site string + Project string +} + +// matchAPI applies the class and site filters. A key the API lists without +// a site (standard keys span sites) matches any --region: the flag then +// disambiguates rather than hides. +func (f listFilter) matchAPI(k apiKey) bool { + if f.StorageClass != "" && k.StorageClass != f.StorageClass { + return false + } + if f.Site != "" && k.Site != "" && !strings.EqualFold(k.Site, f.Site) { + return false + } + return true +} + +// withSite returns k with the --region value filled in when the API left the +// site empty, so a single match carries the site the user named. +func (f listFilter) withSite(k apiKey) apiKey { + if k.Site == "" && f.Site != "" { + k.Site = f.Site + } + return k +} + +func (f listFilter) matchSaved(k config.StoredAccessKey) bool { + if f.StorageClass != "" && k.StorageClass != f.StorageClass { + return false + } + // Same wildcard as matchAPI: a key stored without a site (standard keys + // span sites) matches any --region instead of being hidden by it. + if f.Site != "" && k.Site != "" && !strings.EqualFold(k.Site, f.Site) { + return false + } + // --project takes an ID or a slug, while the key stores whichever was + // known when it was saved. Comparing across the two kinds would hide keys + // instead of filtering them, so only like-for-like counts as a mismatch. + if f.Project != "" && k.ProjectID != "" && comparableProjects(k.ProjectID, f.Project) && k.ProjectID != f.Project { + return false + } + return true +} + +// comparableProjects reports whether two project references are of the same +// kind (both proj_ IDs or both slugs) and can therefore be compared. +func comparableProjects(a, b string) bool { + return strings.HasPrefix(a, "proj_") == strings.HasPrefix(b, "proj_") +} + +func runAccessKeysList(cmd *cobra.Command, _ []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + class, err := storageClassFlag(cmd) + if err != nil { + return printErr(err) + } + site, err := regionFlag(cmd) + if err != nil { + return printErr(err) + } + filter := listFilter{StorageClass: class, Site: site, Project: projectFlag(cmd)} + + if saved, _ := cmd.Flags().GetBool("saved"); saved { + return runAccessKeysListSaved(ctx, cmd, filter) + } + + api := newKeysAPI() + r := newResolver(cmd) + var keys []apiKey + slugs := map[string]string{} + multiProject := false + if bucket, _ := cmd.Flags().GetString("bucket"); bucket != "" { + b, err := r.Resolve(ctx, bucket) + if err != nil { + return printErr(err) + } + if b.EndpointOverride { + return printErr(exitcode.Errorf(exitcode.Usage, "access keys are managed through the Latitude API; unset --endpoint-url")) + } + if err := r.FillSite(ctx, b); err != nil { + lsh.LogDebugf("[s3] site lookup failed for %s: %v", b.ID, err) + } + keys, err = api.listForBucket(ctx, b) + if err != nil { + return printErr(err) + } + if b.ProjectSlug != "" { + slugs[b.ProjectRef()] = b.ProjectSlug + } + } else { + // Pick the project like `lsh s3 list`: --project / LSH_PROJECT, or + // --all-projects, otherwise the interactive picker (with "All projects"). + project, allProjects, perr := cli.PickProjectForList(cmd) + if perr != nil { + return printErr(perr) + } + var projects []string + if allProjects { + projects, slugs, err = teamProjects(ctx, r) + if err != nil { + return printErr(err) + } + multiProject = len(projects) > 1 + } else { + projects = []string{project} + filter.Project = project + } + keys, err = api.listProjects(ctx, projects) + if err != nil { + return printErr(err) + } + } + + savedByID, _, _ := savedKeysByID(cmd) + var rows []keyRow + for _, k := range keys { + if !filter.matchAPI(k) { + continue + } + rows = append(rows, newKeyRow(k, savedByID, slugs)) + } + if isHuman() { + if len(rows) == 0 { + objectstorage.Hintf("no access keys found") + return nil + } + writeKeyTable(os.Stdout, rows, multiProject) + return nil + } + out := make([]renderer.ResponseData, 0, len(rows)) + for _, row := range rows { + out = append(out, row) + } + render(out) + return nil +} + +// writeKeyTable prints the human listing: an aligned table with a header, no +// borders, the way the plan shows it. +func writeKeyTable(w io.Writer, rows []keyRow, withProject bool) { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + header := "NAME\tACCESS KEY ID\tSCOPE\tBUCKETS\tCLASS\tSITE\tSTATUS\tSAVED" + if withProject { + header = "PROJECT\t" + header + } + fmt.Fprintln(tw, header) + for _, r := range rows { + line := strings.Join([]string{r.Name, r.AccessKeyID, r.scopeCell(), r.bucketsCell(), r.StorageClass, dash(r.Site), dash(r.Status), yesNo(r.Saved)}, "\t") + if withProject { + line = dash(r.Project) + "\t" + line + } + fmt.Fprintln(tw, line) + } + tw.Flush() +} + +// runAccessKeysListSaved lists the profile's keys, cross-checking the API +// when it is reachable. +func runAccessKeysListSaved(ctx context.Context, cmd *cobra.Command, filter listFilter) error { + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + keys := p.ObjectStorageKeys() + names := p.SortedObjectStorageKeyNames() + + // Best-effort API cross-check over the projects the saved keys mention. + known, queried := apiKeyIDs(ctx, cmd, keys, filter.Project) + + var rows []savedKeyRow + for _, name := range names { + k := keys[name] + if !filter.matchSaved(k) { + continue + } + row := newSavedKeyRow(name, k, profileName) + // A key whose project was never listed cannot be judged: leaving the + // column empty is honest, "missing on API" would not be. + if known != nil && k.ProjectID != "" && queried[k.ProjectID] { + if known[k.AccessKeyID] { + row.API = "ok" + } else { + row.API = "missing on API" + } + } + rows = append(rows, row) + } + if isHuman() { + if len(rows) == 0 { + objectstorage.Hintf("no access keys saved in profile %s; run 'lsh s3 configure' or 'lsh s3 access-keys create --save'", profileName) + return nil + } + writeSavedKeyTable(os.Stdout, rows, known != nil) + return nil + } + out := make([]renderer.ResponseData, 0, len(rows)) + for _, row := range rows { + out = append(out, row) + } + render(out) + return nil +} + +// apiKeyIDs returns the set of access key IDs the API knows for the projects +// referenced by the saved keys (or the --project filter), or nil when the API +// could not be consulted (not logged in, network error, no project known). +func apiKeyIDs(ctx context.Context, cmd *cobra.Command, keys map[string]config.StoredAccessKey, project string) (known, queried map[string]bool) { + if endpointOverride(cmd) != "" { + return nil, nil + } + seen := map[string]bool{} + var projects []string + if project != "" { + projects = []string{project} + } else { + for _, k := range keys { + if k.ProjectID != "" && !seen[k.ProjectID] { + seen[k.ProjectID] = true + projects = append(projects, k.ProjectID) + } + } + sort.Strings(projects) + } + if len(projects) == 0 { + return nil, nil + } + api := newKeysAPI() + known = map[string]bool{} + queried = map[string]bool{} + for _, p := range projects { + list, err := api.list(ctx, p) + if err != nil { + lsh.LogDebugf("[s3] could not cross-check saved keys with the API for project %s: %v", p, err) + return nil, nil + } + queried[p] = true + for _, k := range list { + known[k.AccessKeyID] = true + } + } + return known, queried +} + +// writeSavedKeyTable prints the --saved listing. +func writeSavedKeyTable(w io.Writer, rows []savedKeyRow, withAPI bool) { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + header := "NAME\tACCESS KEY ID\tCLASS\tSITE\tPROJECT\tSCOPE\tBUCKETS\tSOURCE\tCREATED" + if withAPI { + header += "\tAPI" + } + fmt.Fprintln(tw, header) + for _, r := range rows { + line := strings.Join([]string{r.Name, r.AccessKeyID, dash(r.StorageClass), dash(r.Site), dash(r.ProjectID), dash(r.Scope), r.bucketsCell(), dash(r.Source), dash(r.CreatedAt)}, "\t") + if withAPI { + line += "\t" + dash(r.API) + } + fmt.Fprintln(tw, line) + } + tw.Flush() +} diff --git a/cmd/s3/accesskeys_list_test.go b/cmd/s3/accesskeys_list_test.go new file mode 100644 index 0000000..1b62036 --- /dev/null +++ b/cmd/s3/accesskeys_list_test.go @@ -0,0 +1,241 @@ +package s3 + +import ( + "bytes" + "context" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func mustTime(t *testing.T, s string) time.Time { + t.Helper() + v, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatal(err) + } + return v +} + +func TestListMergesRawScopes(t *testing.T) { + f := newFakeAPI(t) + f.keys["proj_1"] = keysDocument + + keys, err := f.client().list(context.Background(), "proj_1") + if err != nil { + t.Fatal(err) + } + if len(keys) != 3 { + t.Fatalf("want 3 keys, got %d: %+v", len(keys), keys) + } + // One GET per project: the raw document already carries every field. + reqs := f.Requests() + if len(reqs) != 1 || reqs[0].Method != "GET" || reqs[0].Path != "/storage/access_keys" || reqs[0].Query.Get("project") != "proj_1" { + t.Fatalf("list must issue exactly one GET /storage/access_keys, got %+v", reqs) + } + if !strings.HasPrefix(reqs[0].Query.Get("project"), "proj_") { + t.Fatalf("project query wrong: %+v", reqs[0].Query) + } + byID := map[string]apiKey{} + for _, k := range keys { + byID[k.AccessKeyID] = k + } + ops := byID["OPSKEYID000000000001"] + if ops.Access != "fullaccess" || ops.StorageClass != "standard" || len(ops.Buckets) != 2 || ops.scope() != config.ScopeFullAccess { + t.Fatalf("ops key not merged with raw scope: %+v", ops) + } + ci := byID["CIKEYID0000000000002"] + if ci.Access != "rw" || ci.scope() != config.ScopeLimitedAccess || ci.Buckets[0] != "backups-7f3a" { + t.Fatalf("ci key not merged: %+v", ci) + } + hp := byID["HPKEYID0000000000003"] + if hp.StorageClass != "high_performance" || hp.Site != "TYO4" || hp.Access != "readonly" { + t.Fatalf("hp key wrong: %+v", hp) + } + + // Rows: SAVED comes from the profile match, SCOPE from the raw access. + rows := []keyRow{} + saved := map[string]string{"CIKEYID0000000000002": "ci-deploy"} + for _, k := range keys { + rows = append(rows, newKeyRow(k, saved, map[string]string{"proj_1": "my-project"})) + } + var buf bytes.Buffer + writeKeyTable(&buf, rows, false) + out := buf.String() + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) != 4 || !strings.HasPrefix(lines[0], "NAME") || !strings.Contains(lines[0], "SAVED") { + t.Fatalf("unexpected table:\n%s", out) + } + if !strings.Contains(out, "ci-deploy") || !strings.Contains(out, "yes") || !strings.Contains(out, "backups-7f3a") || !strings.Contains(out, "all") { + t.Fatalf("table missing columns:\n%s", out) + } + if strings.Contains(out, "secret") { + t.Fatalf("listing must never mention secrets:\n%s", out) + } + for _, r := range rows { + if r.Project != "my-project" { + t.Fatalf("project slug not applied: %+v", r) + } + } +} + +// TestListWithoutScopeFields replaces the former "raw scopes unavailable" +// test: listing is now a single raw request, so there is no second call to +// degrade from. Records without `access`/`buckets` (older backends) must +// still list and render '-' for the unknown scope. +func TestListWithoutScopeFields(t *testing.T) { + f := newFakeAPI(t) + f.keys["proj_1"] = `{"data":{"standard":[{"name":"plain","username":"p@example.com","access_key_id":"PLAINKEYID0000000001","status":"Active","created_at":"2026-09-01T10:00:00Z"}],"high_performance":[]}}` + + keys, err := f.client().list(context.Background(), "proj_1") + if err != nil { + t.Fatal(err) + } + if len(keys) != 1 || keys[0].Access != "" || keys[0].StorageClass != "standard" || keys[0].Site != "" { + t.Fatalf("listing must tolerate records without scope fields: %+v", keys) + } + row := newKeyRow(keys[0], nil, nil) + if row.scopeCell() != emptyCell || row.bucketsCell() != emptyCell { + t.Fatalf("unknown scope must render the shared placeholder: %+v", row) + } + if len(f.Requests()) != 1 { + t.Fatalf("want exactly one request, got %d", len(f.Requests())) + } +} + +func TestListProjectNotFound(t *testing.T) { + f := newFakeAPI(t) + _, err := f.client().list(context.Background(), "nope") + if err == nil || exitcode.Of(err) != exitcode.NotFound { + t.Fatalf("404 must map to exit 3, got %v", err) + } + // Without a token the raw call fails locally with exit 4 (no request made). + api := f.client() + api.token = "" + if _, err := api.list(context.Background(), "proj_1"); err == nil || exitcode.Of(err) != exitcode.Credentials { + t.Fatalf("missing token must be exit 4, got %v", err) + } +} + +// TestSiteFilterTreatsEmptyAPISiteAsWildcard covers F17: standard keys the +// API lists without a site must not vanish behind --region, and the single +// match gets the requested site filled in. +func TestSiteFilterTreatsEmptyAPISiteAsWildcard(t *testing.T) { + noSite := apiKey{Name: "ops", AccessKeyID: "A1", Username: "u1", StorageClass: "standard", Project: "proj_1"} + dal := apiKey{Name: "ci", AccessKeyID: "A2", Username: "u2", StorageClass: "standard", Site: "DAL", Project: "proj_1"} + tyo := apiKey{Name: "fast", AccessKeyID: "A3", Username: "u3", StorageClass: "high_performance", Site: "TYO4", Project: "proj_1"} + filter := listFilter{Site: "DAL"} + if !filter.matchAPI(noSite) || !filter.matchAPI(dal) || filter.matchAPI(tyo) { + t.Fatalf("empty API site must match any --region; a differing site must not") + } + if got := filter.withSite(noSite); got.Site != "DAL" { + t.Fatalf("withSite must fill the empty site, got %q", got.Site) + } + if got := filter.withSite(tyo); got.Site != "TYO4" { + t.Fatalf("withSite must not override an API site, got %q", got.Site) + } + keys := []apiKey{noSite, dal, tyo} + if got := findAPIKeys("ops", keys, filter); len(got) != 1 || got[0].AccessKeyID != "A1" { + t.Fatalf("get with --region must still find the standard key without a site: %+v", got) + } + target, err := resolveDeleteTarget("ops", keys, nil, filter) + if err != nil || target.AccessKeyID != "A1" || target.Site != "DAL" { + t.Fatalf("delete with --region must resolve the key and carry the site, got %+v err=%v", target, err) + } +} + +func TestResolveDeleteTarget(t *testing.T) { + keys := []apiKey{ + {Name: "backup", AccessKeyID: "A1", Username: "u1", StorageClass: "standard", Site: "DAL", Project: "proj_1"}, + {Name: "backup", AccessKeyID: "A2", Username: "u2", StorageClass: "high_performance", Site: "TYO4", Project: "proj_1"}, + {Name: "other", AccessKeyID: "A3", Username: "u3", StorageClass: "standard", Project: "proj_2"}, + } + saved := map[string]config.StoredAccessKey{ + "mine": {AccessKeyID: "A3", Username: "u3", StorageClass: "standard", ProjectID: "proj_2", Scope: config.ScopeFullAccess}, + "orphaned": {AccessKeyID: "ZZ", Username: "uz", StorageClass: "standard", ProjectID: "proj_9", Scope: config.ScopeFullAccess}, + } + + _, err := resolveDeleteTarget("backup", keys, saved, listFilter{}) + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--storage-class") { + t.Fatalf("ambiguous name must exit 2 with the disambiguating flags, got %v", err) + } + k, err := resolveDeleteTarget("backup", keys, saved, listFilter{StorageClass: "high_performance"}) + if err != nil || k.AccessKeyID != "A2" { + t.Fatalf("--storage-class must disambiguate, got %+v err=%v", k, err) + } + k, err = resolveDeleteTarget("backup", keys, saved, listFilter{Site: "dal"}) + if err != nil || k.AccessKeyID != "A1" { + t.Fatalf("--region must disambiguate, got %+v err=%v", k, err) + } + k, err = resolveDeleteTarget("u2", keys, saved, listFilter{}) + if err != nil || k.AccessKeyID != "A2" { + t.Fatalf("username must resolve, got %+v err=%v", k, err) + } + // A saved name resolves to its access key ID on the API. + k, err = resolveDeleteTarget("mine", keys, saved, listFilter{}) + if err != nil || k.AccessKeyID != "A3" || k.Username != "u3" { + t.Fatalf("saved name must resolve through the API, got %+v err=%v", k, err) + } + // A saved key the API no longer lists is deleted with its stored metadata. + k, err = resolveDeleteTarget("orphaned", keys, saved, listFilter{}) + if err != nil || k.AccessKeyID != "ZZ" || k.Username != "uz" || k.Project != "proj_9" { + t.Fatalf("orphaned saved key must resolve from the profile, got %+v err=%v", k, err) + } + if _, err := resolveDeleteTarget("missing", keys, saved, listFilter{}); err == nil || exitcode.Of(err) != exitcode.NotFound { + t.Fatalf("unknown token must exit 3, got %v", err) + } +} + +func TestDeleteSendsRegionOnlyForHighPerformance(t *testing.T) { + f := newFakeAPI(t) + api := f.client() + ctx := context.Background() + if err := api.delete(ctx, "u1", "standard", "proj_1", "DAL"); err != nil { + t.Fatal(err) + } + if err := api.delete(ctx, "u2", "high_performance", "proj_1", "TYO4"); err != nil { + t.Fatal(err) + } + if err := api.delete(ctx, "u2", "high_performance", "proj_1", ""); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("high_performance without site must fail with exit 2, got %v", err) + } + if err := api.delete(ctx, "", "standard", "proj_1", ""); err == nil { + t.Fatal("missing username must fail") + } + reqs := f.Requests() + if len(reqs) != 2 { + t.Fatalf("want 2 DELETE calls, got %d", len(reqs)) + } + if reqs[0].Path != "/storage/access_keys/u1/standard" || reqs[0].Query.Get("project") != "proj_1" || reqs[0].Query.Has("region") { + t.Fatalf("standard delete request wrong: %+v", reqs[0]) + } + if reqs[1].Path != "/storage/access_keys/u2/high_performance" || reqs[1].Query.Get("region") != "TYO4" { + t.Fatalf("high_performance delete request wrong: %+v", reqs[1]) + } + + f.deleteStatus = 404 + if err := api.delete(ctx, "gone", "standard", "proj_1", ""); err == nil || exitcode.Of(err) != exitcode.NotFound { + t.Fatalf("404 must map to exit 3, got %v", err) + } +} + +func TestSavedKeyRowNeverHoldsSecret(t *testing.T) { + k := config.StoredAccessKey{AccessKeyID: "ID", SecretAccessKey: "topsecret", StorageClass: "standard", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw"}, Source: "import", CreatedAt: mustTime(t, "2026-09-07T10:00:00Z")} + row := newSavedKeyRow("mine", k, "test") + var buf bytes.Buffer + writeSavedKeyTable(&buf, []savedKeyRow{row}, true) + if strings.Contains(buf.String(), "topsecret") { + t.Fatalf("secret leaked: %s", buf.String()) + } + if !strings.Contains(buf.String(), "bkt_1=rw") || !strings.Contains(buf.String(), "API") { + t.Fatalf("unexpected saved table:\n%s", buf.String()) + } + var out bytes.Buffer + printSavedKeyDetails(&out, row) + if strings.Contains(out.String(), "topsecret") || !strings.Contains(out.String(), "Access Key ID: ID") { + t.Fatalf("unexpected details:\n%s", out.String()) + } +} diff --git a/cmd/s3/accesskeys_rotate.go b/cmd/s3/accesskeys_rotate.go new file mode 100644 index 0000000..108ab0b --- /dev/null +++ b/cmd/s3/accesskeys_rotate.go @@ -0,0 +1,280 @@ +package s3 + +import ( + "context" + "fmt" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +func newAccessKeysRotateCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "rotate ", + Short: "Create a new key with the same scope as a saved key and save it in its place", + Long: `Rotate a saved access key: a new key with the same storage class, site, +project and scope (fullaccess or the same buckets and permissions) is created +and saved under the same name, replacing the old one on this machine. + +The old key keeps working until it is deleted: pass --delete-old to delete it +right away (after confirmation), or delete it later once every consumer has +the new secret.`, + Example: ` lsh s3 access-keys rotate ci-deploy + lsh s3 access-keys rotate ci-deploy --delete-old --yes + lsh s3 access-keys rotate ci-deploy --name ci-deploy-2026q4`, + Args: cobra.ExactArgs(1), + RunE: runAccessKeysRotate, + }) + addProjectFlag(cmd, true, "project of the key (ID or slug) when the saved key does not record it") + cmd.Flags().Bool("delete-old", false, "delete the old key on the API after the new one is saved") + cmd.Flags().String("name", "", "name of the new key (default: the saved name with a - suffix)") + cmd.Flags().String("region", "", "site of the key (e.g. TYO4) when the saved high_performance key does not record it") + addYesFlag(cmd) + return cmd +} + +// rotatedName appends the date suffix used for the new key by default, plus a +// sequence number from the second rotation of the same day on (the API rejects +// a duplicate name). The result always fits the API's name limit, so a long +// alias does not produce a request the API refuses. +func rotatedName(base string, now time.Time, seq int) string { + base = sanitizeKeyPart(base) + if base == "" { + base = "lsh-key" + } + suffix := "-" + now.UTC().Format("20060102") + if seq > 1 { + suffix += "-" + strconv.Itoa(seq) + } + if len(base)+len(suffix) > maxAccessKeyNameLen { + base = strings.Trim(base[:maxAccessKeyNameLen-len(suffix)], "-") + } + return base + suffix +} + +// rotateCreateKey creates the new key, working around a name that is already +// taken. Unlike create, rotate does not fall back to a random pet name: the +// name is meant to stay recognisable, so it grows a sequence number instead. +func rotateCreateKey(ctx context.Context, cmd *cobra.Command, req accessKeyRequest, base string, autoName bool) (*createdAccessKey, error) { + for seq := 2; ; seq++ { + created, err := createAccessKeyRetrying(ctx, cmd, req, false) + if err == nil || !autoName || !isNameConflict(err) || seq > 9 { + return created, err + } + req.Name = rotatedName(base, time.Now(), seq) + } +} + +// rotateResult is the structured output of rotate. +type rotateResult struct { + Name string `json:"name"` + SavedAs string `json:"saved_as"` + Profile string `json:"profile"` + OldAccessKeyID string `json:"old_access_key_id"` + NewAccessKeyID string `json:"new_access_key_id,omitempty"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Scope string `json:"scope"` + OldDeleted bool `json:"old_deleted"` + DryRun bool `json:"dry_run,omitempty"` +} + +func (r rotateResult) TableRow() table.Row { + status := "rotated" + if r.DryRun { + status = "dryrun" + } + return table.Row{ + "name": {Label: "Name", Value: r.Name}, + "old_access_key_id": {Label: "Old Access Key ID", Value: r.OldAccessKeyID}, + "new_access_key_id": {Label: "New Access Key ID", Value: dash(r.NewAccessKeyID)}, + "storage_class": {Label: "Class", Value: r.StorageClass}, + "region": {Label: "Site", Value: dash(r.Site)}, + "status": {Label: "Status", Value: status}, + } +} + +// rotateRequest builds the create request that mirrors a saved key. +func rotateRequest(k config.StoredAccessKey, project, site, newName string) (accessKeyRequest, error) { + if k.Scope == config.ScopeUnknown || k.Scope == "" { + return accessKeyRequest{}, exitcode.Errorf(exitcode.Usage, "the saved key has unknown scope, so its scope cannot be reproduced; create a new key with 'lsh s3 access-keys create' instead") + } + req := accessKeyRequest{ + Project: firstNonEmptyStr(k.ProjectID, project), + StorageClass: k.StorageClass, + Site: strings.ToUpper(firstNonEmptyStr(site, k.Site)), + Name: newName, + Scope: k.Scope, + } + if k.Scope == config.ScopeLimitedAccess { + req.Buckets = make(map[string]string, len(k.Buckets)) + for id, perm := range k.Buckets { + req.Buckets[id] = perm + } + } + if req.Project == "" { + return accessKeyRequest{}, exitcode.Errorf(exitcode.Usage, "the saved key does not record its project; pass --project ") + } + if req.StorageClass == "" { + return accessKeyRequest{}, exitcode.Errorf(exitcode.Usage, "the saved key does not record its storage class; re-import it with --storage-class or create a new key") + } + return req, nil +} + +func runAccessKeysRotate(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + savedName := strings.TrimSpace(args[0]) + + site, err := regionFlag(cmd) + if err != nil { + return printErr(err) + } + deleteOld, _ := cmd.Flags().GetBool("delete-old") + yes, _ := cmd.Flags().GetBool(flagYes) + newName, _ := cmd.Flags().GetString("name") + newName = strings.TrimSpace(newName) + + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return printErr(err) + } + old, ok := p.ObjectStorageKeys()[savedName] + if !ok { + return printErr(exitcode.Errorf(exitcode.NotFound, "no saved access key named %q in profile %s; run 'lsh s3 access-keys list --saved'", savedName, profileName)) + } + autoName := newName == "" + if autoName { + newName = rotatedName(savedName, time.Now(), 1) + } else if err := validateAccessKeyName(newName); err != nil { + return printErr(err) + } + req, err := rotateRequest(old, projectFlag(cmd), site, newName) + if err != nil { + return printErr(err) + } + if err := req.validate(); err != nil { + return printErr(err) + } + // Do not create a key we would then refuse to finish handling. + if deleteOld && !yes && !dryRun() && !objectstorage.CanPrompt(cmd) { + return printErr(exitcode.Errorf(exitcode.Refused, "--delete-old asks for confirmation — pass --yes in a non-interactive session")) + } + + res := rotateResult{ + Name: newName, SavedAs: savedName, Profile: profileName, OldAccessKeyID: old.AccessKeyID, + StorageClass: req.StorageClass, Site: req.Site, Project: req.Project, Scope: req.Scope, DryRun: dryRun(), + } + deleteHint := fmt.Sprintf("lsh s3 access-keys delete %s --project %s --storage-class %s", old.AccessKeyID, req.Project, req.StorageClass) + if req.StorageClass == objectstorage.ClassHighPerformance { + deleteHint += " --region " + req.Site + } + + if dryRun() { + if isHuman() { + fmt.Printf("(dryrun) rotate: %s -> new key %q (class=%s site=%s project=%s scope=%s), saved as %q in profile %s\n", + old.AccessKeyID, newName, req.StorageClass, dash(req.Site), req.Project, req.Scope, savedName, profileName) + if deleteOld { + fmt.Printf("(dryrun) delete_access_key: %s\n", old.AccessKeyID) + } + return nil + } + render([]renderer.ResponseData{res}) + return nil + } + + api := newKeysAPI() + created, err := rotateCreateKey(ctx, cmd, req, savedName, autoName) + if err != nil { + return printErr(err) + } + res.Name = created.Name + res.NewAccessKeyID = created.AccessKeyID + // Record the project as a proj_ ID: the old key's when it has one, + // otherwise --project resolved through the project's buckets. + projectID := old.ProjectID + if projectID == "" { + projectID = resolveProjectID(ctx, newResolver(cmd), req.Project) + } + stored := created.stored(projectID, req.Buckets, config.KeySourceCreate) + // Rotate replaces the saved key in place (same name), unlike create. + if _, err := objectstorage.SaveKey(profileFlag(cmd), savedName, stored); err != nil { + // The old key is still live and still saved, so the new one can simply + // be removed: rotating again is cheaper than handling a printed secret. + // The view renders the scope from the plan, so the request's buckets + // have to be there for the fallback print to be complete. + created.Buckets = req.Buckets + reportUnsavedKey(ctx, os.Stderr, req, created, &createPlan{Request: req, Buckets: rotateScopedBuckets(req)}, savedName, false, err) + objectstorage.Hintf("the old key %s is untouched and still saved as %q", old.AccessKeyID, savedName) + return printErr(objectstorage.Humanize(err, nil, nil)) + } + objectstorage.Hintf("new key %s saved as %q in profile %s (replacing %s)", created.AccessKeyID, savedName, profileName, old.AccessKeyID) + + if deleteOld { + username := old.Username + if username == "" { + if keys, err := api.list(ctx, req.Project); err == nil { + for _, k := range keys { + if k.AccessKeyID == old.AccessKeyID { + username = k.Username + break + } + } + } + } + switch { + case username == "": + objectstorage.Warnf("could not find the username of the old key %s on the API; delete it later with: %s", old.AccessKeyID, deleteHint) + default: + err := objectstorage.ConfirmOrRefuse(cmd, yes, fmt.Sprintf("Delete the old access key %s now? Anything still using it stops working", old.AccessKeyID)) + if err != nil { + objectstorage.Warnf("old key %s kept; delete it later with: %s", old.AccessKeyID, deleteHint) + } else if err := api.delete(ctx, username, req.StorageClass, req.Project, req.Site); err != nil { + objectstorage.Warnf("could not delete the old key %s: %v — delete it later with: %s", old.AccessKeyID, err, deleteHint) + } else { + res.OldDeleted = true + } + } + } else { + objectstorage.Hintf("old key %s kept so existing consumers keep working; once they have the new secret run: %s", old.AccessKeyID, deleteHint) + } + + if isHuman() { + fmt.Printf("rotate: %s -> %s (saved as %q in profile %s)\n", old.AccessKeyID, created.AccessKeyID, savedName, profileName) + if res.OldDeleted { + fmt.Printf("delete_access_key: %s\n", old.AccessKeyID) + } + return nil + } + render([]renderer.ResponseData{res}) + return nil +} + +// rotateScopedBuckets rebuilds the plan's bucket list from the request. Rotate +// copies the old key's scope, which is stored as bkt_ IDs, so only the IDs and +// permissions are known — enough for the printed scope of an unsaved key. +func rotateScopedBuckets(req accessKeyRequest) []scopedBucket { + if len(req.Buckets) == 0 { + return nil + } + ids := make([]string, 0, len(req.Buckets)) + for id := range req.Buckets { + ids = append(ids, id) + } + sort.Strings(ids) + out := make([]scopedBucket, 0, len(ids)) + for _, id := range ids { + out = append(out, scopedBucket{Bucket: &objectstorage.Bucket{ID: id, Name: id}, Permission: req.Buckets[id]}) + } + return out +} diff --git a/cmd/s3/accesskeys_shared.go b/cmd/s3/accesskeys_shared.go new file mode 100644 index 0000000..65bcef4 --- /dev/null +++ b/cmd/s3/accesskeys_shared.go @@ -0,0 +1,964 @@ +package s3 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "sort" + "strings" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/version" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// accessKeyRequest describes an access key to create through the API. +type accessKeyRequest struct { + Project string // project ID or slug (required) + StorageClass string // standard | high_performance (required) + Site string // site slug; required for high_performance + Name string // key name (normalized server-side) + Scope string // config.ScopeFullAccess | config.ScopeLimitedAccess + Buckets map[string]string // bkt_ id → rw|readonly (limited_access only) +} + +// createdAccessKey is the normalized result of a create call, independent of +// the backend's field names (Wasabi: access_key_id/secret_access_key; VAST: +// access_key/secret_key). +type createdAccessKey struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key,omitempty"` + Username string `json:"username,omitempty"` + Status string `json:"status,omitempty"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Scope string `json:"scope"` + // Buckets lists bucket display names → permission for limited keys. Right + // after createAccessKey it holds the bkt_ IDs of the request; the create + // command swaps them for display names before printing. + Buckets map[string]string `json:"buckets,omitempty"` + // Endpoint and SigningRegion are filled by the caller from the buckets + // the key covers (the API does not return them). + Endpoint string `json:"endpoint,omitempty"` + SigningRegion string `json:"signing_region,omitempty"` +} + +// stored converts the created key into the config.json representation. +func (k createdAccessKey) stored(projectID string, bucketPerms map[string]string, source string) config.StoredAccessKey { + return config.StoredAccessKey{ + AccessKeyID: k.AccessKeyID, + SecretAccessKey: k.SecretAccessKey, + StorageClass: k.StorageClass, + Site: k.Site, + ProjectID: projectID, + Scope: k.Scope, + Buckets: bucketPerms, + Username: k.Username, + Source: source, + CreatedAt: time.Now().UTC(), + } +} + +// createAccessKey calls POST /storage/access_keys with the CLI's API client. +// mb and configure call it to offer a key right after bucket creation. +func createAccessKey(ctx context.Context, cmd *cobra.Command, req accessKeyRequest) (*createdAccessKey, error) { + _ = cmd + return newKeysAPI().create(ctx, req) +} + +// provisioningRetryDelays are the waits before re-sending a create that failed +// because the backend had not registered the new key yet. Creating a key is a +// multi-step operation on the provider side (create the user, create the key, +// then attach the bucket policy), and the steps do not become visible to each +// other instantly, so the first attempt can fail with a 404 that succeeds a +// few seconds later. The API rolls its own steps back before returning the +// error, so re-sending the same request is safe. +var provisioningRetryDelays = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second} + +// createAccessKeyRetrying creates the key, retrying two failures the API +// itself considers recoverable: a duplicate name (only when the name was +// auto-generated — a fresh pet name is rolled, the way the dashboard avoids +// collisions) and the propagation lag above (same request, after a wait). +func createAccessKeyRetrying(ctx context.Context, cmd *cobra.Command, req accessKeyRequest, autoName bool) (*createdAccessKey, error) { + return retryCreate(ctx, req, autoName, func(r accessKeyRequest) (*createdAccessKey, error) { + return createAccessKey(ctx, cmd, r) + }) +} + +// retryCreate holds the retry policy of createAccessKeyRetrying, with the +// create call injected so the policy can be tested without the API. +func retryCreate(ctx context.Context, req accessKeyRequest, autoName bool, create func(accessKeyRequest) (*createdAccessKey, error)) (*createdAccessKey, error) { + namesLeft, waited := 5, 0 + for { + created, err := create(req) + if err == nil { + return created, nil + } + switch { + case autoName && isNameConflict(err) && namesLeft > 1: + namesLeft-- + req.Name = generateKeyName(req.StorageClass) + case isProvisioningLag(err) && waited < len(provisioningRetryDelays): + delay := provisioningRetryDelays[waited] + waited++ + if waited == 1 { + objectstorage.Hintf("the storage backend has not registered the key yet; retrying in %s…", delay) + } + if waitErr := sleepCtx(ctx, delay); waitErr != nil { + return nil, waitErr + } + case isProvisioningLag(err): + return nil, exitcode.Errorf(exitcode.Of(err), "%v — the storage backend did not register the key in time; retry in a few seconds", err) + default: + return nil, err + } + } +} + +// isNameConflict reports whether err is the API rejecting a duplicate key name. +func isNameConflict(err error) bool { + m := strings.ToLower(err.Error()) + if !strings.Contains(m, "name") { + return false + } + return strings.Contains(m, "taken") || strings.Contains(m, "already") || + strings.Contains(m, "exist") || strings.Contains(m, "unique") || + strings.Contains(m, "in use") +} + +// isProvisioningLag reports whether err is the API surfacing a backend that is +// still converging: STORAGE_RESOURCE_NOT_FOUND ("Storage resource not found.") +// while the new key propagates, or STORAGE_UNAVAILABLE. Both are transient and +// the same request succeeds once the backend catches up. +func isProvisioningLag(err error) bool { + m := strings.ToLower(err.Error()) + return strings.Contains(m, "storage resource not found") || + strings.Contains(m, "temporarily unavailable") +} + +// sleepCtx waits for d, returning early (with the right exit code) if the +// context is cancelled by Ctrl-C or a deadline. +func sleepCtx(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.Canceled) { + return exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + return exitcode.Errorf(exitcode.Generic, "timed out waiting for the storage backend: %v", ctx.Err()) + } +} + +// keysAPI wraps the Latitude API calls of the access-keys commands so the +// command cores can run against a test server. Create and delete go through +// the SDK; listing reads the raw JSON:API document because the generated +// model drops `buckets[]` and `access` (see list). +type keysAPI struct { + sdk *sdk.Latitudesh + opts []operations.Option + // baseURL, token and apiVersion configure the raw GET (same settings the + // SDK client is built with). + baseURL string + token string + apiVersion string + http *http.Client +} + +// newKeysAPI builds the wrapper around the CLI's authenticated client. +func newKeysAPI() *keysAPI { + host := viper.GetString("hostname") + if host == "" { + host = "api.latitude.sh" + } + scheme := viper.GetString("scheme") + if scheme == "" { + scheme = "https" + } + return &keysAPI{ + sdk: apiClient(), + opts: []operations.Option{operations.WithRetries(lsh.RetryConfig())}, + baseURL: scheme + "://" + host, + token: viper.GetString("Authorization"), + apiVersion: viper.GetString("api-version"), + } +} + +// rawGet performs an authenticated GET and returns the body. Non-2xx +// responses are mapped through objectstorage.HumanizeAPI (as an SDK +// *components.APIError) so exit codes match the SDK calls. +func (a *keysAPI) rawGet(ctx context.Context, path string, query url.Values, what string) ([]byte, error) { + if a.token == "" { + return nil, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login' first") + } + u := a.baseURL + path + if len(query) > 0 { + u += "?" + query.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+strings.TrimPrefix(a.token, "Bearer ")) + req.Header.Set("Accept", "application/vnd.api+json, application/json") + req.Header.Set("User-Agent", "lsh/"+version.Version) + if a.apiVersion != "" { + req.Header.Set("API-Version", a.apiVersion) + } + client := a.http + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + resp, err := client.Do(req) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, what) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, objectstorage.HumanizeAPI(&components.APIError{Message: "API error", StatusCode: resp.StatusCode, Body: string(body), RawResponse: resp}, what) + } + return body, nil +} + +// validate checks the request before it reaches the API. +func (r accessKeyRequest) validate() error { + switch { + case r.Project == "": + return exitcode.Errorf(exitcode.Usage, "a project is required to create an access key; pass --project") + case r.StorageClass != objectstorage.ClassStandard && r.StorageClass != objectstorage.ClassHighPerformance: + return exitcode.Errorf(exitcode.Usage, "invalid storage class %q (use standard or high_performance)", r.StorageClass) + case r.Name == "": + return exitcode.Errorf(exitcode.Usage, "an access key name is required; pass --name") + case r.StorageClass == objectstorage.ClassHighPerformance && r.Site == "": + return exitcode.Errorf(exitcode.Usage, "--region is required for high_performance keys (a Latitude site slug such as TYO4 or DAL)") + case r.Scope == config.ScopeLimitedAccess && len(r.Buckets) == 0: + return exitcode.Errorf(exitcode.Usage, "a limited_access key needs at least one --bucket") + case r.Scope != config.ScopeLimitedAccess && r.Scope != config.ScopeFullAccess: + return exitcode.Errorf(exitcode.Usage, "invalid access scope %q", r.Scope) + } + for id, perm := range r.Buckets { + if perm != config.PermissionRW && perm != config.PermissionReadOnly { + return exitcode.Errorf(exitcode.Usage, "invalid permission %q for bucket %s (use rw or readonly)", perm, id) + } + } + return nil +} + +// buildCreateRequest converts the request into the SDK body. Bucket +// permissions are sorted so the payload (and dry-run output) is stable. +func buildCreateRequest(req accessKeyRequest) operations.PostStorageAccessKeysRequestBody { + attrs := operations.PostStorageAccessKeysAttributes{ + Project: req.Project, + AccessKeyStorageClass: operations.AccessKeyStorageClass(req.StorageClass), + Name: req.Name, + AccessScope: operations.AccessScope(req.Scope), + Region: req.Site, + } + if req.Scope == config.ScopeLimitedAccess { + ids := make([]string, 0, len(req.Buckets)) + for id := range req.Buckets { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + attrs.BucketPermissions = append(attrs.BucketPermissions, operations.BucketPermissions{ + BucketID: id, + Permission: operations.Permission(req.Buckets[id]), + }) + } + } + return operations.PostStorageAccessKeysRequestBody{ + Data: operations.PostStorageAccessKeysData{ + Type: operations.PostStorageAccessKeysTypeAccessKeys, + Attributes: attrs, + }, + } +} + +// create performs the POST and normalizes the response. +func (a *keysAPI) create(ctx context.Context, req accessKeyRequest) (*createdAccessKey, error) { + if err := req.validate(); err != nil { + return nil, err + } + resp, err := a.sdk.ObjectStorage.PostStorageAccessKeys(ctx, buildCreateRequest(req), a.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "") + } + return normalizeCreated(resp, req) +} + +// normalizeCreated maps both backend shapes onto createdAccessKey. +func normalizeCreated(resp *operations.PostStorageAccessKeysResponse, req accessKeyRequest) (*createdAccessKey, error) { + var ak *operations.AccessKey + if resp != nil && resp.Object != nil && resp.Object.Data != nil && resp.Object.Data.Attributes != nil { + ak = resp.Object.Data.Attributes.AccessKey + } + if ak == nil { + return nil, exitcode.Errorf(exitcode.Generic, "the API created the access key but returned no credentials; check 'lsh s3 access-keys list --project %s'", req.Project) + } + k := &createdAccessKey{ + Name: firstNonEmptyStr(ptrStr(ak.Name), req.Name), + AccessKeyID: firstNonEmptyStr(ptrStr(ak.AccessKeyID), ptrStr(ak.AccessKey)), + SecretAccessKey: firstNonEmptyStr(ptrStr(ak.SecretAccessKey), ptrStr(ak.SecretKey)), + Username: ptrStr(ak.Username), + Status: ptrStr(ak.Status), + StorageClass: req.StorageClass, + Site: req.Site, + Project: req.Project, + Scope: req.Scope, + } + if k.Scope == config.ScopeLimitedAccess && len(req.Buckets) > 0 { + k.Buckets = make(map[string]string, len(req.Buckets)) + for id, perm := range req.Buckets { + k.Buckets[id] = perm + } + } + if k.AccessKeyID == "" || k.SecretAccessKey == "" { + return nil, exitcode.Errorf(exitcode.Generic, "the API response is missing the access key id or secret (backend %s); the key may exist: check 'lsh s3 access-keys list --project %s'", req.StorageClass, req.Project) + } + return k, nil +} + +// apiKey is one access key as listed by the API, merged with the raw +// scope information (buckets[] and access) the SDK model drops. +type apiKey struct { + Name string + Username string + AccessKeyID string + Status string + CreatedAt string + StorageClass string + Site string + // Project is the project reference (ID or slug) the key was listed under. + Project string + // Access is fullaccess, rw, readonly or "" when unknown. + Access string + // Buckets are backend bucket names. + Buckets []string +} + +// scope maps the API access value onto the config scope constants. +func (k apiKey) scope() string { + switch k.Access { + case config.ScopeFullAccess: + return config.ScopeFullAccess + case config.PermissionRW, config.PermissionReadOnly: + return config.ScopeLimitedAccess + } + return config.ScopeUnknown +} + +// rawAccessKey is one record of GET /storage/access_keys as the API returns +// it (both storage classes share the shape; `buckets` and `access` are the +// fields the SDK model drops). +type rawAccessKey struct { + Name string `json:"name"` + Username string `json:"username"` + AccessKeyID string `json:"access_key_id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + Region string `json:"region"` + Access string `json:"access"` + Buckets []string `json:"buckets"` +} + +// list returns the project's keys of both storage classes with one GET +// /storage/access_keys call: the raw document carries every field the rows +// need, so the SDK call (which would drop buckets[]/access) is not made. +func (a *keysAPI) list(ctx context.Context, project string) ([]apiKey, error) { + if project == "" { + return nil, exitcode.Errorf(exitcode.Usage, "a project is required to list access keys; pass --project") + } + what := fmt.Sprintf("project %q", project) + body, err := a.rawGet(ctx, "/storage/access_keys", url.Values{"project": {project}}, what) + if err != nil { + return nil, err + } + var env struct { + Data map[string][]rawAccessKey `json:"data"` + } + if err := json.Unmarshal(body, &env); err != nil { + return nil, exitcode.Errorf(exitcode.Generic, "could not parse the access keys of %s: %v", what, err) + } + var out []apiKey + for class, keys := range env.Data { + for _, k := range keys { + out = append(out, apiKey{ + Name: k.Name, Username: k.Username, AccessKeyID: k.AccessKeyID, Status: k.Status, + CreatedAt: k.CreatedAt, Site: k.Region, Access: k.Access, Buckets: k.Buckets, + StorageClass: class, Project: project, + }) + } + } + sortAPIKeys(out) + return out, nil +} + +// listForBucket returns the keys covering one bucket (GET /storage/buckets/{id}/access_keys). +func (a *keysAPI) listForBucket(ctx context.Context, b *objectstorage.Bucket) ([]apiKey, error) { + resp, err := a.sdk.ObjectStorage.GetStorageBucketAccessKeys(ctx, b.ID, a.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("bucket %s", b.Display())) + } + var out []apiKey + if resp.Object != nil { + for _, d := range resp.Object.Data { + out = append(out, apiKey{ + Name: ptrStr(d.Name), Username: ptrStr(d.Username), AccessKeyID: ptrStr(d.AccessKeyID), + Status: ptrStr(d.Status), CreatedAt: ptrStr(d.CreatedAt), Access: ptrStr(d.Access), + StorageClass: b.StorageClass, Site: b.Site, Project: b.ProjectRef(), + Buckets: []string{b.BucketName}, + }) + } + } + sortAPIKeys(out) + return out, nil +} + +// listProjects lists the keys of several projects, concatenated. +func (a *keysAPI) listProjects(ctx context.Context, projects []string) ([]apiKey, error) { + var out []apiKey + for _, p := range projects { + keys, err := a.list(ctx, p) + if err != nil { + return nil, err + } + out = append(out, keys...) + } + return out, nil +} + +// delete removes a key. site is only sent for high_performance keys (the API +// ignores it for standard ones). +func (a *keysAPI) delete(ctx context.Context, username, storageClass, project, site string) error { + if username == "" { + return exitcode.Errorf(exitcode.Generic, "the access key has no username; the API needs it to delete the key — run 'lsh s3 access-keys list --project %s' to refresh it", project) + } + if project == "" { + return exitcode.Errorf(exitcode.Usage, "a project is required to delete an access key; pass --project") + } + var region *string + if storageClass == objectstorage.ClassHighPerformance { + if site == "" { + return exitcode.Errorf(exitcode.Usage, "--region is required to delete a high_performance key (a Latitude site slug such as TYO4)") + } + region = &site + } + _, err := a.sdk.ObjectStorage.DeleteStorageAccessKeysUsername(ctx, username, operations.PathParamStorageClass(storageClass), project, region, a.opts...) + if err != nil { + return objectstorage.HumanizeAPI(err, fmt.Sprintf("access key %q", username)) + } + return nil +} + +// resolveProjectID returns the proj_ ID for a project reference (ID or +// slug) by looking at the project's buckets; it falls back to the reference +// itself when nothing resolves so callers can still store something useful. +func resolveProjectID(ctx context.Context, r *objectstorage.Resolver, project string) string { + if project == "" || strings.HasPrefix(project, "proj_") || r == nil || r.API == nil { + return project + } + scoped := *r + scoped.Project = project + list, err := scoped.ListBuckets(ctx) + if err != nil { + lsh.LogDebugf("[s3] could not resolve project %q to an ID: %v", project, err) + return project + } + for _, d := range list { + if id := objectstorage.BucketFromData(d).ProjectID; id != "" { + return id + } + } + return project +} + +// teamProjects returns the distinct projects (IDs, with a slug lookup for +// display) that own buckets in the team, or the one given as filter. +func teamProjects(ctx context.Context, r *objectstorage.Resolver) ([]string, map[string]string, error) { + list, err := r.ListBuckets(ctx) + if err != nil { + return nil, nil, err + } + seen := map[string]bool{} + slugs := map[string]string{} + var out []string + for _, d := range list { + b := objectstorage.BucketFromData(d) + ref := b.ProjectID + if ref == "" { + ref = b.ProjectSlug + } + if ref == "" || seen[ref] { + continue + } + seen[ref] = true + out = append(out, ref) + if b.ProjectSlug != "" { + slugs[ref] = b.ProjectSlug + } + } + sort.Strings(out) + return out, slugs, nil +} + +func sortAPIKeys(keys []apiKey) { + sort.SliceStable(keys, func(i, j int) bool { + if keys[i].Project != keys[j].Project { + return keys[i].Project < keys[j].Project + } + if keys[i].Name != keys[j].Name { + return keys[i].Name < keys[j].Name + } + return keys[i].AccessKeyID < keys[j].AccessKeyID + }) +} + +// bucketSpec is one parsed --bucket value: [=rw|readonly]. +type bucketSpec struct { + Token string + Permission string +} + +// bucketSpecUsage documents the --bucket grammar once for every command +// that accepts it (create, import). +const bucketSpecUsage = "bucket the key covers, as [=rw|readonly] (':' also separates; 'write' = rw, 'ro'/'read' = readonly; default rw; repeatable or comma-separated; a bucket, never s3://bucket/key)" + +// parseBucketSpec accepts "name", "name=rw", "name:readonly" (and "s3://" +// prefixes). The permission defaults to rw. Anything with a key part +// ("bucket/dir/file") is refused: keys grant access to whole buckets. +func parseBucketSpec(s string) (bucketSpec, error) { + raw := strings.TrimSpace(s) + raw = strings.TrimPrefix(raw, "s3://") + raw = strings.TrimSuffix(raw, "/") + if raw == "" { + return bucketSpec{}, exitcode.Errorf(exitcode.Usage, "empty --bucket value") + } + if strings.Contains(raw, "/") { + return bucketSpec{}, exitcode.Errorf(exitcode.Usage, "invalid --bucket %q: access keys cover whole buckets, not keys; pass the bucket only ([=rw|readonly])", s) + } + token, perm := raw, config.PermissionRW + if i := strings.LastIndexAny(raw, "=:"); i >= 0 { + token, perm = raw[:i], strings.ToLower(strings.TrimSpace(raw[i+1:])) + } + token = strings.TrimSpace(token) + if token == "" { + return bucketSpec{}, exitcode.Errorf(exitcode.Usage, "invalid --bucket %q: missing bucket name", s) + } + switch perm { + case config.PermissionRW, "readwrite", "read-write", "write": + perm = config.PermissionRW + case config.PermissionReadOnly, "ro", "read-only", "read": + perm = config.PermissionReadOnly + default: + return bucketSpec{}, exitcode.Errorf(exitcode.Usage, "invalid permission %q in --bucket %q (use =rw or =readonly)", perm, s) + } + return bucketSpec{Token: token, Permission: perm}, nil +} + +// parseBucketSpecs parses every --bucket flag, rejecting duplicates. +func parseBucketSpecs(values []string) ([]bucketSpec, error) { + out := make([]bucketSpec, 0, len(values)) + seen := map[string]bool{} + for _, v := range values { + // Allow comma-separated lists in one flag too. + for _, part := range strings.Split(v, ",") { + if strings.TrimSpace(part) == "" { + continue + } + spec, err := parseBucketSpec(part) + if err != nil { + return nil, err + } + if seen[spec.Token] { + return nil, exitcode.Errorf(exitcode.Usage, "bucket %q given more than once in --bucket", spec.Token) + } + seen[spec.Token] = true + out = append(out, spec) + } + } + return out, nil +} + +// scopedBucket is a resolved bucket plus the permission requested on it. +type scopedBucket struct { + Bucket *objectstorage.Bucket + Permission string +} + +// validateSameGroup makes sure every bucket shares storage class and +// project, and (for high_performance) site. It returns exit 2 listing the +// groups otherwise, since one key cannot span them. +func validateSameGroup(buckets []scopedBucket) error { + if len(buckets) < 2 { + return nil + } + groups := map[string][]string{} + var order []string + for _, sb := range buckets { + b := sb.Bucket + key := fmt.Sprintf("class=%s project=%s", b.StorageClass, firstNonEmptyStr(b.ProjectID, b.ProjectSlug)) + if b.StorageClass == objectstorage.ClassHighPerformance { + key += " site=" + strings.ToUpper(b.Site) + } + if _, ok := groups[key]; !ok { + order = append(order, key) + } + groups[key] = append(groups[key], b.Name) + } + if len(groups) < 2 { + return nil + } + lines := make([]string, 0, len(order)) + for _, k := range order { + lines = append(lines, fmt.Sprintf(" %s: %s", k, strings.Join(groups[k], ", "))) + } + return exitcode.Errorf(exitcode.Usage, "one access key cannot cover buckets of different storage classes, sites or projects:\n%s\ncreate one key per group", strings.Join(lines, "\n")) +} + +var keyNameUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) +var keyNameDashes = regexp.MustCompile(`-{2,}`) + +// sanitizeKeyPart lower-cases s and replaces anything outside [a-z0-9-] +// with '-', collapsing runs. +func sanitizeKeyPart(s string) string { + s = keyNameUnsafe.ReplaceAllString(strings.ToLower(strings.TrimSpace(s)), "-") + s = keyNameDashes.ReplaceAllString(s, "-") + return strings.Trim(s, "-") +} + +// maxAccessKeyNameLen is the API limit on an access key name. +const maxAccessKeyNameLen = 25 + +// capKeyName truncates a name to the API limit and drops a trailing hyphen. +func capKeyName(s string) string { + if len(s) > maxAccessKeyNameLen { + s = s[:maxAccessKeyNameLen] + } + return strings.Trim(s, "-") +} + +// validateAccessKeyName rejects an explicit --name that the API would refuse, +// before the request is sent, with an actionable usage error. +func validateAccessKeyName(name string) error { + if len(name) > maxAccessKeyNameLen { + return exitcode.Errorf(exitcode.Usage, "access key name %q is %d characters; the API allows at most %d", name, len(name), maxAccessKeyNameLen) + } + return nil +} + +// discardUnsavedKey deletes a key that was created but could not be stored in +// the active profile. The API returns the secret exactly once, so a key whose +// secret nobody holds is unusable and would linger with live permissions: +// removing it is the recovery path that neither discloses the secret nor +// leaves a credential behind. Callers fall back to printing the secret (on +// stderr) only when this fails, or when the user asked for it explicitly. +// discardKey is the indirection tests replace; production always deletes +// through the API. +var discardKey = discardUnsavedKey + +func discardUnsavedKey(ctx context.Context, req accessKeyRequest, created *createdAccessKey) error { + if created == nil || created.Username == "" { + return exitcode.Errorf(exitcode.Generic, "the API did not return the key's username, which is required to delete it") + } + return newKeysAPI().delete(ctx, created.Username, req.StorageClass, req.Project, req.Site) +} + +// reportUnsavedKey is the shared tail of a failed save: it removes the key when +// it can, and otherwise prints the only copy of the secret (always on stderr, +// never on stdout) together with the command that stores it. showSecret skips +// the removal because the user asked to keep and print the key. +func reportUnsavedKey(ctx context.Context, w io.Writer, req accessKeyRequest, created *createdAccessKey, plan *createPlan, name string, showSecret bool, saveErr error) { + if !showSecret { + if delErr := discardKey(ctx, req, created); delErr == nil { + fmt.Fprintf(w, "warning: access key %q was created but could not be saved to the profile: %v\n", name, saveErr) + fmt.Fprintf(w, "It was deleted again, so nothing was left behind and no secret was printed. Retry once the profile is writable.\n") + return + } else { + fmt.Fprintf(w, "warning: access key %q could not be saved (%v) nor deleted again (%v), so its secret is printed below — it cannot be retrieved another way.\n", name, saveErr, delErr) + } + } else { + fmt.Fprintf(w, "warning: access key %q was created but could not be saved to the profile: %v\n", name, saveErr) + } + printCreatedHuman(w, newCreatedKeyView(created, plan, true), true) + fmt.Fprintf(w, "Save it once the profile is writable with: %s\n", importHint(name, created, req, created.Buckets)) +} + +// importHint renders the `access-keys import` command that stores a key whose +// secret could not be saved automatically. buckets maps bucket display names +// to permissions and is only used for a limited_access key. +func importHint(name string, created *createdAccessKey, req accessKeyRequest, buckets map[string]string) string { + hint := fmt.Sprintf("lsh s3 access-keys import --name %s --access-key-id %s --project %s --storage-class %s", + name, created.AccessKeyID, req.Project, req.StorageClass) + if req.StorageClass == objectstorage.ClassHighPerformance && req.Site != "" { + hint += " --region " + req.Site + } + if req.Scope == config.ScopeLimitedAccess && len(buckets) > 0 { + names := make([]string, 0, len(buckets)) + for b := range buckets { + names = append(names, b) + } + sort.Strings(names) + for _, b := range names { + hint += fmt.Sprintf(" --bucket %s=%s", b, buckets[b]) + } + return hint + } + return hint + " --all-buckets" +} + +// secretDecision says whether the freshly created secret may be printed and +// which warning (if any) explains an omission. +type secretDecision struct { + Show bool + Warning string +} + +// decideSecretDisplay implements the display rules of the plan (J3): +// - human output, not saving: the secret is printed once in clear; +// - saving (--save): only with --show-secret, in every format; +// - structured output: only when -o was passed explicitly or --show-secret +// (never because of LSH_OUTPUT/config), otherwise omitted with a warning. +func decideSecretDisplay(human, save, showSecret, outputExplicit bool) secretDecision { + switch { + case showSecret: + return secretDecision{Show: true} + case save: + return secretDecision{Warning: "secret omitted: it was saved in the profile; pass --show-secret to print it"} + case human: + return secretDecision{Show: true} + case outputExplicit: + return secretDecision{Show: true} + default: + return secretDecision{Warning: "secret omitted: pass -o json explicitly or --show-secret"} + } +} + +// keyRow renders one API access key (list/get). +type keyRow struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + Scope string `json:"scope"` + Access string `json:"access,omitempty"` + Buckets []string `json:"buckets,omitempty"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project,omitempty"` + Status string `json:"status,omitempty"` + Username string `json:"username,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + Saved bool `json:"saved"` + SavedAs string `json:"saved_as,omitempty"` +} + +// newKeyRow builds the row, marking it saved when a profile key has the +// same access key ID. +func newKeyRow(k apiKey, saved map[string]string, slugs map[string]string) keyRow { + project := k.Project + if s, ok := slugs[project]; ok && s != "" { + project = s + } + row := keyRow{ + Name: k.Name, AccessKeyID: k.AccessKeyID, Scope: k.scope(), Access: k.Access, Buckets: k.Buckets, + StorageClass: k.StorageClass, Site: strings.ToUpper(k.Site), Project: project, Status: k.Status, + Username: k.Username, CreatedAt: k.CreatedAt, + } + if name, ok := saved[k.AccessKeyID]; ok { + row.Saved, row.SavedAs = true, name + } + return row +} + +func (r keyRow) TableRow() table.Row { + return table.Row{ + "name": {Label: "Name", Value: r.Name}, + "access_key_id": {Label: "Access Key ID", Value: r.AccessKeyID}, + "scope": {Label: "Scope", Value: r.scopeCell()}, + "buckets": {Label: "Buckets", Value: r.bucketsCell(), MaxLength: 40}, + "storage_class": {Label: "Class", Value: r.StorageClass}, + "region": {Label: "Site", Value: dash(r.Site)}, + "project": {Label: "Project", Value: dash(r.Project)}, + "status": {Label: "Status", Value: dash(r.Status)}, + "saved": {Label: "Saved", Value: yesNo(r.Saved)}, + } +} + +// scopeCell is the SCOPE column: the raw access (fullaccess|rw|readonly) or '-'. +func (r keyRow) scopeCell() string { return dash(r.Access) } + +// bucketsCell is the BUCKETS column: "all" for fullaccess, the backend names +// otherwise (truncated). +func (r keyRow) bucketsCell() string { + if r.Access == config.ScopeFullAccess { + return "all" + } + if len(r.Buckets) == 0 { + return emptyCell + } + return truncateText(strings.Join(r.Buckets, ", "), 40) +} + +// savedKeyRow renders one saved (profile) key; never the secret. +type savedKeyRow struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + ProjectID string `json:"project_id,omitempty"` + Scope string `json:"scope"` + Buckets map[string]string `json:"buckets,omitempty"` + Username string `json:"username,omitempty"` + Source string `json:"source,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + Profile string `json:"profile,omitempty"` + // API is "ok", "missing on API" or "" when no listing was available. + API string `json:"api_status,omitempty"` +} + +func newSavedKeyRow(name string, k config.StoredAccessKey, profile string) savedKeyRow { + created := "" + if !k.CreatedAt.IsZero() { + created = k.CreatedAt.UTC().Format(time.RFC3339) + } + return savedKeyRow{ + Name: name, AccessKeyID: k.AccessKeyID, StorageClass: k.StorageClass, Site: strings.ToUpper(k.Site), + ProjectID: k.ProjectID, Scope: k.Scope, Buckets: k.Buckets, Username: k.Username, Source: k.Source, + CreatedAt: created, Profile: profile, + } +} + +func (r savedKeyRow) TableRow() table.Row { + return table.Row{ + "name": {Label: "Name", Value: r.Name}, + "access_key_id": {Label: "Access Key ID", Value: r.AccessKeyID}, + "storage_class": {Label: "Class", Value: dash(r.StorageClass)}, + "region": {Label: "Site", Value: dash(r.Site)}, + "project": {Label: "Project", Value: dash(r.ProjectID)}, + "scope": {Label: "Scope", Value: dash(r.Scope)}, + "buckets": {Label: "Buckets", Value: r.bucketsCell(), MaxLength: 40}, + "source": {Label: "Source", Value: dash(r.Source)}, + "created_at": {Label: "Created", Value: dash(r.CreatedAt)}, + "status": {Label: "API", Value: dash(r.API)}, + } +} + +// bucketsCell renders "bkt_a=rw bkt_b=readonly" (sorted, truncated). +func (r savedKeyRow) bucketsCell() string { + if r.Scope == config.ScopeFullAccess { + return "all" + } + if len(r.Buckets) == 0 { + return emptyCell + } + return truncateText(formatPerms(r.Buckets), 40) +} + +// formatPerms renders a name→permission map as "a=rw b=readonly", sorted. +func formatPerms(perms map[string]string) string { + ids := make([]string, 0, len(perms)) + for id := range perms { + ids = append(ids, id) + } + sort.Strings(ids) + parts := make([]string, 0, len(ids)) + for _, id := range ids { + parts = append(parts, id+"="+perms[id]) + } + return strings.Join(parts, " ") +} + +// savedKeysByID indexes the active profile's keys by access key ID. Errors +// (no profile) yield an empty map so listings still work under LATITUDESH_TOKEN. +func savedKeysByID(cmd *cobra.Command) (map[string]string, string, map[string]config.StoredAccessKey) { + _, profileName, p, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return map[string]string{}, "", map[string]config.StoredAccessKey{} + } + keys := p.ObjectStorageKeys() + byID := make(map[string]string, len(keys)) + for name, k := range keys { + if k.AccessKeyID != "" { + byID[k.AccessKeyID] = name + } + } + return byID, profileName, keys +} + +// truncateText shortens s to max runes with an ellipsis. +func truncateText(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + if max <= 1 { + return "…" + } + return string(r[:max-1]) + "…" +} + +// dash is the access-keys spelling of the shared table placeholder. +func dash(s string) string { return orEmptyCell(s) } + +func ptrStr(s *string) string { + if s == nil { + return "" + } + return *s +} + +func firstNonEmptyStr(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +// storageClassFlag validates --storage-class. +func storageClassFlag(cmd *cobra.Command) (string, error) { + v, _ := cmd.Flags().GetString("storage-class") + v = strings.ToLower(strings.TrimSpace(v)) + switch v { + case "", objectstorage.ClassStandard, objectstorage.ClassHighPerformance: + return v, nil + case "hp", "high-performance", "highperformance": + return objectstorage.ClassHighPerformance, nil + } + return "", exitcode.Errorf(exitcode.Usage, "invalid --storage-class %q (use standard or high_performance)", v) +} + +// regionFlag returns --region upper-cased, rejecting AWS-looking values. +func regionFlag(cmd *cobra.Command) (string, error) { + v, _ := cmd.Flags().GetString("region") + v = strings.TrimSpace(v) + if v == "" { + return "", nil + } + if looksLikeAWSRegion(v) { + return "", exitcode.Errorf(exitcode.Usage, "--region %q is not a Latitude site; use a site slug such as DAL, NYC or TYO4 (see 'lsh regions list')", v) + } + return strings.ToUpper(v), nil +} diff --git a/cmd/s3/accesskeys_shared_test.go b/cmd/s3/accesskeys_shared_test.go new file mode 100644 index 0000000..707b215 --- /dev/null +++ b/cmd/s3/accesskeys_shared_test.go @@ -0,0 +1,457 @@ +package s3 + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func TestParseBucketSpec(t *testing.T) { + cases := []struct { + in string + token string + perm string + fail bool + }{ + {"backups", "backups", config.PermissionRW, false}, + {"backups=rw", "backups", config.PermissionRW, false}, + {"logs=readonly", "logs", config.PermissionReadOnly, false}, + {"logs:readonly", "logs", config.PermissionReadOnly, false}, + {"logs:ro", "logs", config.PermissionReadOnly, false}, + {"s3://backups/", "backups", config.PermissionRW, false}, + {"bkt_1Gjbang9n0L2w=readonly", "bkt_1Gjbang9n0L2w", config.PermissionReadOnly, false}, + {"backups=write", "backups", config.PermissionRW, false}, + {"backups=admin", "", "", true}, + {"=rw", "", "", true}, + {"", "", "", true}, + // A key part is a usage error: access keys cover whole buckets. + {"backups/2026/", "", "", true}, + {"s3://backups/dump.sql=rw", "", "", true}, + {"backups/dir:readonly", "", "", true}, + } + for _, c := range cases { + spec, err := parseBucketSpec(c.in) + if c.fail { + if err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("%q: want usage error, got spec=%+v err=%v", c.in, spec, err) + } + continue + } + if err != nil { + t.Errorf("%q: unexpected error %v", c.in, err) + continue + } + if spec.Token != c.token || spec.Permission != c.perm { + t.Errorf("%q: got %+v, want token=%q perm=%q", c.in, spec, c.token, c.perm) + } + } +} + +func TestParseBucketSpecsRejectsDuplicatesAndSplitsCommas(t *testing.T) { + specs, err := parseBucketSpecs([]string{"a=rw,b=readonly", "c"}) + if err != nil || len(specs) != 3 { + t.Fatalf("got %+v err=%v", specs, err) + } + if specs[1].Token != "b" || specs[1].Permission != config.PermissionReadOnly { + t.Fatalf("comma split failed: %+v", specs) + } + if _, err := parseBucketSpecs([]string{"a", "a=readonly"}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("duplicate must fail with exit 2, got %v", err) + } +} + +func TestValidateSameGroup(t *testing.T) { + std := func(name, project string) scopedBucket { + return scopedBucket{Bucket: &objectstorage.Bucket{Name: name, StorageClass: objectstorage.ClassStandard, ProjectID: project, Site: "DAL"}, Permission: "rw"} + } + hp := func(name, project, site string) scopedBucket { + return scopedBucket{Bucket: &objectstorage.Bucket{Name: name, StorageClass: objectstorage.ClassHighPerformance, ProjectID: project, Site: site}, Permission: "rw"} + } + if err := validateSameGroup([]scopedBucket{std("a", "p1"), std("b", "p1")}); err != nil { + t.Fatalf("same class/project must pass: %v", err) + } + // Standard keys span sites: differing sites are fine. + other := std("c", "p1") + other.Bucket.Site = "NYC" + if err := validateSameGroup([]scopedBucket{std("a", "p1"), other}); err != nil { + t.Fatalf("standard buckets in different sites must pass: %v", err) + } + err := validateSameGroup([]scopedBucket{std("a", "p1"), hp("b", "p1", "TYO4")}) + if err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("mixed classes must fail with exit 2, got %v", err) + } + if !strings.Contains(err.Error(), "class=standard") || !strings.Contains(err.Error(), "class=high_performance") || !strings.Contains(err.Error(), "a") { + t.Fatalf("error must list the groups: %v", err) + } + if err := validateSameGroup([]scopedBucket{hp("a", "p1", "TYO4"), hp("b", "p1", "DAL")}); err == nil { + t.Fatal("high_performance buckets in different sites must fail") + } + if err := validateSameGroup([]scopedBucket{std("a", "p1"), std("b", "p2")}); err == nil { + t.Fatal("different projects must fail") + } +} + +func TestNormalizeCreatedBothShapes(t *testing.T) { + req := accessKeyRequest{Project: "proj_1", StorageClass: objectstorage.ClassStandard, Site: "DAL", Name: "ci", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw"}} + s := func(v string) *string { return &v } + wasabi := &operations.PostStorageAccessKeysResponse{Object: &operations.PostStorageAccessKeysResponseBody{Data: &operations.PostStorageAccessKeysObjectStorageData{ + Attributes: &operations.PostStorageAccessKeysObjectStorageAttributes{AccessKey: &operations.AccessKey{ + AccessKeyID: s("WASABIID"), SecretAccessKey: s("wasabi-secret"), Name: s("ci"), Status: s("Active"), Username: s("u+ci@x"), + }}, + }}} + k, err := normalizeCreated(wasabi, req) + if err != nil { + t.Fatal(err) + } + if k.AccessKeyID != "WASABIID" || k.SecretAccessKey != "wasabi-secret" || k.Username != "u+ci@x" || k.Scope != config.ScopeLimitedAccess || k.Buckets["bkt_1"] != "rw" { + t.Fatalf("wasabi shape not normalized: %+v", *k) + } + + req.StorageClass = objectstorage.ClassHighPerformance + req.Site = "TYO4" + vast := &operations.PostStorageAccessKeysResponse{Object: &operations.PostStorageAccessKeysResponseBody{Data: &operations.PostStorageAccessKeysObjectStorageData{ + Attributes: &operations.PostStorageAccessKeysObjectStorageAttributes{AccessKey: &operations.AccessKey{ + AccessKey: s("VASTID"), SecretKey: s("vast-secret"), Status: s("Active"), + }}, + }}} + k, err = normalizeCreated(vast, req) + if err != nil { + t.Fatal(err) + } + if k.AccessKeyID != "VASTID" || k.SecretAccessKey != "vast-secret" || k.Name != "ci" || k.Site != "TYO4" { + t.Fatalf("vast shape not normalized: %+v", *k) + } + // Both shapes marshal to identical field names. + b, _ := json.Marshal(k) + for _, field := range []string{`"access_key_id"`, `"secret_access_key"`, `"storage_class"`, `"scope"`} { + if !strings.Contains(string(b), field) { + t.Fatalf("JSON missing %s: %s", field, b) + } + } + if strings.Contains(string(b), `"secret_key"`) || strings.Contains(string(b), `"access_key":`) { + t.Fatalf("backend field names leaked into JSON: %s", b) + } + + if _, err := normalizeCreated(&operations.PostStorageAccessKeysResponse{}, req); err == nil { + t.Fatal("empty response must fail") + } + noSecret := &operations.PostStorageAccessKeysResponse{Object: &operations.PostStorageAccessKeysResponseBody{Data: &operations.PostStorageAccessKeysObjectStorageData{ + Attributes: &operations.PostStorageAccessKeysObjectStorageAttributes{AccessKey: &operations.AccessKey{AccessKeyID: s("X")}}, + }}} + if _, err := normalizeCreated(noSecret, req); err == nil { + t.Fatal("response without secret must fail") + } +} + +func TestBuildCreateRequestSortsBuckets(t *testing.T) { + req := accessKeyRequest{Project: "p", StorageClass: objectstorage.ClassStandard, Site: "DAL", Name: "n", Scope: config.ScopeLimitedAccess, + Buckets: map[string]string{"bkt_b": "readonly", "bkt_a": "rw"}} + body := buildCreateRequest(req) + a := body.Data.Attributes + if a.AccessScope != operations.AccessScopeLimitedAccess || a.Region != "DAL" || len(a.BucketPermissions) != 2 { + t.Fatalf("unexpected attributes: %+v", a) + } + if a.BucketPermissions[0].BucketID != "bkt_a" || a.BucketPermissions[0].Permission != operations.PermissionRw || a.BucketPermissions[1].Permission != operations.PermissionReadonly { + t.Fatalf("permissions not sorted/mapped: %+v", a.BucketPermissions) + } + req.Scope = config.ScopeFullAccess + if body := buildCreateRequest(req); len(body.Data.Attributes.BucketPermissions) != 0 { + t.Fatal("fullaccess must not send bucket_permissions") + } +} + +func TestAccessKeyRequestValidate(t *testing.T) { + base := accessKeyRequest{Project: "p", StorageClass: objectstorage.ClassStandard, Site: "DAL", Name: "n", Scope: config.ScopeFullAccess} + if err := base.validate(); err != nil { + t.Fatalf("valid request rejected: %v", err) + } + hp := base + hp.StorageClass = objectstorage.ClassHighPerformance + hp.Site = "" + err := hp.validate() + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "TYO4") { + t.Fatalf("high_performance without site must name the expected value, got %v", err) + } + limited := base + limited.Scope = config.ScopeLimitedAccess + if err := limited.validate(); err == nil { + t.Fatal("limited_access without buckets must fail") + } + limited.Buckets = map[string]string{"bkt_1": "admin"} + if err := limited.validate(); err == nil { + t.Fatal("invalid permission must fail") + } +} + +func TestDecideSecretDisplay(t *testing.T) { + cases := []struct { + human, save, show, explicit bool + wantShow bool + wantWarn bool + }{ + {human: true, wantShow: true}, // TTY, not saving: shown once in clear + {human: true, save: true, wantShow: false, wantWarn: true}, // saved: not printed + {human: true, save: true, show: true, wantShow: true}, // saved + --show-secret + {human: false, explicit: true, wantShow: true}, // -o json explicit + {human: false, explicit: false, wantShow: false, wantWarn: true}, // LSH_OUTPUT=json: omitted + warning + {human: false, explicit: false, show: true, wantShow: true}, // --show-secret overrides + {human: false, save: true, explicit: true, wantShow: false, wantWarn: true}, // saving wins over explicit -o + {human: false, save: true, show: true, wantShow: true}, + } + for i, c := range cases { + d := decideSecretDisplay(c.human, c.save, c.show, c.explicit) + if d.Show != c.wantShow || (d.Warning != "") != c.wantWarn { + t.Errorf("case %d (%+v): got show=%v warn=%q", i, c, d.Show, d.Warning) + } + } +} + +func TestTruncateAndFormatHelpers(t *testing.T) { + if got := truncateText("abcdef", 4); got != "abc…" { + t.Fatalf("truncateText = %q", got) + } + if got := truncateText("abc", 4); got != "abc" { + t.Fatalf("truncateText short = %q", got) + } + if got := formatPerms(map[string]string{"bkt_b": "rw", "bkt_a": "readonly"}); got != "bkt_a=readonly bkt_b=rw" { + t.Fatalf("formatPerms = %q", got) + } + if !looksLikeAWSRegion("us-east-1") || looksLikeAWSRegion("TYO4") || looksLikeAWSRegion("DAL") { + t.Fatal("AWS region detection is off") + } +} + +func TestRotatedName(t *testing.T) { + now := mustTime(t, "2026-09-07T12:00:00Z") + if got := rotatedName("ci-deploy", now, 1); got != "ci-deploy-20260907" { + t.Fatalf("rotatedName = %q", got) + } + req, err := rotateRequest(config.StoredAccessKey{ + AccessKeyID: "OLD", StorageClass: objectstorage.ClassHighPerformance, Site: "tyo4", ProjectID: "proj_1", + Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": "rw"}, + }, "", "", "ci-deploy-20260907") + if err != nil { + t.Fatal(err) + } + if req.Project != "proj_1" || req.Site != "TYO4" || req.Scope != config.ScopeLimitedAccess || req.Buckets["bkt_1"] != "rw" || req.Name != "ci-deploy-20260907" { + t.Fatalf("rotateRequest = %+v", req) + } + if _, err := rotateRequest(config.StoredAccessKey{Scope: config.ScopeUnknown}, "p", "", "n"); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("unknown scope must fail with exit 2, got %v", err) + } + if _, err := rotateRequest(config.StoredAccessKey{Scope: config.ScopeFullAccess, StorageClass: "standard"}, "", "", "n"); err == nil { + t.Fatal("missing project must fail") + } +} + +func TestValidateAccessKeyName(t *testing.T) { + if err := validateAccessKeyName("ci-deploy"); err != nil { + t.Errorf("valid name rejected: %v", err) + } + if err := validateAccessKeyName("this-name-is-way-too-long-for-the-api"); err == nil { + t.Error("over-long name must be rejected") + } +} + +// lagError reproduces what the API returns while the backend converges: +// 404 STORAGE_RESOURCE_NOT_FOUND, humanized by objectstorage.HumanizeAPI. +func lagError() error { + return exitcode.Errorf(exitcode.NotFound, "not found: Storage resource not found.") +} + +func TestIsProvisioningLag(t *testing.T) { + cases := []struct { + err error + want bool + }{ + {lagError(), true}, + {exitcode.Errorf(exitcode.Generic, "the API returned 502: Storage service is temporarily unavailable. Please try again."), true}, + {exitcode.Errorf(exitcode.Usage, "the API rejected the request: name size cannot be greater than 25"), false}, + {exitcode.Errorf(exitcode.NotFound, "project %q not found", "nope"), false}, + } + for _, c := range cases { + if got := isProvisioningLag(c.err); got != c.want { + t.Errorf("isProvisioningLag(%q) = %v, want %v", c.err, got, c.want) + } + } +} + +// TestRetryCreateProvisioningLag covers the fix for a key created right after +// its bucket: the first attempts fail with the propagation 404 and the same +// request is re-sent (same name) until it succeeds. +func TestRetryCreateProvisioningLag(t *testing.T) { + restore := provisioningRetryDelays + provisioningRetryDelays = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond} + defer func() { provisioningRetryDelays = restore }() + + var names []string + calls := 0 + req := accessKeyRequest{Name: "key-brave-otter-std", StorageClass: objectstorage.ClassStandard} + created, err := retryCreate(context.Background(), req, true, func(r accessKeyRequest) (*createdAccessKey, error) { + calls++ + names = append(names, r.Name) + if calls < 3 { + return nil, lagError() + } + return &createdAccessKey{Name: r.Name}, nil + }) + if err != nil { + t.Fatalf("retryCreate: %v", err) + } + if created.Name != "key-brave-otter-std" { + t.Errorf("created name = %q, want the requested one", created.Name) + } + if calls != 3 { + t.Errorf("calls = %d, want 3", calls) + } + for _, n := range names { + if n != "key-brave-otter-std" { + t.Errorf("name changed to %q; a propagation retry must re-send the same request", n) + } + } +} + +func TestRetryCreateProvisioningLagExhausted(t *testing.T) { + restore := provisioningRetryDelays + provisioningRetryDelays = []time.Duration{time.Millisecond, time.Millisecond} + defer func() { provisioningRetryDelays = restore }() + + calls := 0 + _, err := retryCreate(context.Background(), accessKeyRequest{Name: "key-x"}, true, func(accessKeyRequest) (*createdAccessKey, error) { + calls++ + return nil, lagError() + }) + if err == nil { + t.Fatal("expected an error once the retries are exhausted") + } + if calls != 3 { + t.Errorf("calls = %d, want 3 (initial + 2 retries)", calls) + } + if code := exitcode.Of(err); code != exitcode.NotFound { + t.Errorf("exit code = %d, want %d", code, exitcode.NotFound) + } + if !strings.Contains(err.Error(), "retry in a few seconds") { + t.Errorf("error %q should tell the user the failure is transient", err) + } +} + +func TestRetryCreateNameConflict(t *testing.T) { + calls := 0 + var names []string + req := accessKeyRequest{Name: "key-taken", StorageClass: objectstorage.ClassStandard} + if _, err := retryCreate(context.Background(), req, true, func(r accessKeyRequest) (*createdAccessKey, error) { + calls++ + names = append(names, r.Name) + if calls < 2 { + return nil, exitcode.Errorf(exitcode.Usage, "the API rejected the request: name has already been taken") + } + return &createdAccessKey{Name: r.Name}, nil + }); err != nil { + t.Fatalf("retryCreate: %v", err) + } + if len(names) != 2 || names[0] == names[1] { + t.Errorf("names = %v; a conflict must re-roll the generated name", names) + } + + // An explicit --name is never rewritten: the conflict surfaces instead. + calls = 0 + if _, err := retryCreate(context.Background(), req, false, func(accessKeyRequest) (*createdAccessKey, error) { + calls++ + return nil, exitcode.Errorf(exitcode.Usage, "the API rejected the request: name has already been taken") + }); err == nil { + t.Fatal("expected the conflict to surface for an explicit name") + } + if calls != 1 { + t.Errorf("calls = %d, want 1", calls) + } +} + +func TestSleepCtxCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := sleepCtx(ctx, time.Hour) + if err == nil { + t.Fatal("expected an error for a cancelled context") + } + if code := exitcode.Of(err); code != exitcode.Interrupted { + t.Errorf("exit code = %d, want %d", code, exitcode.Interrupted) + } +} + +func TestImportHint(t *testing.T) { + created := &createdAccessKey{AccessKeyID: "AKIAEXAMPLE"} + limited := accessKeyRequest{Project: "my-project", StorageClass: objectstorage.ClassStandard, Scope: config.ScopeLimitedAccess} + got := importHint("k1", created, limited, map[string]string{"logs": config.PermissionReadOnly, "backups": config.PermissionRW}) + want := "lsh s3 access-keys import --name k1 --access-key-id AKIAEXAMPLE --project my-project --storage-class standard --bucket backups=rw --bucket logs=readonly" + if got != want { + t.Errorf("limited hint =\n%s\nwant\n%s", got, want) + } + + full := accessKeyRequest{Project: "p", StorageClass: objectstorage.ClassHighPerformance, Site: "TYO4", Scope: config.ScopeFullAccess} + got = importHint("k2", created, full, nil) + want = "lsh s3 access-keys import --name k2 --access-key-id AKIAEXAMPLE --project p --storage-class high_performance --region TYO4 --all-buckets" + if got != want { + t.Errorf("fullaccess hint =\n%s\nwant\n%s", got, want) + } +} + +// TestRotatedNameSequenceAndLimit covers same-day re-rotation (a deterministic +// name would collide on the API) and the 25-character name cap. +func TestRotatedNameSequenceAndLimit(t *testing.T) { + now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) + cases := []struct { + base string + seq int + want string + }{ + {"ci-deploy", 1, "ci-deploy-20260907"}, + {"ci-deploy", 2, "ci-deploy-20260907-2"}, + {"", 1, "lsh-key-20260907"}, + // 20 chars + "-20260907" would be 29: the base is trimmed instead. + {"backups-nightly-full", 1, "backups-nightly-20260907"}, + {"backups-nightly-full", 3, "backups-nightl-20260907-3"}, + } + for _, c := range cases { + got := rotatedName(c.base, now, c.seq) + if got != c.want { + t.Errorf("rotatedName(%q, seq=%d) = %q, want %q", c.base, c.seq, got, c.want) + } + if len(got) > maxAccessKeyNameLen { + t.Errorf("rotatedName(%q, seq=%d) = %q is %d chars, over the API limit", c.base, c.seq, got, len(got)) + } + } +} + +// TestMatchSavedFiltersLikeMatchAPI locks the two rules matchSaved was missing: +// an empty site is a wildcard (as in matchAPI) and a project reference only +// filters when it is comparable with what the key stored. +func TestMatchSavedFiltersLikeMatchAPI(t *testing.T) { + standard := config.StoredAccessKey{StorageClass: objectstorage.ClassStandard, ProjectID: "proj_1"} + cases := []struct { + name string + filter listFilter + key config.StoredAccessKey + want bool + }{ + {"no filter", listFilter{}, standard, true}, + {"site filter, key without a site", listFilter{Site: "DAL"}, standard, true}, + {"site filter, key in another site", listFilter{Site: "DAL"}, config.StoredAccessKey{Site: "NYC"}, false}, + {"site filter, same site", listFilter{Site: "DAL"}, config.StoredAccessKey{Site: "dal"}, true}, + {"project slug vs stored ID", listFilter{Project: "my-project"}, standard, true}, + {"project ID mismatch", listFilter{Project: "proj_2"}, standard, false}, + {"project ID match", listFilter{Project: "proj_1"}, standard, true}, + {"class mismatch", listFilter{StorageClass: objectstorage.ClassHighPerformance}, standard, false}, + } + for _, c := range cases { + if got := c.filter.matchSaved(c.key); got != c.want { + t.Errorf("%s: matchSaved = %v, want %v", c.name, got, c.want) + } + } +} diff --git a/cmd/s3/accesskeys_test.go b/cmd/s3/accesskeys_test.go new file mode 100644 index 0000000..b3e1e7f --- /dev/null +++ b/cmd/s3/accesskeys_test.go @@ -0,0 +1,158 @@ +package s3 + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/lsh/internal/config" + homedir "github.com/mitchellh/go-homedir" +) + +// fakeAPI is a minimal Latitude API for the access-keys endpoints. It records +// the requests it receives and answers with canned JSON:API documents. +type fakeAPI struct { + mu sync.Mutex + srv *httptest.Server + requests []fakeRequest + // createShape is "wasabi" (access_key_id/secret_access_key) or "vast" + // (access_key/secret_key). + createShape string + // keys is the GET /storage/access_keys document keyed by project. + keys map[string]string + // buckets is the GET /storage/buckets document. + buckets string + // deleteStatus is returned by DELETE (204 by default). + deleteStatus int +} + +type fakeRequest struct { + Method string + Path string + Query url.Values + Body string +} + +func newFakeAPI(t *testing.T) *fakeAPI { + t.Helper() + f := &fakeAPI{createShape: "wasabi", keys: map[string]string{}, deleteStatus: http.StatusNoContent} + f.srv = httptest.NewServer(http.HandlerFunc(f.handle)) + t.Cleanup(f.srv.Close) + return f +} + +// client builds a keysAPI whose SDK client and raw GET both point at the fake +// server. +func (f *fakeAPI) client() *keysAPI { + return &keysAPI{ + sdk: sdk.New(sdk.WithServerURL(f.srv.URL), sdk.WithSecurity("test-token")), + baseURL: f.srv.URL, + token: "test-token", + } +} + +func (f *fakeAPI) Requests() []fakeRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]fakeRequest(nil), f.requests...) +} + +func (f *fakeAPI) handle(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + f.mu.Lock() + f.requests = append(f.requests, fakeRequest{Method: r.Method, Path: r.URL.Path, Query: r.URL.Query(), Body: string(body)}) + f.mu.Unlock() + w.Header().Set("Content-Type", "application/vnd.api+json") + + switch { + case r.Method == http.MethodPost && r.URL.Path == "/storage/access_keys": + var req struct { + Data struct { + Attributes struct { + Name string `json:"name"` + } `json:"attributes"` + } `json:"data"` + } + _ = json.Unmarshal(body, &req) + name := strings.ToLower(req.Data.Attributes.Name) + w.WriteHeader(http.StatusCreated) + if f.createShape == "vast" { + io.WriteString(w, `{"data":{"type":"access_keys","attributes":{"access_key":{"access_key":"VASTKEYID","secret_key":"vast-secret-value","name":"`+name+`","status":"Active","username":"user_1-`+name+`"}}}}`) + return + } + io.WriteString(w, `{"data":{"type":"access_keys","attributes":{"access_key":{"access_key_id":"WASABIKEYID","secret_access_key":"wasabi-secret-value","name":"`+name+`","status":"Active","username":"someone+`+name+`@example.com"}}}}`) + case r.Method == http.MethodGet && r.URL.Path == "/storage/access_keys": + doc, ok := f.keys[r.URL.Query().Get("project")] + if !ok { + w.WriteHeader(http.StatusNotFound) + io.WriteString(w, `{"errors":[{"status":"404","title":"Not Found","detail":"project not found"}]}`) + return + } + io.WriteString(w, doc) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/storage/access_keys/"): + w.WriteHeader(f.deleteStatus) + if f.deleteStatus == http.StatusNotFound { + io.WriteString(w, `{"errors":[{"status":"404","title":"Not Found","detail":"access key not found"}]}`) + } + case r.Method == http.MethodGet && r.URL.Path == "/storage/buckets": + if f.buckets == "" { + io.WriteString(w, `{"data":[]}`) + return + } + io.WriteString(w, f.buckets) + default: + w.WriteHeader(http.StatusNotFound) + io.WriteString(w, `{"errors":[{"status":"404","title":"Not Found"}]}`) + } +} + +// keysDocument is the canned GET /storage/access_keys payload: one standard +// fullaccess key, one standard limited key and one high_performance key, +// including the buckets[]/access fields the SDK drops. +const keysDocument = `{"data":{ + "standard":[ + {"name":"ops","username":"ops@example.com","access_key_id":"OPSKEYID000000000001","status":"Active","created_at":"2026-09-01T10:00:00Z","region":"DAL","access":"fullaccess","buckets":["backups-7f3a","logs-91aa"]}, + {"name":"ci-deploy","username":"ci@example.com","access_key_id":"CIKEYID0000000000002","status":"Active","created_at":"2026-09-02T10:00:00Z","region":"DAL","access":"rw","buckets":["backups-7f3a"]} + ], + "high_performance":[ + {"name":"fast","username":"user_1-fast","access_key_id":"HPKEYID0000000000003","status":"Active","created_at":"2026-09-03T10:00:00Z","region":"TYO4","access":"readonly","buckets":["fast-bucket-01"]} + ]}}` + +// withTempProfile points the config file at a temporary HOME holding one +// profile named "test" with the given saved keys, and returns the directory. +func withTempProfile(t *testing.T, keys map[string]config.StoredAccessKey) string { + t.Helper() + dir := t.TempDir() + homedir.DisableCache = true + t.Setenv("HOME", dir) + t.Setenv("LSH_PROFILE", "") + t.Setenv("LATITUDESH_TOKEN", "") + f := &config.File{ + DefaultProfile: "test", + Profiles: map[string]config.Profile{ + "test": {Authorization: "tok", Email: "Lanusse.Morais@latitude.sh", ObjectStorage: &config.ObjectStorageConfig{Keys: keys}}, + }, + } + if err := config.Save(f); err != nil { + t.Fatalf("save config: %v", err) + } + return dir +} + +// readConfigRaw returns the on-disk config so tests can assert on it. +func readConfigRaw(t *testing.T, home string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(home, ".config", "lsh", "config.json")) + if err != nil { + t.Fatalf("read config: %v", err) + } + return string(b) +} diff --git a/cmd/s3/bucket_render.go b/cmd/s3/bucket_render.go new file mode 100644 index 0000000..4292344 --- /dev/null +++ b/cmd/s3/bucket_render.go @@ -0,0 +1,118 @@ +package s3 + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" +) + +// BucketRow wraps the SDK bucket payload so the shared renderer can print it +// (-o table|json|yaml|csv, --query). JSON output is the API document as-is. +type BucketRow struct { + components.ObjectStorageData + // Site is the Latitude site slug when known (the SDK model drops it). + Site string `json:"-"` + // resolved is set when the row came from a resolved bucket. It carries the + // fields that exist without an API document (endpoint-override mode). + resolved *objectstorage.Bucket +} + +// MarshalJSON emits the API document plus the site slug (which the SDK model +// drops) so `-o json` carries everything `stat` shows. +func (m *BucketRow) MarshalJSON() ([]byte, error) { + raw, err := json.Marshal(m.ObjectStorageData) + if err != nil { + return nil, err + } + var doc map[string]json.RawMessage + if unmarshalErr := json.Unmarshal(raw, &doc); unmarshalErr != nil { + doc = nil + } + if len(doc) == 0 && m.resolved != nil { + // No API payload (endpoint-override mode): describe the bucket from + // what was resolved locally instead of emitting an empty object. + return json.Marshal(map[string]string{ + "name": m.resolved.Name, + "bucket_name": m.resolved.BucketName, + "endpoint": m.resolved.Endpoint, + "signing_region": m.resolved.SigningRegion, + }) + } + if m.Site == "" || doc == nil { + return raw, nil + } + site, _ := json.Marshal(m.Site) + doc["site"] = site + return json.Marshal(doc) +} + +// NewBucketRow builds a row from a resolved bucket. +func NewBucketRow(b *objectstorage.Bucket) *BucketRow { + return &BucketRow{ObjectStorageData: b.Data, Site: b.Site, resolved: b} +} + +// BucketRows converts SDK payloads into renderable rows. +func BucketRows(data []components.ObjectStorageData, sites map[string]string) []renderer.ResponseData { + out := make([]renderer.ResponseData, 0, len(data)) + for i := range data { + row := &BucketRow{ObjectStorageData: data[i]} + if data[i].ID != nil && sites != nil { + row.Site = sites[*data[i].ID] + } + out = append(out, row) + } + return out +} + +func (m *BucketRow) TableRow() table.Row { + b := objectstorage.BucketFromData(m.ObjectStorageData) + site := m.Site + if site == "" { + site = b.Site + } + if site == "" { + site = b.City + } + created := "" + if b.CreatedAt != nil { + created = objectstorage.FormatTime(*b.CreatedAt) + } + return table.Row{ + "id": {Label: "ID", Value: b.ID}, + "name": {Label: "Name", Value: b.Name}, + "bucket_name": {Label: "Bucket Name", Value: b.BucketName}, + "project": {Label: "Project", Value: b.ProjectRef()}, + "storage_class": {Label: "Class", Value: b.StorageClass}, + "region": {Label: "Site", Value: site}, + "endpoint": {Label: "Endpoint", Value: b.Endpoint, MaxLength: 45}, + "versioning": {Label: "Versioning", Value: yesNo(b.Versioning)}, + "locking": {Label: "Locking", Value: lockingLabel(b)}, + "created_at": {Label: "Created At", Value: created}, + } +} + +func yesNo(v bool) string { + if v { + return "yes" + } + return "no" +} + +func lockingLabel(b *objectstorage.Bucket) string { + if !b.Locking { + return "no" + } + mode := strings.ToUpper(b.RetentionMode) + if mode == "" || mode == "NONE" { + return "yes" + } + if b.RetentionDays > 0 { + return fmt.Sprintf("%s (%dd)", mode, b.RetentionDays) + } + return mode +} diff --git a/cmd/s3/configure.go b/cmd/s3/configure.go new file mode 100644 index 0000000..4b38c6a --- /dev/null +++ b/cmd/s3/configure.go @@ -0,0 +1,965 @@ +package s3 + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +// Flag names of `lsh s3 configure`. +const ( + flagCfgStorageClass = "storage-class" + flagCfgRegion = "region" + flagCfgAllBuckets = "all-buckets" + flagCfgBucket = "bucket" + flagCfgName = "name" + flagCfgSave = "save" + flagCfgShowSecret = "show-secret" +) + +// configureNonInteractiveHint lists the alternatives to the wizard for +// scripts and CI (exit 2 when a prompt would be needed without a terminal). +const configureNonInteractiveHint = "pass every input as a flag (--project, --storage-class, --region, --all-buckets or --bucket [=rw|readonly]), " + + "or run 'lsh s3 access-keys create --bucket --save', or set " + objectstorage.EnvAccessKeyID + "/" + objectstorage.EnvSecretAccessKey + +// NewConfigureCmd builds `lsh s3 configure` (alias `credentials`): the +// interactive equivalent of `aws configure` for S3 access keys. +func NewConfigureCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "configure", + Aliases: []string{"credentials"}, + GroupID: groupCredentials, + Short: "Create an access key and save it for this machine", + Long: `Create an S3 access key and save it in the active lsh profile so the object +commands (list, copy, move, delete, get, presign) can use it automatically. + +This is the path for the machine you are typing on. For an application, a CI +job or a colleague, create a separate key with 'lsh s3 access-keys create': +it prints the key once (and saves it here too with --save) instead of +walking through the wizard. + +The wizard asks for the project, the storage class (standard = general +purpose; high_performance = low latency on selected sites), the site for +high_performance, the scope (all buckets of that class in the project, or +specific buckets with rw/readonly permission) and a name. Every question can +be answered with a flag; when all of them are given the wizard runs without +prompts, which is how scripts should call it. The site of a standard key is +inferred from the project's standard buckets when --region is omitted. + +The access key is separate from your API token. It is stored in +~/.config/lsh/config.json (mode 0600) and never printed unless --save=false +is passed, in which case it is shown once in clear and not stored.`, + Example: ` lsh s3 configure + lsh s3 configure --project my-project --storage-class standard --all-buckets + lsh s3 configure --project my-project --storage-class high_performance --region TYO4 --all-buckets + lsh s3 configure --project my-project --bucket backups=rw --bucket logs=readonly --name laptop + lsh s3 configure --project my-project --storage-class standard --all-buckets --save=false -o json`, + Args: cobra.NoArgs, + RunE: runConfigure, + }) + addProjectFlag(cmd, true, "project ID or slug the key belongs to (default: LSH_PROJECT, or a menu)") + cmd.Flags().String(flagCfgStorageClass, "", "storage class of the key: standard | high_performance") + cmd.Flags().String(flagCfgRegion, "", "site slug (DAL, TYO4…) of the key; inferred from the project's buckets when omitted") + cmd.Flags().Bool(flagCfgAllBuckets, false, "key for all buckets of the class (and site) in the project (fullaccess)") + cmd.Flags().StringArray(flagCfgBucket, nil, "restrict the key to a bucket: [=rw|readonly] (repeatable)") + cmd.Flags().String(flagCfgName, "", "key name (default: lsh--)") + cmd.Flags().Bool(flagCfgSave, true, "save the key in the active profile; --save=false prints the secret once instead") + cmd.Flags().Bool(flagCfgShowSecret, false, "include the secret in structured output even when -o comes from the environment") + cmd.AddCommand(newConfigureExportCmd()) + return cmd +} + +// configureOptions collects the flag values of `lsh s3 configure`. +type configureOptions struct { + Project string + StorageClass string + Site string + AllBuckets bool + Buckets []bucketSpec + Name string + Save bool +} + +// parseConfigureOptions reads and validates the flags. +func parseConfigureOptions(cmd *cobra.Command) (configureOptions, error) { + o := configureOptions{Project: projectFlag(cmd)} + rawClass, _ := cmd.Flags().GetString(flagCfgStorageClass) + class, err := objectstorage.ParseStorageClass(rawClass) + if err != nil { + return o, err + } + o.StorageClass = class + if o.Site, err = regionFlag(cmd); err != nil { + return o, err + } + o.AllBuckets, _ = cmd.Flags().GetBool(flagCfgAllBuckets) + values, _ := cmd.Flags().GetStringArray(flagCfgBucket) + if o.Buckets, err = parseBucketSpecs(values); err != nil { + return o, err + } + if o.AllBuckets && len(o.Buckets) > 0 { + return o, objectstorage.ErrUsagef("--all-buckets and --bucket are mutually exclusive") + } + o.Name, _ = cmd.Flags().GetString(flagCfgName) + o.Name = strings.TrimSpace(o.Name) + o.Save, _ = cmd.Flags().GetBool(flagCfgSave) + return o, nil +} + +// cfgSelectedBucket is a bucket chosen for a limited_access key. +type cfgSelectedBucket struct { + Bucket *objectstorage.Bucket + Permission string +} + +// cfgValidateSelection checks that every selected bucket shares the storage +// class, site and project, which is what the API requires of one key. It +// returns the common values. +func cfgValidateSelection(sel []cfgSelectedBucket) (class, site, projectID string, err error) { + if len(sel) == 0 { + return "", "", "", objectstorage.ErrUsagef("no bucket selected") + } + first := sel[0].Bucket + class, site, projectID = first.StorageClass, first.Site, first.ProjectID + for _, s := range sel[1:] { + b := s.Bucket + if b.StorageClass != class { + return "", "", "", objectstorage.ErrUsagef("buckets %s (%s) and %s (%s) have different storage classes; one key covers a single class", + first.Display(), class, b.Display(), b.StorageClass) + } + if class == objectstorage.ClassHighPerformance && site != "" && b.Site != "" && !strings.EqualFold(site, b.Site) { + return "", "", "", objectstorage.ErrUsagef("buckets %s (%s) and %s (%s) live on different sites; a high_performance key covers a single site", + first.Display(), site, b.Display(), b.Site) + } + if projectID != "" && b.ProjectID != "" && projectID != b.ProjectID { + return "", "", "", objectstorage.ErrUsagef("buckets %s (%s) and %s (%s) belong to different projects; one key covers a single project", + first.Display(), first.ProjectRef(), b.Display(), b.ProjectRef()) + } + if site == "" { + site = b.Site + } + if projectID == "" { + projectID = b.ProjectID + } + } + return class, site, projectID, nil +} + +// configurePlan is the structured description of the key the wizard creates +// (rendered for --dry-run and as the result when the key is saved). +type configurePlan struct { + Name string `json:"name"` + Project string `json:"project"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Scope string `json:"scope"` + Buckets map[string]string `json:"buckets,omitempty"` + Save bool `json:"save"` + DryRun bool `json:"dry_run,omitempty"` + AccessKeyID string `json:"access_key_id,omitempty"` + // SavedAs is the name the key was stored under, which differs from Name + // when that name already held another key (saveNewKey appends -2, -3…). + SavedAs string `json:"saved_as,omitempty"` + Profile string `json:"profile,omitempty"` +} + +func (p configurePlan) TableRow() table.Row { + status := "created" + if p.DryRun { + status = "dryrun" + } + row := table.Row{ + "name": {Label: "Name", Value: p.Name}, + "project": {Label: "Project", Value: p.Project}, + "storage_class": {Label: "Class", Value: p.StorageClass}, + "site": {Label: "Site", Value: orEmptyCell(p.Site)}, + "scope": {Label: "Scope", Value: p.Scope}, + "buckets": {Label: "Buckets", Value: cfgFormatBucketPerms(p.Buckets), MaxLength: 60}, + "save": {Label: "Saved", Value: yesNo(p.Save && !p.DryRun)}, + "status": {Label: "Status", Value: status}, + } + if p.AccessKeyID != "" { + row["access_key_id"] = table.Cell{Label: "Access Key ID", Value: p.AccessKeyID} + } + if p.SavedAs != "" { + row["saved_as"] = table.Cell{Label: "Saved As", Value: p.SavedAs} + } + return row +} + +// cfgFormatBucketPerms renders "a=rw b=readonly" sorted by name (the shared +// empty-cell placeholder when empty). +func cfgFormatBucketPerms(perms map[string]string) string { + if len(perms) == 0 { + return emptyCell + } + parts := make([]string, 0, len(perms)) + for name, perm := range perms { + parts = append(parts, name+"="+perm) + } + sort.Strings(parts) + return strings.Join(parts, " ") +} + +// configureSecretResult is the --save=false output: the key shown once. +type configureSecretResult struct { + Name string `json:"name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey *string `json:"secret_access_key"` + Scope string `json:"scope"` + StorageClass string `json:"storage_class"` + Site string `json:"site,omitempty"` + Project string `json:"project"` + Endpoint string `json:"endpoint,omitempty"` + SigningRegion string `json:"signing_region,omitempty"` + Buckets map[string]string `json:"buckets,omitempty"` +} + +func (r configureSecretResult) TableRow() table.Row { + return table.Row{ + "name": {Label: "Name", Value: r.Name}, + "access_key_id": {Label: "Access Key ID", Value: r.AccessKeyID}, + "scope": {Label: "Scope", Value: r.Scope}, + "storage_class": {Label: "Class", Value: r.StorageClass}, + "site": {Label: "Site", Value: orEmptyCell(r.Site)}, + "project": {Label: "Project", Value: r.Project}, + "buckets": {Label: "Buckets", Value: cfgFormatBucketPerms(r.Buckets), MaxLength: 60}, + } +} + +// configureNeedsPrompt returns the exit-2 error for a question the wizard +// cannot ask without a terminal. +func configureNeedsPrompt(what string) error { + return exitcode.Errorf(exitcode.Usage, "lsh s3 configure needs a terminal to ask for %s; %s", what, configureNonInteractiveHint) +} + +// cfgProjectChoice is one project offered by the wizard. +type cfgProjectChoice struct { + ID string + Slug string + Name string +} + +// configureListProjects fetches the team's projects (all pages, capped). +func configureListProjects(ctx context.Context, api *sdk.Latitudesh) ([]cfgProjectChoice, error) { + size := int64(100) + resp, err := api.Projects.List(ctx, operations.GetProjectsRequest{PageSize: &size}, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "projects") + } + var out []cfgProjectChoice + for page := 0; resp != nil && resp.Projects != nil && page < 20; page++ { + for _, p := range resp.Projects.Data { + c := cfgProjectChoice{ID: ptrStr(p.ID)} + if p.Attributes != nil { + c.Slug = ptrStr(p.Attributes.Slug) + c.Name = ptrStr(p.Attributes.Name) + } + if c.ID != "" { + out = append(out, c) + } + } + if resp.Next == nil { + break + } + next, err := resp.Next() + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "projects") + } + if next == nil { + break + } + resp = next + } + sort.Slice(out, func(i, j int) bool { return out[i].label() < out[j].label() }) + return out, nil +} + +func (p cfgProjectChoice) label() string { + switch { + case p.Slug != "" && p.Name != "" && p.Slug != p.Name: + return fmt.Sprintf("%s (%s)", p.Slug, p.Name) + case p.Slug != "": + return p.Slug + case p.Name != "": + return p.Name + } + return p.ID +} + +// cfgResolveProjectID turns a project token (ID or slug) into the proj_ ID the +// saved key is bound to. Unknown tokens are returned unchanged. +func cfgResolveProjectID(ctx context.Context, api *sdk.Latitudesh, token string) string { + if token == "" || strings.HasPrefix(token, "proj_") || api == nil { + return token + } + projects, err := configureListProjects(ctx, api) + if err != nil { + return token + } + for _, p := range projects { + if p.Slug == token { + return p.ID + } + } + for _, p := range projects { + if p.Name == token { + return p.ID + } + } + return token +} + +// cfgSiteChoice is one region offered by the wizard. +type cfgSiteChoice struct { + Slug string + Name string + Country string +} + +func (s cfgSiteChoice) label() string { + return joinNonEmpty(" — ", s.Slug, joinNonEmpty(", ", s.Name, s.Country)) +} + +// configureListSites fetches every region (including storage-only ones). +func configureListSites(ctx context.Context, api *sdk.Latitudesh) ([]cfgSiteChoice, error) { + size := int64(100) + includeCustom := true + resp, err := api.Regions.Get(ctx, operations.GetRegionsRequest{IncludeCustom: &includeCustom, PageSize: &size}, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "regions") + } + var out []cfgSiteChoice + for page := 0; resp != nil && resp.Regions != nil && page < 20; page++ { + for _, r := range resp.Regions.Data { + if r.Attributes == nil { + continue + } + c := cfgSiteChoice{Slug: ptrStr(r.Attributes.Slug), Name: ptrStr(r.Attributes.Name)} + if r.Attributes.Country != nil { + c.Country = ptrStr(r.Attributes.Country.Name) + } + if c.Slug != "" { + out = append(out, c) + } + } + if resp.Next == nil { + break + } + next, err := resp.Next() + if err != nil { + return nil, objectstorage.HumanizeAPI(err, "regions") + } + if next == nil { + break + } + resp = next + } + sort.Slice(out, func(i, j int) bool { return out[i].Slug < out[j].Slug }) + return out, nil +} + +// configureState is the wizard's accumulated answers. +type configureState struct { + ProjectToken string // as given (ID or slug) + ProjectID string + StorageClass string + Site string + Scope string + Selected []cfgSelectedBucket + Name string + // Endpoint/SigningRegion are known when specific buckets were selected. + Endpoint string + SigningRegion string +} + +// project returns the best project reference known so far. +func (s *configureState) project() string { + return firstNonEmptyStr(s.ProjectToken, s.ProjectID) +} + +// bucketPerms returns bkt_ ID → permission for a limited selection. +func (s *configureState) bucketPerms() map[string]string { + if s.Scope != config.ScopeLimitedAccess { + return nil + } + out := map[string]string{} + for _, sel := range s.Selected { + out[sel.Bucket.ID] = sel.Permission + } + return out +} + +// bucketNamePerms returns display name → permission for messages. +func (s *configureState) bucketNamePerms() map[string]string { + if s.Scope != config.ScopeLimitedAccess { + return nil + } + out := map[string]string{} + for _, sel := range s.Selected { + out[sel.Bucket.Name] = sel.Permission + } + return out +} + +func runConfigure(cmd *cobra.Command, _ []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + opts, err := parseConfigureOptions(cmd) + if err != nil { + return printErr(err) + } + + canPrompt := objectstorage.CanPrompt(cmd) + api := apiClient() + + // Saving needs an active profile; check before creating anything so a + // LATITUDESH_TOKEN-only session does not end up with an unsaved key. + if opts.Save { + if _, _, _, err := objectstorage.ActiveProfile(profileFlag(cmd)); err != nil { + return printErr(exitcode.Errorf(exitcode.Of(err), "%v\n configure saves the key in a profile; run 'lsh login' first, or pass --save=false to print the key instead", err)) + } + } + + st := &configureState{StorageClass: opts.StorageClass, Site: opts.Site} + + // Step 4 first when --bucket was given: the buckets pin the project, + // class and site, so nothing else has to be asked. + if len(opts.Buckets) > 0 { + if err := configureSelectFromSpecs(ctx, cmd, st, opts); err != nil { + return printErr(err) + } + } + + // Step 1: project. + if st.ProjectID == "" { + st.ProjectToken = opts.Project + if st.ProjectToken == "" { + if !canPrompt { + return printErr(configureNeedsPrompt("the project (--project or LSH_PROJECT)")) + } + projects, err := configureListProjects(ctx, api) + if err != nil { + return printErr(err) + } + if len(projects) == 0 { + return printErr(exitcode.Errorf(exitcode.NotFound, "no projects found; create one with 'lsh projects create'")) + } + labels := make([]string, len(projects)) + for i, p := range projects { + labels[i] = p.label() + } + idx, err := objectstorage.Choose("Which project should the key belong to?", labels, 0) + if err != nil { + return printErr(err) + } + if idx < 0 { + return printErr(exitcode.Errorf(exitcode.Refused, "cancelled")) + } + st.ProjectToken, st.ProjectID = projects[idx].ID, projects[idx].ID + } else { + st.ProjectID = cfgResolveProjectID(ctx, api, st.ProjectToken) + } + } + + // Step 2: storage class. + if st.StorageClass == "" { + if !canPrompt { + return printErr(configureNeedsPrompt("the storage class (--storage-class standard|high_performance)")) + } + idx, err := objectstorage.Choose("Which storage class?", []string{ + "standard — general purpose object storage, one key covers every site of the project", + "high_performance — low latency storage on selected sites, one key per site", + }, 0) + if err != nil { + return printErr(err) + } + if idx < 0 { + return printErr(exitcode.Errorf(exitcode.Refused, "cancelled")) + } + st.StorageClass = []string{objectstorage.ClassStandard, objectstorage.ClassHighPerformance}[idx] + } + + // Step 3: site. A high_performance key is bound to one cluster, so the + // site is asked up front (it also filters the bucket menu below). + if st.StorageClass == objectstorage.ClassHighPerformance && st.Site == "" { + if err := configureAskSite(ctx, api, st, canPrompt, "the site of a high_performance key"); err != nil { + return printErr(err) + } + } + + // Step 4: scope. + if st.Scope == "" { + switch { + case opts.AllBuckets: + st.Scope = config.ScopeFullAccess + case !canPrompt: + return printErr(configureNeedsPrompt("the scope (--all-buckets or --bucket [=rw|readonly])")) + default: + where := st.StorageClass + " buckets" + if st.Site != "" { + where += " in " + st.Site + } + idx, err := objectstorage.Choose("Which buckets should the key cover?", []string{ + "all " + where + " of the project, including future ones (fullaccess, recommended)", + "specific buckets (limited_access, rw or readonly)", + }, 0) + if err != nil { + return printErr(err) + } + switch idx { + case 0: + st.Scope = config.ScopeFullAccess + case 1: + if err := configureSelectInteractive(ctx, cmd, st); err != nil { + return printErr(err) + } + default: + return printErr(exitcode.Errorf(exitcode.Refused, "cancelled")) + } + } + } + + // A fullaccess key still needs a site (the API requires one). Without + // --region it is inferred from the project's buckets of that class, the way + // 'access-keys create --all-buckets' does; a standard key covers every site + // so any site of a standard bucket will do. + if st.Scope == config.ScopeFullAccess && st.Site == "" { + site, err := configureInferSite(ctx, cmd, st) + if err != nil { + return printErr(err) + } + if site == "" { + what := fmt.Sprintf("the site of the key (project %s has no %s buckets to infer it from)", st.project(), st.StorageClass) + if err := configureAskSite(ctx, api, st, canPrompt, what); err != nil { + return printErr(err) + } + } else { + st.Site = site + } + } + + // Step 5: name. + st.Name = opts.Name + if st.Name == "" { + st.Name = generateKeyName(st.StorageClass) + if canPrompt && !dryRun() { + answer, err := objectstorage.ReadLine(fmt.Sprintf("Key name [%s]: ", st.Name)) + if err == nil && answer != "" { + st.Name = answer + } + } + } + + plan := configurePlan{ + Name: st.Name, + Project: st.project(), + StorageClass: st.StorageClass, + Site: st.Site, + Scope: st.Scope, + Buckets: st.bucketNamePerms(), + Save: opts.Save, + DryRun: dryRun(), + } + + // Dry-run: describe, never create. + if dryRun() { + if isHuman() { + fmt.Println(configurePlanLine(plan)) + } else { + render([]renderer.ResponseData{plan}) + } + return nil + } + + // Step 6: create (and save). When the name was auto-generated (the user did + // not pass --name), let a collision re-roll a fresh pet name. + created, err := createAccessKeyRetrying(ctx, cmd, accessKeyRequest{ + Project: firstNonEmptyStr(st.ProjectID, st.ProjectToken), + StorageClass: st.StorageClass, + Site: st.Site, + Name: st.Name, + Scope: st.Scope, + Buckets: st.bucketPerms(), + }, opts.Name == "") + if err != nil { + return printErr(objectstorage.HumanizeAPI(err, "access key")) + } + if created.Name != "" { + // The API normalizes names; keep the server's spelling as the index. + st.Name = created.Name + plan.Name = created.Name + } + plan.AccessKeyID = created.AccessKeyID + + if !opts.Save { + return configurePrintSecret(cmd, st, created) + } + + savedAs, profileName, err := saveNewKey(cmd, st.Name, created.stored(st.ProjectID, st.bucketPerms(), config.KeySourceConfigure)) + if err != nil { + // configure creates the key on the user's behalf, so a key it cannot + // store is removed again instead of having its one-time secret + // printed. --show-secret opts into keeping and printing it. + showSecret, _ := cmd.Flags().GetBool(flagCfgShowSecret) + if !showSecret { + req := accessKeyRequest{Project: firstNonEmptyStr(st.ProjectID, st.ProjectToken), StorageClass: st.StorageClass, Site: st.Site, Scope: st.Scope} + if delErr := discardKey(ctx, req, created); delErr == nil { + return printErr(exitcode.Errorf(exitcode.Generic, + "access key %q was created but could not be saved: %v — it was deleted again, so nothing was left behind; retry once the profile is writable", st.Name, err)) + } + objectstorage.Warnf("access key %q could not be saved (%v) nor deleted again, so its secret is shown below", st.Name, err) + return configurePrintSecret(cmd, st, created) + } + objectstorage.Warnf("access key %q was created but could not be saved: %v", st.Name, err) + return configurePrintSecret(cmd, st, created) + } + plan.Profile, plan.SavedAs = profileName, savedAs + path, _ := config.Path() + if path == "" { + path = "~/.config/lsh/config.json" + } + objectstorage.Hintf("Saved to %s (0600) as %q in profile %s. The CLI uses it automatically.", path, savedAs, profileName) + if savedAs != st.Name { + objectstorage.Hintf("(%q already held another key, so this one was saved as %q)", st.Name, savedAs) + } + objectstorage.Hintf("Next: lsh s3 list%s", configureExampleBucket(st, " lsh s3 copy ./file s3://%s/")) + objectstorage.Hintf("For apps or CI create a separate key: lsh s3 access-keys create --bucket --name ") + objectstorage.Hintf("Export it for other tools: lsh s3 configure export --format env%s", configureExampleBucket(st, " s3://%s")) + if !isHuman() { + render([]renderer.ResponseData{plan}) + } + return nil +} + +// configureAskSite fills st.Site from a menu of Latitude sites, or fails with +// exit 2 naming --region when no terminal is available. what describes the +// value being asked for in the error message. +func configureAskSite(ctx context.Context, api *sdk.Latitudesh, st *configureState, canPrompt bool, what string) error { + sites, listErr := configureListSites(ctx, api) + if !canPrompt { + slugs := make([]string, 0, len(sites)) + for _, s := range sites { + slugs = append(slugs, s.Slug) + } + hint := "" + if len(slugs) > 0 { + hint = "; available: " + strings.Join(slugs, ", ") + } + return configureNeedsPrompt(what + " (--region " + hint + ")") + } + if listErr != nil { + return listErr + } + if len(sites) == 0 { + return exitcode.Errorf(exitcode.NotFound, "no sites returned by the API; pass --region ") + } + labels := make([]string, len(sites)) + for i, s := range sites { + labels[i] = s.label() + } + question := "Which site should the key be created in?" + if st.StorageClass == objectstorage.ClassHighPerformance { + question = "Which site is the high_performance cluster on?" + } + idx, err := objectstorage.Choose(question, labels, 0) + if err != nil { + return err + } + if idx < 0 { + return exitcode.Errorf(exitcode.Refused, "cancelled") + } + st.Site = strings.ToUpper(sites[idx].Slug) + return nil +} + +// configureInferSite lists the project's buckets and returns the site for a +// fullaccess key of st.StorageClass, or "" when the project has no bucket of +// that class. Sites come from the raw API because the SDK model drops them. +func configureInferSite(ctx context.Context, cmd *cobra.Command, st *configureState) (string, error) { + r := newResolver(cmd) + r.Project = st.project() + if r.EndpointURL != "" { + return "", objectstorage.ErrUsagef("configure needs the Latitude API; unset --endpoint-url/%s", objectstorage.EnvEndpointURL) + } + list, err := r.ListBuckets(ctx) + if err != nil { + return "", err + } + buckets := make([]*objectstorage.Bucket, 0, len(list)) + for _, d := range list { + buckets = append(buckets, objectstorage.BucketFromData(d)) + } + var sites map[string]string + if len(buckets) > 0 { + if sites, err = objectstorage.RawBucketSitesForProject(ctx, "", r.Project); err != nil { + lsh.LogDebugf("[s3] site lookup failed for project %s: %v", r.Project, err) + } + } + return configureSiteFromBuckets(st.StorageClass, r.Project, buckets, sites) +} + +// configureSiteFromBuckets picks the site for a fullaccess key from the +// project's buckets of the given class (sites supplies bkt_ ID → site for +// buckets whose Site is empty). A standard key covers every site, so the +// first site in sorted order is used; several high_performance sites are +// ambiguous and rejected with exit 2. "" means nothing to infer from. +func configureSiteFromBuckets(class, project string, buckets []*objectstorage.Bucket, sites map[string]string) (string, error) { + distinct := map[string]bool{} + for _, b := range buckets { + if b.StorageClass != class { + continue + } + if s := strings.ToUpper(firstNonEmptyStr(b.Site, sites[b.ID])); s != "" { + distinct[s] = true + } + } + if len(distinct) == 0 { + return "", nil + } + names := make([]string, 0, len(distinct)) + for s := range distinct { + names = append(names, s) + } + sort.Strings(names) + if class == objectstorage.ClassHighPerformance && len(names) > 1 { + return "", objectstorage.ErrUsagef("project %s has high_performance buckets in several sites (%s); pass --region ", project, strings.Join(names, ", ")) + } + return names[0], nil +} + +// configureExampleBucket formats an example with the first selected bucket, +// or "" when the key is fullaccess (no bucket to point at). +func configureExampleBucket(st *configureState, format string) string { + if len(st.Selected) == 0 || st.Selected[0].Bucket.Name == "" { + return "" + } + return fmt.Sprintf(format, st.Selected[0].Bucket.Name) +} + +// configurePlanLine renders the dry-run plan as one human line. +func configurePlanLine(p configurePlan) string { + scope := "all " + p.StorageClass + " buckets" + if p.Site != "" { + scope += " in " + p.Site + } + scope += " (fullaccess)" + if p.Scope == config.ScopeLimitedAccess { + scope = "buckets " + cfgFormatBucketPerms(p.Buckets) + " (limited_access)" + } + action := "save it in the active profile" + if !p.Save { + action = "print the secret once (not saved)" + } + return fmt.Sprintf("(dryrun) create access key %q in project %s for %s and %s", p.Name, p.Project, scope, action) +} + +// configurePrintSecret shows the created key once, in clear (--save=false). +// Structured output embeds the secret only when -o/--json was given on the +// command line (never because of LSH_OUTPUT or the config file). +func configurePrintSecret(cmd *cobra.Command, st *configureState, created *createdAccessKey) error { + res := configureSecretResult{ + Name: st.Name, + AccessKeyID: created.AccessKeyID, + Scope: st.Scope, + StorageClass: st.StorageClass, + Site: st.Site, + Project: st.project(), + Endpoint: st.Endpoint, + SigningRegion: st.SigningRegion, + Buckets: st.bucketNamePerms(), + } + showSecret, _ := cmd.Flags().GetBool(flagCfgShowSecret) + decision := decideSecretDisplay(isHuman(), false, showSecret, outputExplicit(cmd)) + if isHuman() { + fmt.Printf("Access key %q created. The secret is shown once and cannot be retrieved again:\n", res.Name) + fmt.Printf(" Access Key ID: %s\n", created.AccessKeyID) + fmt.Printf(" Secret Access Key: %s\n", created.SecretAccessKey) + if res.Endpoint != "" { + fmt.Printf(" Endpoint: %s Signing region: %s\n", res.Endpoint, res.SigningRegion) + } + if len(res.Buckets) > 0 { + fmt.Printf(" Buckets: %s\n", cfgFormatBucketPerms(res.Buckets)) + } else { + fmt.Printf(" Scope: all %s buckets of the project%s\n", res.StorageClass, cfgSiteSuffix(res.Site)) + } + objectstorage.Hintf("Save it later with: lsh s3 access-keys import --name %s --access-key-id %s", res.Name, created.AccessKeyID) + return nil + } + if decision.Show { + secret := created.SecretAccessKey + res.SecretAccessKey = &secret + } else { + // The key exists on the API and the secret is returned exactly once, so + // it cannot simply be dropped: stdout stays clean (it was not asked for + // explicitly and CI captures it) and the secret goes to stderr. + objectstorage.Warnf("secret_access_key omitted from stdout: structured output came from LSH_OUTPUT/config, not -o; pass -o json or --show-secret to include it. The secret is printed on stderr below — it cannot be retrieved again.") + fmt.Fprintf(os.Stderr, " Access Key ID: %s\n", created.AccessKeyID) + fmt.Fprintf(os.Stderr, " Secret Access Key: %s\n", created.SecretAccessKey) + fmt.Fprintf(os.Stderr, " Save it with: lsh s3 access-keys import --name %s --access-key-id %s\n", res.Name, created.AccessKeyID) + } + render([]renderer.ResponseData{res}) + return nil +} + +func cfgSiteSuffix(site string) string { + if site == "" { + return "" + } + return " in " + site +} + +// configureSelectFromSpecs resolves the --bucket values, validates them and +// fills the state (class, site, project, scope) from the buckets. +func configureSelectFromSpecs(ctx context.Context, cmd *cobra.Command, st *configureState, opts configureOptions) error { + r := newResolver(cmd) + if r.EndpointURL != "" { + return objectstorage.ErrUsagef("configure needs the Latitude API; unset --endpoint-url/%s", objectstorage.EnvEndpointURL) + } + sel := make([]cfgSelectedBucket, 0, len(opts.Buckets)) + for _, spec := range opts.Buckets { + b, err := r.Resolve(ctx, spec.Token) + if err != nil { + return err + } + if err := r.FillSite(ctx, b); err != nil { + lsh.LogDebugf("[s3] could not fetch site of %s: %v", b.Display(), err) + } + sel = append(sel, cfgSelectedBucket{Bucket: b, Permission: spec.Permission}) + } + class, site, projectID, err := cfgValidateSelection(sel) + if err != nil { + return err + } + if opts.StorageClass != "" && opts.StorageClass != class { + return objectstorage.ErrUsagef("--storage-class %s does not match the selected buckets (%s)", opts.StorageClass, class) + } + if st.Site != "" && site != "" && !strings.EqualFold(st.Site, site) { + return objectstorage.ErrUsagef("--region %s does not match the selected buckets (%s)", st.Site, site) + } + if opts.Project != "" && projectID != "" && opts.Project != projectID && !anyProjectSlugMatches(sel, opts.Project) { + return objectstorage.ErrUsagef("--project %s does not match the selected buckets (%s)", opts.Project, projectID) + } + st.StorageClass = class + if site != "" { + st.Site = strings.ToUpper(site) + } + st.ProjectID = projectID + st.ProjectToken = firstNonEmptyStr(opts.Project, sel[0].Bucket.ProjectRef(), projectID) + st.Scope = config.ScopeLimitedAccess + st.Selected = sel + st.Endpoint = sel[0].Bucket.Endpoint + st.SigningRegion = sel[0].Bucket.SigningRegion + if st.StorageClass == objectstorage.ClassHighPerformance && st.Site == "" { + return objectstorage.ErrUsagef("could not determine the site of the selected high_performance buckets; pass --region ") + } + return nil +} + +func anyProjectSlugMatches(sel []cfgSelectedBucket, token string) bool { + for _, s := range sel { + if s.Bucket.ProjectSlug == token || s.Bucket.ProjectName == token { + return true + } + } + return false +} + +// configureSelectInteractive lists the buckets of the chosen class (and site) +// in the project and lets the user pick some by number, then a permission. +func configureSelectInteractive(ctx context.Context, cmd *cobra.Command, st *configureState) error { + r := newResolver(cmd) + r.Project = st.project() + if r.EndpointURL != "" { + return objectstorage.ErrUsagef("configure needs the Latitude API; unset --endpoint-url/%s", objectstorage.EnvEndpointURL) + } + data, err := r.ListBuckets(ctx) + if err != nil { + return err + } + var sites map[string]string + if st.StorageClass == objectstorage.ClassHighPerformance { + sites, err = objectstorage.RawBucketSitesForProject(ctx, "", r.Project) + if err != nil { + lsh.LogDebugf("[s3] could not fetch bucket sites: %v", err) + sites = nil + } + } + var candidates []*objectstorage.Bucket + for _, d := range data { + b := objectstorage.BucketFromData(d) + if b.StorageClass != st.StorageClass { + continue + } + if b.Site == "" { + b.Site = sites[b.ID] + } + if st.StorageClass == objectstorage.ClassHighPerformance && st.Site != "" && b.Site != "" && !strings.EqualFold(b.Site, st.Site) { + continue + } + candidates = append(candidates, b) + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].Name < candidates[j].Name }) + if len(candidates) == 0 { + where := st.StorageClass + if st.Site != "" { + where += " in " + st.Site + } + return exitcode.Errorf(exitcode.NotFound, "no %s buckets found in project %s; create one with 'lsh s3 create-bucket s3:// --region --project %s'", where, r.Project, r.Project) + } + labels := make([]string, len(candidates)) + for i, b := range candidates { + labels[i] = fmt.Sprintf("%s (%s, %s)", b.Name, b.ID, joinNonEmpty(" ", b.StorageClass, firstNonEmptyStr(b.Site, b.City))) + } + idx, err := objectstorage.ChooseMany("Which buckets should the key cover?", labels) + if err != nil { + return err + } + if len(idx) == 0 { + return exitcode.Errorf(exitcode.Refused, "no bucket selected; pick at least one with space, or pass --bucket ") + } + permIdx, err := objectstorage.Choose("Which permission on these buckets?", []string{ + "rw — read and write", + "readonly — list and download only", + }, 0) + if err != nil { + return err + } + if permIdx < 0 { + return exitcode.Errorf(exitcode.Refused, "cancelled") + } + perm := []string{config.PermissionRW, config.PermissionReadOnly}[permIdx] + sel := make([]cfgSelectedBucket, 0, len(idx)) + for _, i := range idx { + sel = append(sel, cfgSelectedBucket{Bucket: candidates[i], Permission: perm}) + } + class, site, projectID, err := cfgValidateSelection(sel) + if err != nil { + return err + } + st.StorageClass = class + if site != "" { + st.Site = strings.ToUpper(site) + } + if projectID != "" { + st.ProjectID = projectID + } + st.Scope = config.ScopeLimitedAccess + st.Selected = sel + st.Endpoint = sel[0].Bucket.Endpoint + st.SigningRegion = sel[0].Bucket.SigningRegion + return nil +} diff --git a/cmd/s3/configure_export.go b/cmd/s3/configure_export.go new file mode 100644 index 0000000..e0d701f --- /dev/null +++ b/cmd/s3/configure_export.go @@ -0,0 +1,375 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + cobra "github.com/spf13/cobra" +) + +// Flag names of `lsh s3 configure export`. +const ( + flagExportFormat = "format" + flagExportProfileName = "profile-name" + flagExportNoSecret = "no-secret" +) + +// exportFormats lists the supported --format values in help order. +var exportFormats = []string{"env", "aws", "rclone", "mc", "s3cmd", "process"} + +// exportSecretPlaceholder replaces the secret with --no-secret. +const exportSecretPlaceholder = "" + +// newConfigureExportCmd builds `lsh s3 configure export`. +func newConfigureExportCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "export [s3://bucket]", + Short: "Print a saved access key in the format of another S3 tool", + Long: `Print the credential and connection settings for a bucket in the format of +another tool: shell variables (env), an aws CLI profile (aws), an rclone +remote (rclone), a MinIO client alias (mc), an s3cmd config (s3cmd) or the +JSON of an aws credential_process (process). + +With a bucket argument the endpoint, signing region and backend bucket name +come from the Latitude API and the credential is selected like every other +command (LSH_S3_* environment, --access-key, or the least-privileged saved key +covering the bucket). Without a bucket, --access-key names the saved key and +--endpoint-url supplies the endpoint. + +The output goes to stdout and contains the secret unless --no-secret is +given; -o/--output is ignored because the template is the output. The aws +format prints two blocks that belong to two different files (~/.aws/config and +~/.aws/credentials), each labelled with a comment, so it cannot be appended to +a single file in one redirect.`, + Example: ` eval "$(lsh s3 configure export s3://backups --format env)" + lsh s3 configure export s3://backups --format aws # two blocks: one per file, see the comments + lsh s3 configure export s3://backups --format rclone --profile-name latitude >> ~/.config/rclone/rclone.conf + lsh s3 configure export s3://backups --format mc | sh + lsh s3 configure export --access-key ci-deploy --endpoint-url https://s3.us-central-1.storage.sh --format process + lsh s3 configure export s3://backups --format s3cmd --no-secret`, + Args: cobra.MaximumNArgs(1), + RunE: runConfigureExport, + }) + addProjectFlag(cmd, true, "project ID or slug to disambiguate the bucket name") + cmd.Flags().String(flagExportFormat, "", "output format: "+strings.Join(exportFormats, " | ")) + cmd.Flags().String(flagExportProfileName, "", "profile/remote/alias name in the output (default: lsh- or lsh-)") + cmd.Flags().Bool(flagExportNoSecret, false, "replace the secret with a "+exportSecretPlaceholder+" placeholder") + return cmd +} + +// exportTarget is everything a template needs. +type exportTarget struct { + ProfileName string + Endpoint string // scheme://host[:port] + Host string // host[:port] without scheme + Secure bool + SigningRegion string + BucketName string // backend bucket name; empty when unknown + StorageClass string // standard | high_performance | "" + AccessKeyID string + Secret string +} + +// newExportTarget derives Host/Secure from the endpoint. +func newExportTarget(profileName, endpoint, region, bucketName, class, accessKeyID, secret string) (exportTarget, error) { + host, secure, err := objectstorage.EndpointHost(endpoint) + if err != nil { + return exportTarget{}, exitcode.New(exitcode.Usage, err) + } + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + if region == "" { + region = objectstorage.SigningRegion(endpoint) + } + return exportTarget{ + ProfileName: profileName, + Endpoint: strings.TrimRight(endpoint, "/"), + Host: host, + Secure: secure, + SigningRegion: region, + BucketName: bucketName, + StorageClass: class, + AccessKeyID: accessKeyID, + Secret: secret, + }, nil +} + +// bucketOrPlaceholder returns the backend bucket name or "" for the +// example commands when no bucket is known. +func (t exportTarget) bucketOrPlaceholder() string { + if t.BucketName == "" { + return "" + } + return t.BucketName +} + +// header is the comment every text template starts with. +func (t exportTarget) header() string { + if t.BucketName == "" { + return "" + } + return "# bucket name on the endpoint: " + t.BucketName + "\n" +} + +// rcloneProvider maps the storage class to rclone's provider setting: the +// standard tier is Wasabi-backed, high_performance (VAST) is generic S3. +func (t exportTarget) rcloneProvider() string { + switch t.StorageClass { + case objectstorage.ClassStandard: + return "Wasabi" + case objectstorage.ClassHighPerformance: + return "Other" + } + // Infer from the endpoint shape when the class is unknown. + if strings.HasPrefix(strings.ToLower(t.Host), "s3.") { + return "Wasabi" + } + return "Other" +} + +// renderExport dispatches to the template for format. +func renderExport(format string, t exportTarget) (string, error) { + switch strings.ToLower(strings.TrimSpace(format)) { + case "env": + return exportEnv(t), nil + case "aws": + return exportAWS(t), nil + case "rclone": + return exportRclone(t), nil + case "mc": + return exportMc(t), nil + case "s3cmd": + return exportS3cmd(t), nil + case "process": + return exportProcess(t), nil + } + return "", objectstorage.ErrUsagef("--format must be one of %s (got %q)", strings.Join(exportFormats, ", "), format) +} + +// cfgShellValue quotes v for a POSIX shell when it contains characters outside +// the safe set (aws export-credentials prints plain values; secrets are +// base64-like so they normally pass through untouched). +func cfgShellValue(v string) string { + safe := true + for _, r := range v { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case strings.ContainsRune("_./:+=@-", r): + default: + safe = false + } + } + if safe && v != "" { + return v + } + return "'" + strings.ReplaceAll(v, "'", `'\''`) + "'" +} + +// exportEnv renders shell exports for the aws CLI/SDKs and for lsh itself. +func exportEnv(t exportTarget) string { + var b strings.Builder + b.WriteString(t.header()) + fmt.Fprintf(&b, "export AWS_ACCESS_KEY_ID=%s\n", cfgShellValue(t.AccessKeyID)) + fmt.Fprintf(&b, "export AWS_SECRET_ACCESS_KEY=%s\n", cfgShellValue(t.Secret)) + fmt.Fprintf(&b, "export AWS_ENDPOINT_URL_S3=%s\n", cfgShellValue(t.Endpoint)) + fmt.Fprintf(&b, "export AWS_REGION=%s\n", cfgShellValue(t.SigningRegion)) + b.WriteString("export AWS_REQUEST_CHECKSUM_CALCULATION=when_required\n") + b.WriteString("export AWS_RESPONSE_CHECKSUM_VALIDATION=when_required\n") + fmt.Fprintf(&b, "export %s=%s\n", objectstorage.EnvAccessKeyID, cfgShellValue(t.AccessKeyID)) + fmt.Fprintf(&b, "export %s=%s\n", objectstorage.EnvSecretAccessKey, cfgShellValue(t.Secret)) + fmt.Fprintf(&b, "export %s=%s\n", objectstorage.EnvEndpointURL, cfgShellValue(t.Endpoint)) + fmt.Fprintf(&b, "export %s=%s\n", objectstorage.EnvSigningRegion, cfgShellValue(t.SigningRegion)) + fmt.Fprintf(&b, "# aws s3 ls s3://%s/\n", t.bucketOrPlaceholder()) + return b.String() +} + +// exportAWS renders an aws CLI profile: a config block and a credentials +// block, each commented with the file it belongs to. +func exportAWS(t exportTarget) string { + var b strings.Builder + b.WriteString(t.header()) + b.WriteString("# ~/.aws/config\n") + fmt.Fprintf(&b, "[profile %s]\n", t.ProfileName) + fmt.Fprintf(&b, "region = %s\n", t.SigningRegion) + fmt.Fprintf(&b, "endpoint_url = %s\n", t.Endpoint) + b.WriteString("s3 =\n") + b.WriteString(" addressing_style = path\n") + b.WriteString("request_checksum_calculation = when_required\n") + b.WriteString("response_checksum_validation = when_required\n") + b.WriteString("\n") + b.WriteString("# ~/.aws/credentials\n") + fmt.Fprintf(&b, "[%s]\n", t.ProfileName) + fmt.Fprintf(&b, "aws_access_key_id = %s\n", t.AccessKeyID) + fmt.Fprintf(&b, "aws_secret_access_key = %s\n", t.Secret) + b.WriteString("\n") + fmt.Fprintf(&b, "# aws --profile %s s3 ls s3://%s/\n", t.ProfileName, t.bucketOrPlaceholder()) + return b.String() +} + +// exportRclone renders an rclone remote section. +func exportRclone(t exportTarget) string { + var b strings.Builder + b.WriteString(t.header()) + fmt.Fprintf(&b, "[%s]\n", t.ProfileName) + b.WriteString("type = s3\n") + fmt.Fprintf(&b, "provider = %s\n", t.rcloneProvider()) + b.WriteString("env_auth = false\n") + fmt.Fprintf(&b, "access_key_id = %s\n", t.AccessKeyID) + fmt.Fprintf(&b, "secret_access_key = %s\n", t.Secret) + fmt.Fprintf(&b, "endpoint = %s\n", t.Host) + fmt.Fprintf(&b, "region = %s\n", t.SigningRegion) + b.WriteString("force_path_style = true\n") + b.WriteString("\n") + fmt.Fprintf(&b, "# rclone ls %s:%s\n", t.ProfileName, t.bucketOrPlaceholder()) + return b.String() +} + +// exportMc renders the MinIO client alias command. +func exportMc(t exportTarget) string { + var b strings.Builder + b.WriteString(t.header()) + fmt.Fprintf(&b, "mc alias set %s %s %s %s --api S3v4 --path on\n", cfgShellValue(t.ProfileName), cfgShellValue(t.Endpoint), cfgShellValue(t.AccessKeyID), cfgShellValue(t.Secret)) + fmt.Fprintf(&b, "# mc ls %s/%s\n", t.ProfileName, t.bucketOrPlaceholder()) + return b.String() +} + +// exportS3cmd renders an s3cmd configuration. +func exportS3cmd(t exportTarget) string { + useHTTPS := "True" + if !t.Secure { + useHTTPS = "False" + } + var b strings.Builder + b.WriteString(t.header()) + b.WriteString("[default]\n") + fmt.Fprintf(&b, "access_key = %s\n", t.AccessKeyID) + fmt.Fprintf(&b, "secret_key = %s\n", t.Secret) + fmt.Fprintf(&b, "host_base = %s\n", t.Host) + fmt.Fprintf(&b, "host_bucket = %s\n", t.Host) + fmt.Fprintf(&b, "bucket_location = %s\n", t.SigningRegion) + fmt.Fprintf(&b, "use_https = %s\n", useHTTPS) + b.WriteString("signature_v2 = False\n") + b.WriteString("\n") + fmt.Fprintf(&b, "# s3cmd ls s3://%s\n", t.bucketOrPlaceholder()) + return b.String() +} + +// exportProcess renders the JSON an aws credential_process must print. +func exportProcess(t exportTarget) string { + payload := struct { + Version int `json:"Version"` + AccessKeyID string `json:"AccessKeyId"` + SecretAccessKey string `json:"SecretAccessKey"` + }{Version: 1, AccessKeyID: t.AccessKeyID, SecretAccessKey: t.Secret} + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) // keep the placeholder readable + if err := enc.Encode(payload); err != nil { + return "" + } + return b.String() // Encode appends the trailing newline +} + +// exportProfileName picks the default profile/remote name. +func exportProfileName(explicit string, b *objectstorage.Bucket, cred objectstorage.Credential) string { + if explicit = strings.TrimSpace(explicit); explicit != "" { + return explicit + } + if b != nil && b.Name != "" { + return "lsh-" + b.Name + } + if cred.Name != "" { + return "lsh-" + cred.Name + } + if cred.AccessKeyID != "" { + return "lsh-" + strings.ToLower(cred.AccessKeyID) + } + return "lsh" +} + +func runConfigureExport(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + format, _ := cmd.Flags().GetString(flagExportFormat) + if strings.TrimSpace(format) == "" { + return printErr(objectstorage.ErrUsagef("--format is required: one of %s", strings.Join(exportFormats, ", "))) + } + if _, err := renderExport(format, exportTarget{}); err != nil { + return printErr(err) + } + profileName, _ := cmd.Flags().GetString(flagExportProfileName) + noSecret, _ := cmd.Flags().GetBool(flagExportNoSecret) + accessKey, _ := cmd.Flags().GetString(flagAccessKey) + + var ( + bucket *objectstorage.Bucket + cred objectstorage.Credential + ) + if len(args) == 1 { + ref, err := objectstorage.ParseBucketOnly(args[0]) + if err != nil { + return printErr(err) + } + bucket, err = resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + if err := bucket.Validate(); err != nil { + return printErr(err) + } + cred, err = objectstorage.ResolveCredential(bucket, objectstorage.CredentialOptions{ + AccessKeyName: accessKey, + ProfileOverride: profileFlag(cmd), + }) + if err != nil { + return printErr(err) + } + } else { + endpoint := endpointOverride(cmd) + if endpoint == "" { + return printErr(objectstorage.ErrUsagef("pass s3://, or --access-key together with --endpoint-url (or %s)", objectstorage.EnvEndpointURL)) + } + var err error + cred, err = objectstorage.ResolveCredential(nil, objectstorage.CredentialOptions{ + AccessKeyName: accessKey, + ProfileOverride: profileFlag(cmd), + }) + if err != nil { + return printErr(err) + } + bucket = &objectstorage.Bucket{ + Endpoint: endpoint, + SigningRegion: signingRegionOverride(cmd), + StorageClass: cred.Key.StorageClass, + EndpointOverride: true, + } + } + + secret := cred.Secret() + if noSecret { + secret = exportSecretPlaceholder + } + target, err := newExportTarget( + exportProfileName(profileName, bucket, cred), + bucket.Endpoint, bucket.SigningRegion, bucket.BucketName, bucket.StorageClass, + cred.AccessKeyID, secret, + ) + if err != nil { + return printErr(err) + } + out, err := renderExport(format, target) + if err != nil { + return printErr(err) + } + fmt.Print(out) + if !noSecret { + objectstorage.Warnf("output contains a secret; do not commit it") + } + return nil +} diff --git a/cmd/s3/configure_export_test.go b/cmd/s3/configure_export_test.go new file mode 100644 index 0000000..10cf2a2 --- /dev/null +++ b/cmd/s3/configure_export_test.go @@ -0,0 +1,255 @@ +package s3 + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +// fixedExportTarget is the bucket/credential every golden test renders. +func fixedExportTarget(t *testing.T) exportTarget { + t.Helper() + target, err := newExportTarget("lsh-backups", "https://s3.us-central-1.storage.sh", "", "backups-7f3a", + objectstorage.ClassStandard, "XL68DDURVGUUOULWPCAE", "sEcReT/KeY+123") + if err != nil { + t.Fatalf("newExportTarget: %v", err) + } + return target +} + +func TestNewExportTargetDerivesHostAndRegion(t *testing.T) { + tg := fixedExportTarget(t) + if tg.Host != "s3.us-central-1.storage.sh" || !tg.Secure { + t.Fatalf("host/secure: %q %v", tg.Host, tg.Secure) + } + if tg.SigningRegion != "us-central-1" { + t.Fatalf("signing region should come from the endpoint, got %q", tg.SigningRegion) + } + plain, err := newExportTarget("p", "http://127.0.0.1:9000/", "us-east-1", "b", "", "AK", "SK") + if err != nil { + t.Fatal(err) + } + if plain.Secure || plain.Host != "127.0.0.1:9000" || plain.Endpoint != "http://127.0.0.1:9000" { + t.Fatalf("plain http target: %+v", plain) + } + if _, err := newExportTarget("p", "ftp://x", "", "", "", "", ""); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("bad scheme must be a usage error, got %v", err) + } +} + +func TestExportEnvGolden(t *testing.T) { + want := strings.Join([]string{ + "# bucket name on the endpoint: backups-7f3a", + "export AWS_ACCESS_KEY_ID=XL68DDURVGUUOULWPCAE", + "export AWS_SECRET_ACCESS_KEY=sEcReT/KeY+123", + "export AWS_ENDPOINT_URL_S3=https://s3.us-central-1.storage.sh", + "export AWS_REGION=us-central-1", + "export AWS_REQUEST_CHECKSUM_CALCULATION=when_required", + "export AWS_RESPONSE_CHECKSUM_VALIDATION=when_required", + "export LSH_S3_ACCESS_KEY_ID=XL68DDURVGUUOULWPCAE", + "export LSH_S3_SECRET_ACCESS_KEY=sEcReT/KeY+123", + "export LSH_S3_ENDPOINT_URL=https://s3.us-central-1.storage.sh", + "export LSH_S3_SIGNING_REGION=us-central-1", + "# aws s3 ls s3://backups-7f3a/", + "", + }, "\n") + if got := exportEnv(fixedExportTarget(t)); got != want { + t.Fatalf("env template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestExportAWSGolden(t *testing.T) { + want := strings.Join([]string{ + "# bucket name on the endpoint: backups-7f3a", + "# ~/.aws/config", + "[profile lsh-backups]", + "region = us-central-1", + "endpoint_url = https://s3.us-central-1.storage.sh", + "s3 =", + " addressing_style = path", + "request_checksum_calculation = when_required", + "response_checksum_validation = when_required", + "", + "# ~/.aws/credentials", + "[lsh-backups]", + "aws_access_key_id = XL68DDURVGUUOULWPCAE", + "aws_secret_access_key = sEcReT/KeY+123", + "", + "# aws --profile lsh-backups s3 ls s3://backups-7f3a/", + "", + }, "\n") + if got := exportAWS(fixedExportTarget(t)); got != want { + t.Fatalf("aws template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestExportRcloneGolden(t *testing.T) { + want := strings.Join([]string{ + "# bucket name on the endpoint: backups-7f3a", + "[lsh-backups]", + "type = s3", + "provider = Wasabi", + "env_auth = false", + "access_key_id = XL68DDURVGUUOULWPCAE", + "secret_access_key = sEcReT/KeY+123", + "endpoint = s3.us-central-1.storage.sh", + "region = us-central-1", + "force_path_style = true", + "", + "# rclone ls lsh-backups:backups-7f3a", + "", + }, "\n") + if got := exportRclone(fixedExportTarget(t)); got != want { + t.Fatalf("rclone template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + + hp := fixedExportTarget(t) + hp.StorageClass = objectstorage.ClassHighPerformance + if !strings.Contains(exportRclone(hp), "provider = Other\n") { + t.Fatal("high_performance must map to provider = Other") + } + unknown := fixedExportTarget(t) + unknown.StorageClass = "" + unknown.Host = "objects.tyo4.storage.sh" + if !strings.Contains(exportRclone(unknown), "provider = Other\n") { + t.Fatal("objects.* hosts without a class must map to provider = Other") + } +} + +func TestExportMcGolden(t *testing.T) { + want := "# bucket name on the endpoint: backups-7f3a\n" + + "mc alias set lsh-backups https://s3.us-central-1.storage.sh XL68DDURVGUUOULWPCAE sEcReT/KeY+123 --api S3v4 --path on\n" + + "# mc ls lsh-backups/backups-7f3a\n" + if got := exportMc(fixedExportTarget(t)); got != want { + t.Fatalf("mc template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestExportS3cmdGolden(t *testing.T) { + want := strings.Join([]string{ + "# bucket name on the endpoint: backups-7f3a", + "[default]", + "access_key = XL68DDURVGUUOULWPCAE", + "secret_key = sEcReT/KeY+123", + "host_base = s3.us-central-1.storage.sh", + "host_bucket = s3.us-central-1.storage.sh", + "bucket_location = us-central-1", + "use_https = True", + "signature_v2 = False", + "", + "# s3cmd ls s3://backups-7f3a", + "", + }, "\n") + if got := exportS3cmd(fixedExportTarget(t)); got != want { + t.Fatalf("s3cmd template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + plain := fixedExportTarget(t) + plain.Secure = false + if !strings.Contains(exportS3cmd(plain), "use_https = False\n") { + t.Fatal("http endpoints must render use_https = False") + } +} + +func TestExportProcessGolden(t *testing.T) { + want := `{"Version":1,"AccessKeyId":"XL68DDURVGUUOULWPCAE","SecretAccessKey":"sEcReT/KeY+123"}` + "\n" + got := exportProcess(fixedExportTarget(t)) + if got != want { + t.Fatalf("process template mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + var decoded map[string]interface{} + if err := json.Unmarshal([]byte(got), &decoded); err != nil { + t.Fatalf("process output is not valid JSON: %v", err) + } + if decoded["Version"].(float64) != 1 { + t.Fatalf("Version must be 1: %v", decoded) + } +} + +func TestRenderExportDispatchAndNoSecret(t *testing.T) { + base := fixedExportTarget(t) + base.Secret = exportSecretPlaceholder + for _, format := range exportFormats { + out, err := renderExport(format, base) + if err != nil { + t.Fatalf("%s: %v", format, err) + } + if out == "" || !strings.HasSuffix(out, "\n") { + t.Fatalf("%s: output must be non-empty and newline-terminated: %q", format, out) + } + if strings.Contains(out, "sEcReT") { + t.Fatalf("%s: real secret leaked with --no-secret: %s", format, out) + } + if !strings.Contains(out, exportSecretPlaceholder) { + t.Fatalf("%s: placeholder missing: %s", format, out) + } + // Nothing time-bound: the templates never mention expiry. + if strings.Contains(strings.ToLower(out), "expire") { + t.Fatalf("%s: templates must not carry an expiry: %s", format, out) + } + } + if _, err := renderExport("toml", base); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("unknown format must be a usage error, got %v", err) + } + // Case-insensitive format names. + if _, err := renderExport("ENV", base); err != nil { + t.Fatalf("format names must be case-insensitive: %v", err) + } +} + +func TestExportWithoutBucketUsesPlaceholders(t *testing.T) { + tg, err := newExportTarget("lsh-ci", "https://objects.tyo4.storage.sh", "", "", "", "AK", "SK") + if err != nil { + t.Fatal(err) + } + env := exportEnv(tg) + if strings.Contains(env, "# bucket name on the endpoint") { + t.Fatalf("no bucket header expected when the bucket is unknown:\n%s", env) + } + if !strings.HasSuffix(env, "# aws s3 ls s3:///\n") { + t.Fatalf("example must use the placeholder:\n%s", env) + } + if tg.SigningRegion != "tyo4" { + t.Fatalf("signing region derived from objects. host: %q", tg.SigningRegion) + } +} + +func TestShellValueQuotesUnsafeCharacters(t *testing.T) { + cases := map[string]string{ + "abc123": "abc123", + "a/b+c=d": "a/b+c=d", + "has space": "'has space'", + "it's": `'it'\''s'`, + "": "''", + "dollar$sign": "'dollar$sign'", + } + for in, want := range cases { + if got := cfgShellValue(in); got != want { + t.Errorf("cfgShellValue(%q) = %q, want %q", in, got, want) + } + } +} + +func TestExportProfileNameDefaults(t *testing.T) { + b := &objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a"} + saved := objectstorage.NewCredential("AK", "SK", "saved") + saved.Name = "ci-deploy" + if got := exportProfileName(" custom ", b, saved); got != "custom" { + t.Fatalf("explicit name wins: %q", got) + } + if got := exportProfileName("", b, saved); got != "lsh-backups" { + t.Fatalf("bucket display name: %q", got) + } + if got := exportProfileName("", nil, saved); got != "lsh-ci-deploy" { + t.Fatalf("saved key name: %q", got) + } + env := objectstorage.NewCredential("AKIAENV", "SK", "env") + if got := exportProfileName("", nil, env); got != "lsh-akiaenv" { + t.Fatalf("env credential falls back to the key id: %q", got) + } + if got := exportProfileName("", nil, objectstorage.Credential{}); got != "lsh" { + t.Fatalf("last resort: %q", got) + } +} diff --git a/cmd/s3/configure_test.go b/cmd/s3/configure_test.go new file mode 100644 index 0000000..990491c --- /dev/null +++ b/cmd/s3/configure_test.go @@ -0,0 +1,254 @@ +package s3 + +import ( + "regexp" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +// TestConfigureDefaultNameIsPetName covers that configure uses the same +// pet-name generator as the dashboard for the default key name. +func TestConfigureDefaultNameIsPetName(t *testing.T) { + re := regexp.MustCompile(`^key-[a-z]+-[a-z]+-(std|hp)$`) + if got := generateKeyName(objectstorage.ClassHighPerformance); !re.MatchString(got) { + t.Errorf("generateKeyName(hp) = %q", got) + } +} + +// TestConfigureBucketSpecsUseSharedParser checks the --bucket spellings the +// configure help advertises through the parser shared with access-keys +// create (configure used to have its own copy). +func TestConfigureBucketSpecsUseSharedParser(t *testing.T) { + cases := map[string]bucketSpec{ + "backups": {Token: "backups", Permission: config.PermissionRW}, + "backups=rw": {Token: "backups", Permission: config.PermissionRW}, + "logs=readonly": {Token: "logs", Permission: config.PermissionReadOnly}, + " logs = RO ": {Token: "logs", Permission: config.PermissionReadOnly}, + "s3://bkt_1Gjb=rw": {Token: "bkt_1Gjb", Permission: config.PermissionRW}, + "s3://backups/": {Token: "backups", Permission: config.PermissionRW}, + "backups-7f3a=read": {Token: "backups-7f3a", Permission: config.PermissionReadOnly}, + "backups=read-write": {Token: "backups", Permission: config.PermissionRW}, + "bkt_9aQ2xL0mPq7Rt=ro": {Token: "bkt_9aQ2xL0mPq7Rt", Permission: config.PermissionReadOnly}, + } + for in, want := range cases { + specs, err := parseBucketSpecs([]string{in}) + if err != nil { + t.Errorf("parseBucketSpecs(%q): %v", in, err) + continue + } + if len(specs) != 1 || specs[0] != want { + t.Errorf("parseBucketSpecs(%q) = %+v, want %+v", in, specs, want) + } + } + for _, bad := range []string{"", "=rw", "backups=admin", "s3://"} { + if _, err := parseBucketSpec(bad); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("parseBucketSpec(%q) must fail with exit %d, got %v", bad, exitcode.Usage, err) + } + } + if _, err := parseBucketSpecs([]string{"backups", "backups=readonly"}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("duplicate bucket must be a usage error, got %v", err) + } + if specs, err := parseBucketSpecs(nil); err != nil || len(specs) != 0 { + t.Fatalf("empty input: %v %v", specs, err) + } +} + +func TestCfgValidateSelection(t *testing.T) { + std := func(id, name, project string) *objectstorage.Bucket { + return &objectstorage.Bucket{ID: id, Name: name, BucketName: name + "-1", StorageClass: objectstorage.ClassStandard, ProjectID: project, Site: "DAL"} + } + hp := func(id, name, site, project string) *objectstorage.Bucket { + return &objectstorage.Bucket{ID: id, Name: name, BucketName: name + "-1", StorageClass: objectstorage.ClassHighPerformance, ProjectID: project, Site: site} + } + + class, site, project, err := cfgValidateSelection([]cfgSelectedBucket{ + {Bucket: std("bkt_1", "backups", "proj_1"), Permission: config.PermissionRW}, + {Bucket: std("bkt_2", "logs", "proj_1"), Permission: config.PermissionReadOnly}, + }) + if err != nil || class != objectstorage.ClassStandard || site != "DAL" || project != "proj_1" { + t.Fatalf("same group must pass: %q %q %q %v", class, site, project, err) + } + + // Standard buckets on different sites are fine (one key covers every site). + b2 := std("bkt_2", "logs", "proj_1") + b2.Site = "NYC" + if _, _, _, err := cfgValidateSelection([]cfgSelectedBucket{{Bucket: std("bkt_1", "backups", "proj_1")}, {Bucket: b2}}); err != nil { + t.Fatalf("standard buckets may span sites: %v", err) + } + + // Empty site on one side is treated as unknown, not as a conflict. + unknownSite := hp("bkt_3", "fast", "", "proj_1") + _, site, _, err = cfgValidateSelection([]cfgSelectedBucket{{Bucket: hp("bkt_4", "faster", "TYO4", "proj_1")}, {Bucket: unknownSite}}) + if err != nil || site != "TYO4" { + t.Fatalf("unknown site must not conflict and the known one wins: %q %v", site, err) + } + + failures := []struct { + name string + sel []cfgSelectedBucket + want string + }{ + {"class", []cfgSelectedBucket{{Bucket: std("bkt_1", "backups", "proj_1")}, {Bucket: hp("bkt_3", "fast", "DAL", "proj_1")}}, "storage classes"}, + {"site", []cfgSelectedBucket{{Bucket: hp("bkt_3", "fast", "TYO4", "proj_1")}, {Bucket: hp("bkt_4", "faster", "DAL", "proj_1")}}, "different sites"}, + {"project", []cfgSelectedBucket{{Bucket: std("bkt_1", "backups", "proj_1")}, {Bucket: std("bkt_2", "logs", "proj_2")}}, "different projects"}, + {"empty", nil, "no bucket"}, + } + for _, f := range failures { + _, _, _, err := cfgValidateSelection(f.sel) + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), f.want) { + t.Errorf("%s: want usage error containing %q, got %v", f.name, f.want, err) + } + } +} + +// TestConfigureSiteFromBuckets covers the site inference of a fullaccess key +// (F16): a standard key is never created without a site; the site comes from +// the project's buckets of the key's class, the raw site map filling gaps. +func TestConfigureSiteFromBuckets(t *testing.T) { + std := func(id, site string) *objectstorage.Bucket { + return &objectstorage.Bucket{ID: id, Name: id, StorageClass: objectstorage.ClassStandard, Site: site} + } + hp := func(id, site string) *objectstorage.Bucket { + return &objectstorage.Bucket{ID: id, Name: id, StorageClass: objectstorage.ClassHighPerformance, Site: site} + } + + // Standard: any site of a standard bucket is valid; the choice is + // deterministic (sorted) and other classes are ignored. + site, err := configureSiteFromBuckets(objectstorage.ClassStandard, "proj_1", + []*objectstorage.Bucket{std("bkt_1", "nyc"), std("bkt_2", "DAL"), hp("bkt_3", "TYO4")}, nil) + if err != nil || site != "DAL" { + t.Fatalf("standard site: %q %v", site, err) + } + // The raw site map fills buckets whose SDK model lost the site. + site, err = configureSiteFromBuckets(objectstorage.ClassStandard, "proj_1", + []*objectstorage.Bucket{std("bkt_1", "")}, map[string]string{"bkt_1": "sao"}) + if err != nil || site != "SAO" { + t.Fatalf("site from raw map: %q %v", site, err) + } + // Nothing of the class → "" (the caller asks or fails with exit 2). + site, err = configureSiteFromBuckets(objectstorage.ClassStandard, "proj_1", []*objectstorage.Bucket{hp("bkt_3", "TYO4")}, nil) + if err != nil || site != "" { + t.Fatalf("no standard buckets must yield an empty site: %q %v", site, err) + } + site, err = configureSiteFromBuckets(objectstorage.ClassStandard, "proj_1", nil, nil) + if err != nil || site != "" { + t.Fatalf("no buckets at all: %q %v", site, err) + } + // high_performance: one site is fine, several are ambiguous. + site, err = configureSiteFromBuckets(objectstorage.ClassHighPerformance, "proj_1", []*objectstorage.Bucket{hp("bkt_3", "TYO4"), hp("bkt_4", "tyo4")}, nil) + if err != nil || site != "TYO4" { + t.Fatalf("single hp site: %q %v", site, err) + } + _, err = configureSiteFromBuckets(objectstorage.ClassHighPerformance, "proj_1", []*objectstorage.Bucket{hp("bkt_3", "TYO4"), hp("bkt_4", "DAL")}, nil) + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "DAL, TYO4") || !strings.Contains(err.Error(), "--region") { + t.Fatalf("several hp sites must be a usage error naming them and --region, got %v", err) + } +} + +func TestConfigurePlanLineAndBuckets(t *testing.T) { + full := configurePlan{Name: "lsh-lanusse-tyo4", Project: "my-project", StorageClass: objectstorage.ClassHighPerformance, Site: "TYO4", Scope: config.ScopeFullAccess, Save: true} + want := `(dryrun) create access key "lsh-lanusse-tyo4" in project my-project for all high_performance buckets in TYO4 (fullaccess) and save it in the active profile` + if got := configurePlanLine(full); got != want { + t.Fatalf("fullaccess plan line:\n got %s\nwant %s", got, want) + } + // A standard fullaccess key keeps its site too (it is no longer cleared). + std := configurePlan{Name: "lsh-lanusse-standard", Project: "my-project", StorageClass: objectstorage.ClassStandard, Site: "DAL", Scope: config.ScopeFullAccess, Save: true} + if got := configurePlanLine(std); !strings.Contains(got, "all standard buckets in DAL (fullaccess)") { + t.Fatalf("standard plan line must name the site: %s", got) + } + limited := configurePlan{Name: "ci", Project: "proj_1", StorageClass: objectstorage.ClassStandard, Scope: config.ScopeLimitedAccess, + Buckets: map[string]string{"logs": config.PermissionReadOnly, "backups": config.PermissionRW}, Save: false} + want = `(dryrun) create access key "ci" in project proj_1 for buckets backups=rw logs=readonly (limited_access) and print the secret once (not saved)` + if got := configurePlanLine(limited); got != want { + t.Fatalf("limited plan line:\n got %s\nwant %s", got, want) + } + // Empty cells use the placeholder shared by the whole group. + if got := cfgFormatBucketPerms(nil); got != emptyCell { + t.Fatalf("empty perms placeholder: %q", got) + } + row := limited.TableRow() + if row["buckets"].Value != "backups=rw logs=readonly" || row["status"].Value != "created" || row["save"].Value != "no" || row["site"].Value != emptyCell { + t.Fatalf("table row: %+v", row) + } + if _, ok := row["saved_as"]; ok { + t.Fatal("saved_as cell only when the key was stored") + } + saved := full + saved.SavedAs, saved.Profile = "lsh-lanusse-tyo4-2", "default" + if got := saved.TableRow()["saved_as"].Value; got != "lsh-lanusse-tyo4-2" { + t.Fatalf("saved_as cell: %q", got) + } +} + +func TestConfigureStatePerms(t *testing.T) { + st := &configureState{Scope: config.ScopeLimitedAccess, Selected: []cfgSelectedBucket{ + {Bucket: &objectstorage.Bucket{ID: "bkt_1", Name: "backups"}, Permission: config.PermissionRW}, + {Bucket: &objectstorage.Bucket{ID: "bkt_2", Name: "logs"}, Permission: config.PermissionReadOnly}, + }} + if p := st.bucketPerms(); p["bkt_1"] != config.PermissionRW || p["bkt_2"] != config.PermissionReadOnly || len(p) != 2 { + t.Fatalf("bucketPerms by id: %v", p) + } + if p := st.bucketNamePerms(); p["backups"] != config.PermissionRW || p["logs"] != config.PermissionReadOnly { + t.Fatalf("bucketNamePerms by name: %v", p) + } + st.Scope = config.ScopeFullAccess + if st.bucketPerms() != nil || st.bucketNamePerms() != nil { + t.Fatal("fullaccess keys carry no bucket map") + } + if (&configureState{ProjectToken: "my-project", ProjectID: "proj_1"}).project() != "my-project" || (&configureState{ProjectID: "proj_1"}).project() != "proj_1" { + t.Fatal("project() prefers the token as given, then the resolved ID") + } +} + +func TestConfigureNeedsPromptIsUsageError(t *testing.T) { + err := configureNeedsPrompt("the project") + if exitcode.Of(err) != exitcode.Usage { + t.Fatalf("exit code %d, want %d", exitcode.Of(err), exitcode.Usage) + } + msg := err.Error() + for _, want := range []string{"the project", "lsh s3 access-keys create --bucket --save", objectstorage.EnvAccessKeyID, objectstorage.EnvSecretAccessKey} { + if !strings.Contains(msg, want) { + t.Errorf("message must mention %q: %s", want, msg) + } + } +} + +// TestConfigureFlagsParse exercises the flag parsing that now goes through +// the shared helpers (ParseStorageClass, regionFlag, parseBucketSpecs). +func TestConfigureFlagsParse(t *testing.T) { + parse := func(args ...string) (configureOptions, error) { + cmd := NewConfigureCmd() + cmd.SetArgs(args) + if err := cmd.ParseFlags(args); err != nil { + t.Fatalf("ParseFlags(%v): %v", args, err) + } + return parseConfigureOptions(cmd) + } + o, err := parse("--storage-class", "high-performance", "--region", "tyo4", "--bucket", "backups=rw", "--bucket", "logs=ro", "--name", " laptop ") + if err != nil { + t.Fatal(err) + } + if o.StorageClass != objectstorage.ClassHighPerformance || o.Site != "TYO4" || o.Name != "laptop" || !o.Save { + t.Fatalf("options: %+v", o) + } + if len(o.Buckets) != 2 || o.Buckets[1].Token != "logs" || o.Buckets[1].Permission != config.PermissionReadOnly { + t.Fatalf("bucket specs: %+v", o.Buckets) + } + if o, err := parse("--save=false"); err != nil || o.Save { + t.Fatalf("--save=false: %+v %v", o, err) + } + for _, bad := range [][]string{ + {"--storage-class", "glacier"}, + {"--region", "us-east-1"}, + {"--bucket", "backups=admin"}, + {"--all-buckets", "--bucket", "backups"}, + } { + if _, err := parse(bad...); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("%v must be a usage error, got %v", bad, err) + } + } +} diff --git a/cmd/s3/cp.go b/cmd/s3/cp.go new file mode 100644 index 0000000..8673bdd --- /dev/null +++ b/cmd/s3/cp.go @@ -0,0 +1,54 @@ +package s3 + +import ( + "github.com/spf13/cobra" +) + +// NewCpCmd builds `lsh s3 copy `. +func NewCpCmd() *cobra.Command { + var f transferFlags + cmd := newCmd(&cobra.Command{ + Use: "copy ", + Aliases: []string{"cp", "upload", "download"}, + GroupID: groupObjects, + Short: "Copy files and objects (alias: cp)", + Long: `Copy a local file to a bucket, an object to the local disk, or an object to +another object (server-side, same endpoint; when the two buckets resolve +different access keys the object is streamed through this machine instead, +since one signed copy request cannot read with one key and write with +another). The destination is overwritten +by default (--no-overwrite to skip existing ones), +--recursive copies whole directories or prefixes, and --exclude/--include +filter relative paths (rules apply in order; the last match wins). + +` + destinationRulesHelp + ` + +Uploads larger than 16 MiB use multipart; an interrupted upload is aborted. +To clean up leftovers from killed processes create a lifecycle rule: +'lsh s3 lifecycle create s3:// --abort-incomplete-multipart-days 1'. + +Streams from stdin ('-') are always multipart with 16 MiB parts, enough for +160 GiB. For larger streams pass --expected-size so the parts grow to +fit S3's 10000-part limit; the value is only a hint that also drives the +progress meter, and the upload succeeds whatever the real length turns out to be. + +The Content-Type is taken from --content-type, then the file extension, then +the first bytes of the file (--no-guess-mime-type stores binary/octet-stream). +Output lines (upload:/download:/copy:) go to stdout; progress, hints and +errors to stderr. --dry-run (or --dry-run) prints the plan without writing.`, + Example: ` lsh s3 copy ./dump.sql s3://backups/2026/09/ + lsh s3 copy s3://backups/2026/09/dump.sql ./restore/ + lsh s3 copy ./site s3://www --recursive --exclude "*" --include "*.html" --content-type text/html + lsh s3 copy s3://backups/2026/ ./backups/ --recursive --dry-run + pg_dump mydb | lsh s3 copy - s3://backups/mydb.sql + tar cz /data | lsh s3 copy - s3://backups/data.tgz --expected-size 500000000000 # approximate; sizes parts for streams over 160 GiB + lsh s3 copy s3://backups/report.pdf - > report.pdf + lsh s3 copy s3://backups/a.txt s3://archive/2026/a.txt --metadata owner=ops,team=infra`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return runCopyCommand(cmd, args, &f, false) + }, + }) + addTransferFlags(cmd, &f, true) + return cmd +} diff --git a/cmd/s3/e2e_test.go b/cmd/s3/e2e_test.go new file mode 100644 index 0000000..d4723ee --- /dev/null +++ b/cmd/s3/e2e_test.go @@ -0,0 +1,791 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/objectstorage/s3test" +) + +// The binary-level tests below compile lsh once per test process and drive +// the real cobra wiring through exec. They are skipped with -short. +var ( + lshBinOnce sync.Once + lshBinDir string + lshBinPath string + lshBinErr error +) + +// TestMain removes the shared binary built by buildLshBinary once every test +// in the package has run (t.TempDir would delete it after the first test). +func TestMain(m *testing.M) { + code := m.Run() + if lshBinDir != "" { + _ = os.RemoveAll(lshBinDir) + } + os.Exit(code) +} + +// buildLshBinary builds the lsh binary once and returns its path. Every +// binary-level test shares the same build. +func buildLshBinary(t *testing.T) string { + t.Helper() + if testing.Short() { + t.Skip("builds the binary; skipped with -short") + } + lshBinOnce.Do(func() { + root, err := filepath.Abs("../..") + if err != nil { + lshBinErr = err + return + } + dir, err := os.MkdirTemp("", "lsh-e2e-") + if err != nil { + lshBinErr = err + return + } + lshBinDir = dir + lshBinPath = filepath.Join(dir, "lsh-e2e") + build := exec.Command("go", "build", "-o", lshBinPath, ".") + build.Dir = root + if out, err := build.CombinedOutput(); err != nil { + lshBinErr = fmt.Errorf("go build: %v\n%s", err, out) + } + }) + if lshBinErr != nil { + t.Fatal(lshBinErr) + } + return lshBinPath +} + +// runResult is the outcome of one binary invocation. +type runResult struct { + stdout, stderr string + code int +} + +// runLsh executes the binary with env and args in dir, returning stdout, +// stderr and the exit code. Any failure that is not a non-zero exit is fatal. +func runLsh(t *testing.T, bin, dir string, env []string, args ...string) runResult { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, bin, args...) + cmd.Dir = dir + cmd.Env = env + var o, e bytes.Buffer + cmd.Stdout, cmd.Stderr = &o, &e + err := cmd.Run() + res := runResult{stdout: o.String(), stderr: e.String()} + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + res.code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("%v: %v\nstdout=%q\nstderr=%q", args, err, res.stdout, res.stderr) + } + return res +} + +// TestBinaryAgainstFakeS3 builds the real lsh binary and drives the cobra +// wiring end to end in the API-less mode (LSH_S3_ENDPOINT_URL + env +// credentials), covering flag parsing, output formats, dry-run, confirmation +// gating and exit codes that the unit tests bypass. +func TestBinaryAgainstFakeS3(t *testing.T) { + bin := buildLshBinary(t) + + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + srv.AddObject("backups-7f3a", "2026/09/a.sql", []byte("aaa"), "application/sql") + srv.AddObject("backups-7f3a", "2026/09/b.sql", []byte("bbbb"), "application/sql") + srv.AddObject("backups-7f3a", "readme.txt", []byte("hi"), "text/plain") + + home := t.TempDir() + work := t.TempDir() + env := append(os.Environ(), + "HOME="+home, + "LSH_S3_ENDPOINT_URL="+srv.URL(), + "LSH_S3_ACCESS_KEY_ID=AKIAEXAMPLE", + "LSH_S3_SECRET_ACCESS_KEY=topsecret", + "LATITUDESH_TOKEN=", + "LSH_S3_USE_AWS_ENV=", + "LSH_OUTPUT=", + "LSH_CLASSIC_OUTPUT=true", + "NO_COLOR=1", + ) + + run := func(args ...string) (stdout, stderr string, code int) { + r := runLsh(t, bin, work, env, args...) + return r.stdout, r.stderr, r.code + } + + // ls default: the shared lsh table (headers + borders), like every other + // list command; the prefix shows as a PRE row and the object by key. + out, errOut, code := run("s3", "ls", "s3://backups-7f3a/") + if code != 0 { + t.Fatalf("ls exit %d: %s%s", code, out, errOut) + } + if !strings.Contains(out, "KEY") || !strings.Contains(out, "2026/") || !strings.Contains(out, "readme.txt") { + t.Errorf("ls table output unexpected:\n%s", out) + } + if strings.Contains(out, "topsecret") || strings.Contains(errOut, "topsecret") { + t.Fatal("secret leaked") + } + + // ls -o json --query works (aws s3 ls ignores --output). + out, errOut, code = run("s3", "ls", "s3://backups-7f3a/2026/09/", "-o", "json", "--query", "[].key") + if code != 0 { + t.Fatalf("ls json exit %d: %s%s", code, out, errOut) + } + var keys []string + if err := json.Unmarshal([]byte(out), &keys); err != nil || len(keys) != 2 { + t.Errorf("ls -o json --query: %v %q", err, out) + } + + // cp upload with trailing slash keeps the file name; cp download to a + // directory; cp to stdout prints only the payload. + src := filepath.Join(work, "dump.sql") + if err := os.WriteFile(src, []byte("select 1;"), 0o600); err != nil { + t.Fatal(err) + } + out, errOut, code = run("s3", "cp", src, "s3://backups-7f3a/2026/10/") + if code != 0 || !strings.HasPrefix(out, "upload: ") || !strings.Contains(out, "to s3://backups-7f3a/2026/10/dump.sql") { + t.Fatalf("cp upload: exit %d out=%q err=%q", code, out, errOut) + } + if o := srv.Object("backups-7f3a", "2026/10/dump.sql"); o == nil || string(o.Data) != "select 1;" || o.ContentType == "" { + t.Fatalf("uploaded object missing or wrong: %+v", o) + } + if srv.HasChecksumHeaders() { + t.Error("upload sent checksum headers") + } + dest := filepath.Join(work, "restore") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + out, errOut, code = run("s3", "cp", "s3://backups-7f3a/2026/10/dump.sql", dest+string(os.PathSeparator)) + if code != 0 || !strings.HasPrefix(out, "download: ") { + t.Fatalf("cp download: exit %d out=%q err=%q", code, out, errOut) + } + if data, err := os.ReadFile(filepath.Join(dest, "dump.sql")); err != nil || string(data) != "select 1;" { + t.Fatalf("downloaded file: %v %q", err, data) + } + out, _, code = run("s3", "cp", "s3://backups-7f3a/readme.txt", "-") + if code != 0 || out != "hi" { + t.Errorf("cp to stdout: exit %d out=%q", code, out) + } + + // stat object. + out, errOut, code = run("s3", "stat", "s3://backups-7f3a/readme.txt", "-o", "json") + if code != 0 || !strings.Contains(out, `"key": "readme.txt"`) { + t.Errorf("stat: exit %d out=%q err=%q", code, out, errOut) + } + + // presign prints only a URL. + out, _, code = run("s3", "presign", "s3://backups-7f3a/readme.txt", "--expires-in", "15m") + if code != 0 || !strings.HasPrefix(strings.TrimSpace(out), srv.URL()+"/backups-7f3a/readme.txt?") || strings.Count(strings.TrimSpace(out), "\n") != 0 { + t.Errorf("presign: exit %d out=%q", code, out) + } + if _, _, code = run("s3", "presign", "s3://backups-7f3a/readme.txt", "--expires-in", "8d"); code != 2 { + t.Errorf("presign above 7d must be a usage error (2), got %d", code) + } + + // rm --recursive --dryrun: plan lines, zero writes. + srv.ResetRequests() + out, errOut, code = run("s3", "rm", "s3://backups-7f3a/2026/09/", "--recursive", "--dryrun") + if code != 0 || !strings.Contains(out, "(dryrun) delete: s3://backups-7f3a/2026/09/a.sql") { + t.Errorf("rm dryrun: exit %d out=%q err=%q", code, out, errOut) + } + if len(srv.WriteRequests()) != 0 { + t.Error("dry-run issued write requests") + } + + // rm --recursive without --yes in a non-interactive session is refused (7). + if _, errOut, code = run("s3", "rm", "s3://backups-7f3a/2026/09/", "--recursive"); code != 7 || !strings.Contains(errOut, "--yes") { + t.Errorf("rm recursive non-interactive: exit %d err=%q", code, errOut) + } + // Whole bucket requires --all. + if _, _, code = run("s3", "rm", "s3://backups-7f3a/", "--recursive", "--yes"); code != 2 { + t.Errorf("rm whole bucket without --all must exit 2, got %d", code) + } + // With --yes it deletes. + out, _, code = run("s3", "rm", "s3://backups-7f3a/2026/09/", "--recursive", "--yes") + if code != 0 || strings.Count(out, "delete: ") != 2 || srv.Object("backups-7f3a", "2026/09/a.sql") != nil { + t.Errorf("rm recursive: exit %d out=%q", code, out) + } + // Single delete of a missing key is idempotent. + if _, _, code = run("s3", "rm", "s3://backups-7f3a/does-not-exist"); code != 0 { + t.Errorf("rm missing key must exit 0, got %d", code) + } + + // Unknown bucket → 3 (not found). + if _, errOut, code = run("s3", "ls", "s3://nope-1234/"); code != 3 { + t.Errorf("ls unknown bucket: exit %d err=%q", code, errOut) + } + + // Unsupported aws flag gets an explanation, exit 2. + if _, errOut, code = run("s3", "cp", src, "s3://backups-7f3a/x", "--acl", "public-read"); code != 2 || !strings.Contains(errOut, "--acl") { + t.Errorf("--acl: exit %d err=%q", code, errOut) + } + + // rb needs the API: refused in endpoint mode with a usage error. + if _, _, code = run("s3", "rb", "s3://backups-7f3a"); code == 0 { + t.Error("rb in endpoint-override mode must fail") + } + + // mb is an API command: in endpoint-override mode it refuses with a usage + // error; without the override, --dryrun prints the plan even when not + // logged in. + if _, _, code = run("s3", "mb", "s3://newbucket", "--region", "DAL", "--project", "proj_1", "--dryrun"); code != 2 { + t.Errorf("mb with LSH_S3_ENDPOINT_URL set must exit 2, got %d", code) + } + { + noOverride := append(append([]string{}, env...), "LSH_S3_ENDPOINT_URL=") + r := runLsh(t, bin, work, noOverride, "s3", "mb", "s3://newbucket", "--region", "DAL", "--project", "proj_1", "--dryrun") + if r.code != 0 || !strings.Contains(r.stdout, "(dryrun) make_bucket: s3://newbucket") { + t.Errorf("mb dryrun: exit %d out=%q stderr=%q", r.code, r.stdout, r.stderr) + } + } + + // Missing credentials → 4 with the variable names in the message. + noCreds := append(append([]string{}, env...), "LSH_S3_ACCESS_KEY_ID=", "LSH_S3_SECRET_ACCESS_KEY=") + r := runLsh(t, bin, work, noCreds, "s3", "ls", "s3://backups-7f3a/") + if r.code != 4 || !strings.Contains(r.stderr, "LSH_S3_ACCESS_KEY_ID") { + t.Errorf("no credentials: exit %d stderr=%q", r.code, r.stderr) + } +} + +// Fixed identifiers served by the fake Latitude API. +const ( + e2eSecret = "S3CR3T-e2e-xyz" + e2eAccessKeyID = "AKIAE2E" + e2eBucketName = "bkt-e2e-7f3a" + e2eBucketID = "bkt_e2e" +) + +// fakeLatitudeAPI is a minimal JSON:API server for the /storage endpoints the +// access-keys and bucket resolution paths call. It accepts any Authorization +// and API-Version header without validation and records what it served. +type fakeLatitudeAPI struct { + srv *httptest.Server + s3URL string + + mu sync.Mutex + requests []string // "METHOD /path" + created []string // names of the keys created through POST +} + +func newFakeLatitudeAPI(s3URL string) *fakeLatitudeAPI { + f := &fakeLatitudeAPI{s3URL: s3URL} + f.srv = httptest.NewServer(http.HandlerFunc(f.handle)) + return f +} + +func (f *fakeLatitudeAPI) Close() { f.srv.Close() } + +// HostPort returns "127.0.0.1:" for --hostname. +func (f *fakeLatitudeAPI) HostPort() string { return strings.TrimPrefix(f.srv.URL, "http://") } + +func (f *fakeLatitudeAPI) Requests() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.requests...) +} + +func (f *fakeLatitudeAPI) Created() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.created...) +} + +func (f *fakeLatitudeAPI) bucket() map[string]any { + return map[string]any{ + "id": e2eBucketID, + "type": "object_storages", + "attributes": map[string]any{ + "name": "e2e", + "bucket_name": e2eBucketName, + "storage_class": "standard", + "endpoint": f.s3URL, + "versioning": false, + "locking": false, + // The real API serializes this as "" for buckets without object + // lock; the SDK model wants a number, so the CLI must normalize it. + "retention_period": "", + "source": "default", + "region": map[string]any{ + "city": "Dallas", + "site": map[string]any{"slug": "DAL"}, + }, + "project": map[string]any{"id": "proj_1", "slug": "e2e-project", "name": "E2E"}, + }, + } +} + +func (f *fakeLatitudeAPI) handle(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + f.mu.Lock() + f.requests = append(f.requests, r.Method+" "+r.URL.Path) + f.mu.Unlock() + + writeJSON := func(status int, v any) { + w.Header().Set("Content-Type", "application/vnd.api+json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) + } + path := strings.TrimSuffix(r.URL.Path, "/") + switch { + case r.Method == http.MethodGet && path == "/storage/buckets": + writeJSON(200, map[string]any{"data": []any{f.bucket()}}) + case r.Method == http.MethodGet && path == "/storage/buckets/"+e2eBucketID: + writeJSON(200, map[string]any{"data": f.bucket()}) + case r.Method == http.MethodPost && path == "/storage/access_keys": + var req struct { + Data struct { + Attributes struct { + Name string `json:"name"` + } `json:"attributes"` + } `json:"data"` + } + _ = json.Unmarshal(body, &req) + name := req.Data.Attributes.Name + f.mu.Lock() + f.created = append(f.created, name) + f.mu.Unlock() + writeJSON(201, map[string]any{"data": map[string]any{ + "type": "access_keys", + "attributes": map[string]any{ + "access_key": map[string]any{ + "access_key_id": e2eAccessKeyID, + "secret_access_key": e2eSecret, + "name": name, + "username": "e2e+x@latitude.sh", + "status": "Active", + }, + }, + }}) + case r.Method == http.MethodGet && path == "/storage/access_keys": + writeJSON(200, map[string]any{"data": map[string]any{ + "standard": []any{map[string]any{ + "name": "k1", + "username": "e2e+x@latitude.sh", + "access_key_id": e2eAccessKeyID, + "status": "Active", + "created_at": "2026-09-07T10:00:00Z", + "region": "DAL", + "access": "rw", + "buckets": []string{e2eBucketName}, + }}, + "high_performance": []any{}, + }}) + case r.Method == http.MethodDelete && strings.HasPrefix(path, "/storage/access_keys/"): + w.WriteHeader(http.StatusNoContent) + default: + writeJSON(404, map[string]any{"errors": []any{map[string]any{ + "status": "404", "title": "Not Found", "detail": r.Method + " " + r.URL.Path, + }}}) + } +} + +// writeE2EConfig writes ~/.config/lsh/config.json under home with a profile +// named e2e as the default. jsonDefault adds the top-level "json": true that +// the config file uses to make json the default output format. +func writeE2EConfig(t *testing.T, home string, jsonDefault bool) string { + t.Helper() + cfg := map[string]any{ + "default_profile": "e2e", + "profiles": map[string]any{ + "e2e": map[string]any{"authorization": "test-token"}, + }, + } + if jsonDefault { + cfg["json"] = true + } + dir := filepath.Join(home, ".config", "lsh") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + data, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + t.Fatal(err) + } + p := filepath.Join(dir, "config.json") + if err := os.WriteFile(p, data, 0o600); err != nil { + t.Fatal(err) + } + return p +} + +// e2eEnv is the environment for the API-backed runs: token from the +// environment, no S3 endpoint override, no env credentials, and no inherited +// output preference unless the case sets one. +func e2eEnv(home string, extra ...string) []string { + env := append(os.Environ(), + "HOME="+home, + "LATITUDESH_TOKEN=test-token", + "LSH_S3_ENDPOINT_URL=", + "LSH_S3_ACCESS_KEY_ID=", + "LSH_S3_SECRET_ACCESS_KEY=", + "LSH_S3_USE_AWS_ENV=", + "LSH_OUTPUT=", + "LSH_PROFILE=", + "LSH_PROJECT=", + "LSH_CLASSIC_OUTPUT=true", + "NO_COLOR=1", + ) + return append(env, extra...) +} + +// jsonField decodes structured output (one object, or a one-element array) +// and returns the string value of field. +func jsonField(t *testing.T, out, field string) (string, bool) { + t.Helper() + var v any + if err := json.Unmarshal([]byte(out), &v); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + obj, ok := v.(map[string]any) + if arr, isArr := v.([]any); isArr { + if len(arr) != 1 { + t.Fatalf("expected one JSON object, got %d: %s", len(arr), out) + } + obj, ok = arr[0].(map[string]any) + } + if !ok { + t.Fatalf("expected a JSON object: %s", out) + } + val, present := obj[field] + if !present { + return "", false + } + s, _ := val.(string) + return s, true +} + +// TestBinaryNoSecretContract drives the binary against a fake Latitude API +// and the fake S3 endpoint to pin the secret-display contract of +// `access-keys create` (spec section 10) and the cobra error handling of +// newCmd (one message, exit 2). Unless a case says otherwise, neither stdout +// nor stderr may contain the secret. +func TestBinaryNoSecretContract(t *testing.T) { + bin := buildLshBinary(t) + + s3srv := s3test.New() + defer s3srv.Close() + s3srv.CreateBucket(e2eBucketName) + s3srv.AddObject(e2eBucketName, "readme.txt", []byte("hi"), "text/plain") + + api := newFakeLatitudeAPI(s3srv.URL()) + defer api.Close() + + work := t.TempDir() + if err := os.WriteFile(filepath.Join(work, "f"), []byte("payload"), 0o600); err != nil { + t.Fatal(err) + } + apiArgs := []string{"--hostname", api.HostPort(), "--scheme", "http"} + run := func(env []string, args ...string) runResult { + t.Helper() + return runLsh(t, bin, work, env, append(append([]string{}, args...), apiArgs...)...) + } + noSecret := func(name string, r runResult) { + t.Helper() + if strings.Contains(r.stdout, e2eSecret) { + t.Errorf("%s: secret leaked on stdout:\n%s", name, r.stdout) + } + if strings.Contains(r.stderr, e2eSecret) { + t.Errorf("%s: secret leaked on stderr:\n%s", name, r.stderr) + } + } + createArgs := []string{"s3", "access-keys", "create", "--bucket", "e2e", "--name", "k1"} + + home := t.TempDir() + writeE2EConfig(t, home, false) + env := e2eEnv(home) + + // 8. Cobra errors: printed once, exit 2, no usage dump. These never reach + // the API, so they run first and report even when the API cases cannot. + r := run(env, "s3", "ls", "--bogus-flag") + if r.code != 2 || strings.Count(r.stderr, "unknown flag") != 1 { + t.Errorf("ls --bogus-flag: exit %d (want 2), stderr must mention 'unknown flag' once:\n%s", r.code, r.stderr) + } + if strings.Contains(r.stderr, "Usage:") { + t.Errorf("ls --bogus-flag: usage must not be echoed:\n%s", r.stderr) + } + r = run(env, "s3", "stat") + if r.code != 2 || strings.TrimSpace(r.stderr) == "" || strings.Count(r.stderr, "accepts 1 arg") != 1 { + t.Errorf("stat without args: exit %d (want 2), stderr=%q", r.code, r.stderr) + } + r = run(env, "s3", "cp", "onlyone") + if r.code != 2 || strings.TrimSpace(r.stderr) == "" { + t.Errorf("cp with one arg: exit %d (want 2), stderr=%q", r.code, r.stderr) + } + + // 9. aws-style --region on an object command is redirected to + // --signing-region with exit 2 (checked in PreRunE, before any API call). + r = run(env, "s3", "cp", "./f", "s3://e2e/", "--region", "us-east-1") + if r.code != 2 || !strings.Contains(r.stderr, "--signing-region") { + t.Errorf("cp --region: exit %d (want 2), stderr must explain --signing-region:\n%s", r.code, r.stderr) + } + if strings.Contains(r.stderr, "unknown flag") { + t.Errorf("cp --region must not be reported as an unknown flag:\n%s", r.stderr) + } + + // 1. Human output, not saved: the secret is shown in clear, exactly once, + // on stdout (it cannot be retrieved again), never on stderr. + r = run(env, createArgs...) + if r.code != 0 { + reqs := api.Requests() + hint := "" + if len(reqs) == 0 { + hint = "\nthe fake API received no request: the SDK client ignores --hostname/--scheme (objectstorage.NewAPIClient builds sdk.New without WithServerURL)" + } + t.Fatalf("create (human): exit %d\nstdout=%s\nstderr=%s\napi requests=%v%s", r.code, r.stdout, r.stderr, reqs, hint) + } + if n := strings.Count(r.stdout, e2eSecret); n != 1 { + t.Errorf("create (human): secret must be printed exactly once on stdout, found %d:\n%s", n, r.stdout) + } + if strings.Contains(r.stderr, e2eSecret) { + t.Errorf("create (human): secret on stderr:\n%s", r.stderr) + } + if !strings.Contains(r.stdout, e2eAccessKeyID) { + t.Errorf("create (human): access key id missing:\n%s", r.stdout) + } + if got := api.Created(); len(got) != 1 || got[0] != "k1" { + t.Errorf("fake API saw created keys %v, want [k1]", got) + } + + // 2. LSH_OUTPUT=json without -o: structured output does not embed the + // secret and stderr tells how to get it. + r = run(e2eEnv(home, "LSH_OUTPUT=json"), createArgs...) + if r.code != 0 { + t.Fatalf("create (LSH_OUTPUT=json): exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("create (LSH_OUTPUT=json)", r) + if _, present := jsonField(t, r.stdout, "secret_access_key"); present { + t.Errorf("create (LSH_OUTPUT=json): secret_access_key present in JSON:\n%s", r.stdout) + } + if !strings.Contains(r.stderr, "--show-secret") || !strings.Contains(r.stderr, "-o json") { + t.Errorf("create (LSH_OUTPUT=json): stderr must point at --show-secret / -o json:\n%s", r.stderr) + } + + // 3. Explicit -o json embeds the secret. + r = run(env, append(createArgs, "-o", "json")...) + if r.code != 0 { + t.Fatalf("create (-o json): exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + if got, _ := jsonField(t, r.stdout, "secret_access_key"); got != e2eSecret { + t.Errorf("create (-o json): secret_access_key=%q, want %q:\n%s", got, e2eSecret, r.stdout) + } + if strings.Contains(r.stderr, e2eSecret) { + t.Errorf("create (-o json): secret on stderr:\n%s", r.stderr) + } + + // 4. --json on the command line counts as explicit too. + r = run(env, append(createArgs, "--json")...) + if r.code != 0 { + t.Fatalf("create (--json): exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + if got, _ := jsonField(t, r.stdout, "secret_access_key"); got != e2eSecret { + t.Errorf("create (--json): secret_access_key=%q, want %q:\n%s", got, e2eSecret, r.stdout) + } + + // 5. "json": true in the config file is not an explicit request. + { + jsonHome := t.TempDir() + writeE2EConfig(t, jsonHome, true) + r = run(e2eEnv(jsonHome), createArgs...) + if r.code != 0 { + t.Fatalf("create (config json=true): exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("create (config json=true)", r) + if _, present := jsonField(t, r.stdout, "secret_access_key"); present { + t.Errorf("create (config json=true): secret_access_key present in JSON:\n%s", r.stdout) + } + if !strings.Contains(r.stderr, "--show-secret") { + t.Errorf("create (config json=true): stderr must point at --show-secret:\n%s", r.stderr) + } + } + + // 6. --save stores the key in profile e2e and prints no secret anywhere. + r = run(env, append(createArgs, "--save")...) + if r.code != 0 { + t.Fatalf("create --save: exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("create --save", r) + if !strings.Contains(r.stdout+r.stderr, "Saved as") { + t.Errorf("create --save: expected a 'Saved as' confirmation:\nstdout=%s\nstderr=%s", r.stdout, r.stderr) + } + { + data, err := os.ReadFile(filepath.Join(home, ".config", "lsh", "config.json")) + if err != nil { + t.Fatal(err) + } + var cfg struct { + Profiles map[string]struct { + ObjectStorage struct { + Keys map[string]struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + } `json:"keys"` + } `json:"object_storage"` + } `json:"profiles"` + } + if err := json.Unmarshal(data, &cfg); err != nil { + t.Fatalf("config.json: %v\n%s", err, data) + } + found := false + for name, k := range cfg.Profiles["e2e"].ObjectStorage.Keys { + if k.AccessKeyID == e2eAccessKeyID { + found = true + if k.SecretAccessKey != e2eSecret { + t.Errorf("saved key %q has secret %q, want the API secret", name, k.SecretAccessKey) + } + } + } + if !found { + t.Errorf("config.json has no key with access_key_id %s under profile e2e:\n%s", e2eAccessKeyID, data) + } + } + + // 6b. --save with an unwritable profile: the key cannot be stored, so it is + // deleted again. Nothing is left live on the API and the one-time secret is + // never printed on either stream. + if os.Geteuid() != 0 { + roHome := t.TempDir() + cfgDir := filepath.Join(roHome, ".config", "lsh") + writeE2EConfig(t, roHome, false) + if err := os.Chmod(cfgDir, 0o500); err != nil { + t.Fatal(err) + } + // Restore write access so the temp dir can be cleaned up. + t.Cleanup(func() { _ = os.Chmod(cfgDir, 0o700) }) + + r = run(e2eEnv(roHome), append(createArgs, "--save", "-o", "json")...) + // --save means create *and* persist: a caller must not read exit 0 as + // "the credential is in the profile". + if r.code != 1 { + t.Fatalf("create --save (unwritable profile): exit %d (want 1)\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + if !strings.Contains(r.stderr, "not saved in the profile") { + t.Errorf("create --save (unwritable profile): stderr must state the outcome:\n%s", r.stderr) + } + // The key does not exist any more, so there is no document to emit: + // stdout stays empty rather than describing a credential that is gone. + if strings.TrimSpace(r.stdout) != "" { + t.Errorf("create --save (unwritable profile): stdout must stay empty, got:\n%s", r.stdout) + } + if strings.Contains(r.stderr, e2eSecret) { + t.Errorf("create --save (unwritable profile): the key was removed, so no secret may be printed:\n%s", r.stderr) + } + for _, want := range []string{"could not be saved", "It was deleted again", "Retry once the profile is writable"} { + if !strings.Contains(r.stderr, want) { + t.Errorf("create --save (unwritable profile): stderr lacks %q:\n%s", want, r.stderr) + } + } + deletes := 0 + for _, req := range api.Requests() { + if strings.HasPrefix(req, "DELETE ") && strings.Contains(req, "/storage/access_keys") { + deletes++ + } + } + if deletes != 1 { + t.Errorf("create --save (unwritable profile): expected one access-key delete, got %d (%v)", deletes, api.Requests()) + } + if strings.Contains(r.stderr, "it was saved in the profile") { + t.Errorf("create --save (unwritable profile): stderr must not claim the key was saved:\n%s", r.stderr) + } + } + + // 7. The saved key is selected automatically for the bucket; --debug + // tracing never shows the secret and the listing reaches the S3 server. + r = run(env, "s3", "ls", "s3://e2e/", "--debug") + if r.code != 0 { + t.Fatalf("ls --debug with saved key: exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("ls --debug", r) + if !strings.Contains(r.stdout, " readme.txt") { + t.Errorf("ls --debug: listing missing the seeded object:\n%s", r.stdout) + } + if !strings.Contains(r.stderr, "saved key") { + t.Errorf("ls --debug: expected the credential source in the debug trace:\n%s", r.stderr) + } + + // The list command exposes the id and scope but never a secret. + r = run(env, "s3", "access-keys", "list", "--project", "e2e-project", "-o", "json") + if r.code != 0 { + t.Fatalf("access-keys list: exit %d\nstdout=%s\nstderr=%s", r.code, r.stdout, r.stderr) + } + noSecret("access-keys list", r) + if !strings.Contains(r.stdout, e2eAccessKeyID) { + t.Errorf("access-keys list: access key id missing:\n%s", r.stdout) + } +} + +// TestBinaryFlagValidationExitCodes covers the exit-code contract for +// command-line errors detected by the root pre-run: an invalid --output or +// --query is a usage error (2), not a generic failure (1). +func TestBinaryFlagValidationExitCodes(t *testing.T) { + bin := buildLshBinary(t) + home := t.TempDir() + writeE2EConfig(t, home, false) + env := e2eEnv(home) + work := t.TempDir() + + cases := []struct { + name string + args []string + }{ + {"invalid output format", []string{"s3", "ls", "-o", "bogus"}}, + {"invalid query", []string{"s3", "ls", "--query", "[[["}}, + {"invalid page size", []string{"s3", "ls", "--page-size", "-1"}}, + } + for _, c := range cases { + r := runLsh(t, bin, work, env, c.args...) + if r.code != 2 { + t.Errorf("%s: exit %d, want 2\nstderr=%s", c.name, r.code, r.stderr) + } + if strings.TrimSpace(r.stderr) == "" { + t.Errorf("%s: expected an actionable message on stderr", c.name) + } + } +} + +// TestBinaryLegacyExitCodesUnchanged pins the boundary of the documented S3 +// exit codes: they apply to the s3 subtree (including its legacy aliases) and +// must not change what the older command groups return. +func TestBinaryLegacyExitCodesUnchanged(t *testing.T) { + bin := buildLshBinary(t) + home := t.TempDir() + writeE2EConfig(t, home, false) + env := e2eEnv(home) + work := t.TempDir() + + cases := []struct { + name string + args []string + want int + }{ + {"s3 opted in", []string{"s3", "ls", "-o", "bogus"}, 2}, + {"s3 through the legacy alias", []string{"storage-objects", "list", "-o", "bogus"}, 2}, + {"older group keeps exiting 1", []string{"servers", "list", "-o", "bogus"}, 1}, + {"older group, bad page size", []string{"servers", "list", "--page-size", "-1"}, 1}, + } + for _, c := range cases { + r := runLsh(t, bin, work, env, c.args...) + if r.code != c.want { + t.Errorf("%s: exit %d, want %d\nstderr=%s", c.name, r.code, c.want, r.stderr) + } + } +} diff --git a/cmd/s3/lifecycle.go b/cmd/s3/lifecycle.go new file mode 100644 index 0000000..4ab999e --- /dev/null +++ b/cmd/s3/lifecycle.go @@ -0,0 +1,386 @@ +package s3 + +import ( + "context" + "fmt" + "sort" + "strings" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +// Flag names shared by lifecycle create/update. +const ( + flagRuleName = "name" + flagRulePrefix = "prefix" + flagExpirationDays = "expiration-days" + flagExpireDays = "expire-days" // hidden alias of --expiration-days + flagNoncurrentDays = "noncurrent-days" + flagAbortMpuDays = "abort-incomplete-multipart-days" +) + +// lifecycleIDPrefix is the prefix of every lifecycle rule ID returned by the API. +const lifecycleIDPrefix = "lifecycle_" + +// NewLifecycleCmd builds `lsh s3 lifecycle`, the bucket lifecycle rule group +// (list, get, create, update, delete). Rules are managed through the Latitude +// API; the bucket may be given as s3:// or . +func NewLifecycleCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "lifecycle", + Aliases: []string{"lifecycle-rules"}, + GroupID: groupReports, + Short: "Manage lifecycle rules (expire objects automatically)", + Long: `Manage the lifecycle rules of a bucket (automatic expiration of objects, +noncurrent versions and incomplete multipart uploads). + +Rules are stored by the Latitude API, so no S3 credential is needed. Every +rule needs --expiration-days; rules can be addressed by id (lifecycle_...) or +by name in get/update/delete.`, + Example: ` lsh s3 lifecycle create s3://logs --prefix tmp/ --expiration-days 7 + lsh s3 lifecycle create s3://logs --name abort-mpu --expiration-days 3650 --abort-incomplete-multipart-days 2 + lsh s3 lifecycle list s3://logs + lsh s3 lifecycle update s3://logs expire-7d-tmp --expiration-days 14 + lsh s3 lifecycle delete s3://logs expire-7d-tmp`, + } + cmd.AddCommand( + newLifecycleListCmd(), + newLifecycleGetCmd(), + newLifecycleCreateCmd(), + newLifecycleUpdateCmd(), + newLifecycleDeleteCmd(), + ) + return cmd +} + +// LifecycleRule wraps the SDK payload so the shared renderer can print it +// (-o table|json|yaml|csv, --query). JSON output is the API document as-is. +type LifecycleRule struct { + components.LifecycleRuleData +} + +// RuleID returns the rule id or "". +func (r LifecycleRule) RuleID() string { return ruleID(r.LifecycleRuleData) } + +// RuleName returns the rule name or "". +func (r LifecycleRule) RuleName() string { return ruleName(r.LifecycleRuleData) } + +// TableRow renders the rule as a table row: id, name, prefix, expiration, +// noncurrent, abort-mpu and enabled. +func (r LifecycleRule) TableRow() table.Row { + a := r.Attributes + return table.Row{ + "id": {Label: "ID", Value: r.RuleID()}, + "name": {Label: "Name", Value: r.RuleName()}, + "prefix": {Label: "Prefix", Value: strOrDash(a.GetPrefix())}, + "expiration_days": {Label: "Expiration", Value: daysLabel(a.GetExpirationDays())}, + "noncurrent_days": {Label: "Noncurrent", Value: daysLabel(a.GetNoncurrentDays())}, + "abort_mpu_days": {Label: "Abort MPU", Value: daysLabel(a.GetAbortMpuDaysAfterInitiation())}, + "enabled": {Label: "Enabled", Value: boolLabel(a.GetEnabled())}, + } +} + +// lifecycleRuleRows adapts SDK rules for the renderer. +func lifecycleRuleRows(rules []components.LifecycleRuleData) []renderer.ResponseData { + out := make([]renderer.ResponseData, 0, len(rules)) + for _, r := range rules { + out = append(out, LifecycleRule{LifecycleRuleData: r}) + } + return out +} + +// LifecycleMutation is the structured result of create/update/delete +// (create/update render the resulting rule; delete and dry-runs render this). +type LifecycleMutation struct { + // Action is create, update or delete. + Action string `json:"action"` + Bucket string `json:"bucket"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + Error string `json:"error,omitempty"` +} + +func (m LifecycleMutation) TableRow() table.Row { + status := "ok" + switch { + case m.Error != "": + status = "error: " + m.Error + case m.DryRun: + status = "dryrun" + } + return table.Row{ + "action": {Label: "Action", Value: m.Action}, + "bucket": {Label: "Bucket", Value: m.Bucket}, + "id": {Label: "ID", Value: m.ID}, + "name": {Label: "Name", Value: m.Name}, + "status": {Label: "Status", Value: status}, + } +} + +// HumanLine renders the aws-style single line: `lifecycle: created ()`, +// or `(dryrun) lifecycle: create on s3://bucket` for a dry-run. +func (m LifecycleMutation) HumanLine() string { + subject := m.Name + if m.ID != "" && m.Name != "" { + subject = fmt.Sprintf("%s (%s)", m.ID, m.Name) + } else if m.ID != "" { + subject = m.ID + } + if m.DryRun { + return fmt.Sprintf("(dryrun) lifecycle: %s %s on %s", m.Action, subject, m.Bucket) + } + if m.Error != "" { + return fmt.Sprintf("lifecycle: failed to %s %s: %s", m.Action, subject, m.Error) + } + return fmt.Sprintf("lifecycle: %s %s", pastTense(m.Action), subject) +} + +func pastTense(action string) string { + switch action { + case "create": + return "created" + case "update": + return "updated" + case "delete": + return "deleted" + } + return action +} + +// ruleID / ruleName read the optional SDK fields. +func ruleID(r components.LifecycleRuleData) string { + if r.ID == nil { + return "" + } + return *r.ID +} + +func ruleName(r components.LifecycleRuleData) string { + if r.Attributes == nil || r.Attributes.Name == nil { + return "" + } + return *r.Attributes.Name +} + +// daysLabel renders "7d", or the shared table placeholder when unset. +func daysLabel(v *int64) string { + if v == nil { + return emptyCell + } + return fmt.Sprintf("%dd", *v) +} + +// strOrDash renders the string, or the shared table placeholder when unset. +func strOrDash(v *string) string { + if v == nil { + return emptyCell + } + return orEmptyCell(*v) +} + +// boolLabel renders true/false, or the shared table placeholder when unset. +func boolLabel(v *bool) string { + if v == nil { + return emptyCell + } + if *v { + return "true" + } + return "false" +} + +// findLifecycleRule picks the rule addressed by token: an exact id match wins, +// otherwise the rule whose name equals token. It fails with exit 3 when +// nothing matches and exit 2 when several rules share the name. +func findLifecycleRule(rules []components.LifecycleRuleData, token string) (*components.LifecycleRuleData, error) { + token = strings.TrimSpace(token) + if token == "" { + return nil, exitcode.Errorf(exitcode.Usage, "missing lifecycle rule: pass the rule id (lifecycle_...) or its name") + } + for i := range rules { + if ruleID(rules[i]) == token { + return &rules[i], nil + } + } + var matches []int + for i := range rules { + if ruleName(rules[i]) == token { + matches = append(matches, i) + } + } + switch len(matches) { + case 0: + return nil, exitcode.Errorf(exitcode.NotFound, "lifecycle rule %q not found on this bucket; run 'lsh s3 lifecycle list' to see the rules", token) + case 1: + return &rules[matches[0]], nil + } + ids := make([]string, 0, len(matches)) + for _, i := range matches { + ids = append(ids, " "+ruleID(rules[i])) + } + sort.Strings(ids) + return nil, exitcode.Errorf(exitcode.Usage, "lifecycle rule name %q matches several rules:\n%s\nuse the rule id instead", token, strings.Join(ids, "\n")) +} + +// defaultLifecycleRuleName builds the default name of a new rule: +// `expire-d`, plus `-` (sanitized to [a-z0-9-]) when a prefix is set. +func defaultLifecycleRuleName(expirationDays int64, prefix string) string { + name := fmt.Sprintf("expire-%dd", expirationDays) + if s := sanitizeRuleNamePart(prefix); s != "" { + name += "-" + s + } + return name +} + +// sanitizeRuleNamePart lowercases s and collapses every run of characters +// outside [a-z0-9] into a single "-", trimming dashes at both ends. +func sanitizeRuleNamePart(s string) string { + var b strings.Builder + dash := false + for _, r := range strings.ToLower(s) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + dash = false + default: + if b.Len() > 0 && !dash { + b.WriteByte('-') + } + dash = true + } + } + return strings.Trim(b.String(), "-") +} + +// addLifecycleRuleFlags registers the attribute flags shared by create/update. +func addLifecycleRuleFlags(cmd *cobra.Command) { + f := cmd.Flags() + f.String(flagRuleName, "", "rule name (default: expire-d[-])") + f.String(flagRulePrefix, "", "only apply to objects whose key starts with this prefix (default: whole bucket)") + f.Int64(flagExpirationDays, 0, "delete objects this many days after creation (required by the API)") + f.Int64(flagExpireDays, 0, "alias of --expiration-days") + f.Int64(flagNoncurrentDays, 0, "delete noncurrent versions this many days after they become noncurrent (versioned buckets)") + f.Int64(flagAbortMpuDays, 0, "abort incomplete multipart uploads this many days after initiation") + _ = f.MarkHidden(flagExpireDays) +} + +// expirationDaysFlag returns --expiration-days (or its alias --expire-days) +// and whether either was given. +func expirationDaysFlag(cmd *cobra.Command) (int64, bool) { + if cmd.Flags().Changed(flagExpirationDays) { + v, _ := cmd.Flags().GetInt64(flagExpirationDays) + return v, true + } + if cmd.Flags().Changed(flagExpireDays) { + v, _ := cmd.Flags().GetInt64(flagExpireDays) + return v, true + } + return 0, false +} + +// lifecycleAPI bundles the API client and the resolved bucket for one command. +type lifecycleAPI struct { + api *sdk.Latitudesh + bucket *objectstorage.Bucket + opts []operations.Option +} + +// openLifecycleAPI parses the bucket argument, resolves it through the API +// and returns the client used by every lifecycle subcommand. +func openLifecycleAPI(ctx context.Context, cmd *cobra.Command, arg string) (*lifecycleAPI, error) { + ref, err := objectstorage.ParseBucketOnly(arg) + if err != nil { + return nil, err + } + if endpointOverride(cmd) != "" { + return nil, exitcode.Errorf(exitcode.Usage, "lifecycle rules are managed through the Latitude API; unset --endpoint-url / %s to use this command", objectstorage.EnvEndpointURL) + } + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return nil, err + } + if b.EndpointOverride || b.ID == "" { + return nil, exitcode.Errorf(exitcode.Usage, "bucket %s has no API id; lifecycle rules need the Latitude API", b.Display()) + } + return &lifecycleAPI{ + api: apiClient(), + bucket: b, + opts: []operations.Option{operations.WithRetries(lsh.RetryConfig())}, + }, nil +} + +// uri returns the s3:// form of the bucket for messages. +func (l *lifecycleAPI) uri() string { return "s3://" + l.bucket.Name } + +// list returns every rule of the bucket (empty slice when there is none). +func (l *lifecycleAPI) list(ctx context.Context) ([]components.LifecycleRuleData, error) { + resp, err := l.api.ObjectStorage.GetStorageBucketLifecycleRules(ctx, l.bucket.ID, l.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("bucket %s", l.bucket.Display())) + } + if resp.LifecycleRules == nil { + return []components.LifecycleRuleData{}, nil + } + return resp.LifecycleRules.Data, nil +} + +// get resolves token (lifecycle_ id or name) to a rule. Ids are fetched +// directly; names are looked up in the rule list. +func (l *lifecycleAPI) get(ctx context.Context, token string) (*components.LifecycleRuleData, error) { + token = strings.TrimSpace(token) + if strings.HasPrefix(token, lifecycleIDPrefix) { + resp, err := l.api.ObjectStorage.GetStorageBucketLifecycleRule(ctx, l.bucket.ID, token, l.opts...) + switch { + case err != nil && exitcode.Of(objectstorage.HumanizeAPI(err, "")) != exitcode.NotFound: + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("lifecycle rule %q on bucket %s", token, l.bucket.Display())) + case err == nil && resp.Object != nil && resp.Object.Data != nil: + return resp.Object.Data, nil + } + // Rule names are free text, so a name may start with the ID prefix: + // fall through to the name lookup instead of reporting "not found". + } + rules, err := l.list(ctx) + if err != nil { + return nil, err + } + return findLifecycleRule(rules, token) +} + +func (l *lifecycleAPI) create(ctx context.Context, body operations.PostStorageBucketLifecycleRulesRequestBody) (*components.LifecycleRuleData, error) { + resp, err := l.api.ObjectStorage.PostStorageBucketLifecycleRules(ctx, l.bucket.ID, body, l.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("bucket %s", l.bucket.Display())) + } + if resp.Object == nil || resp.Object.Data == nil { + return nil, exitcode.Errorf(exitcode.Generic, "the API returned no lifecycle rule; run 'lsh s3 lifecycle list %s' to check", l.uri()) + } + return resp.Object.Data, nil +} + +func (l *lifecycleAPI) update(ctx context.Context, id string, body operations.PatchStorageBucketLifecycleRuleRequestBody) (*components.LifecycleRuleData, error) { + resp, err := l.api.ObjectStorage.PatchStorageBucketLifecycleRule(ctx, l.bucket.ID, id, body, l.opts...) + if err != nil { + return nil, objectstorage.HumanizeAPI(err, fmt.Sprintf("lifecycle rule %q on bucket %s", id, l.bucket.Display())) + } + if resp.Object == nil || resp.Object.Data == nil { + return nil, exitcode.Errorf(exitcode.Generic, "the API returned no lifecycle rule; run 'lsh s3 lifecycle get %s %s' to check", l.uri(), id) + } + return resp.Object.Data, nil +} + +func (l *lifecycleAPI) delete(ctx context.Context, id string) error { + _, err := l.api.ObjectStorage.DeleteStorageBucketLifecycleRule(ctx, l.bucket.ID, id, l.opts...) + if err != nil { + return objectstorage.HumanizeAPI(err, fmt.Sprintf("lifecycle rule %q on bucket %s", id, l.bucket.Display())) + } + return nil +} diff --git a/cmd/s3/lifecycle_create.go b/cmd/s3/lifecycle_create.go new file mode 100644 index 0000000..1bd251f --- /dev/null +++ b/cmd/s3/lifecycle_create.go @@ -0,0 +1,156 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +const flagRuleDisabled = "disabled" + +// lifecycleCreateInput is the flag set of `lifecycle create`, kept separate +// from cobra so the request builder is testable. +type lifecycleCreateInput struct { + Name string + Prefix string + ExpirationDays int64 + NoncurrentDays int64 + AbortMpuDays int64 + Disabled bool +} + +// buildLifecycleCreateRequest validates the input and builds the POST body. +// --expiration-days is mandatory on the API side, so it is checked here with +// a clear message instead of a 422. +func buildLifecycleCreateRequest(in lifecycleCreateInput) (operations.PostStorageBucketLifecycleRulesRequestBody, error) { + var body operations.PostStorageBucketLifecycleRulesRequestBody + if in.ExpirationDays <= 0 { + return body, exitcode.Errorf(exitcode.Usage, "the API requires --expiration-days on every rule (a positive number of days after which objects are deleted)") + } + if in.NoncurrentDays < 0 { + return body, exitcode.Errorf(exitcode.Usage, "--%s must be a positive number of days", flagNoncurrentDays) + } + if in.AbortMpuDays < 0 { + return body, exitcode.Errorf(exitcode.Usage, "--%s must be a positive number of days", flagAbortMpuDays) + } + name := strings.TrimSpace(in.Name) + if name == "" { + name = defaultLifecycleRuleName(in.ExpirationDays, in.Prefix) + } + enabled := !in.Disabled + attrs := operations.PostStorageBucketLifecycleRulesAttributes{ + Name: name, + Enabled: &enabled, + ExpirationDays: in.ExpirationDays, + } + if p := in.Prefix; p != "" { + attrs.Prefix = &p + } + if in.NoncurrentDays > 0 { + v := in.NoncurrentDays + attrs.NoncurrentDays = &v + } + if in.AbortMpuDays > 0 { + v := in.AbortMpuDays + attrs.AbortMpuDaysAfterInitiation = &v + } + body.Data = operations.PostStorageBucketLifecycleRulesData{ + Type: operations.PostStorageBucketLifecycleRulesTypeLifecycleRules, + Attributes: attrs, + } + return body, nil +} + +// plannedLifecycleRule turns a create request into a renderable rule for +// --dry-run (no id, since nothing was created). +func plannedLifecycleRule(body operations.PostStorageBucketLifecycleRulesRequestBody) LifecycleRule { + a := body.Data.Attributes + name := a.Name + exp := a.ExpirationDays + typ := components.LifecycleRuleDataTypeLifecycleRules + return LifecycleRule{LifecycleRuleData: components.LifecycleRuleData{ + Type: &typ, + Attributes: &components.LifecycleRuleDataAttributes{ + Name: &name, + Enabled: a.Enabled, + Prefix: a.Prefix, + ExpirationDays: &exp, + NoncurrentDays: a.NoncurrentDays, + AbortMpuDaysAfterInitiation: a.AbortMpuDaysAfterInitiation, + }, + }} +} + +// newLifecycleCreateCmd builds `lsh s3 lifecycle create s3://bucket`. +func newLifecycleCreateCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "create ", + Aliases: []string{"add"}, + Short: "Create a lifecycle rule", + Long: `Create a lifecycle rule on a bucket. + +--expiration-days is required by the API on every rule, even when the goal is +only to abort incomplete multipart uploads or expire noncurrent versions. The +default name is expire-d, with - appended when --prefix is given.`, + Example: ` lsh s3 lifecycle create s3://logs --prefix tmp/ --expiration-days 7 + lsh s3 lifecycle create s3://logs --name abort-mpu --expiration-days 3650 --abort-incomplete-multipart-days 2 + lsh s3 lifecycle create s3://backups --expiration-days 90 --noncurrent-days 30 --disabled`, + Args: cobra.ExactArgs(1), + RunE: runLifecycleCreate, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + addLifecycleRuleFlags(cmd) + cmd.Flags().Bool(flagRuleDisabled, false, "create the rule disabled") + return cmd +} + +func runLifecycleCreate(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + in := lifecycleCreateInput{} + in.Name, _ = cmd.Flags().GetString(flagRuleName) + in.Prefix, _ = cmd.Flags().GetString(flagRulePrefix) + in.ExpirationDays, _ = expirationDaysFlag(cmd) + in.NoncurrentDays, _ = cmd.Flags().GetInt64(flagNoncurrentDays) + in.AbortMpuDays, _ = cmd.Flags().GetInt64(flagAbortMpuDays) + in.Disabled, _ = cmd.Flags().GetBool(flagRuleDisabled) + + // Validate the flags before touching the API so usage errors are cheap. + body, err := buildLifecycleCreateRequest(in) + if err != nil { + return printErr(err) + } + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + + if dryRun() { + if isHuman() { + fmt.Println(LifecycleMutation{Action: "create", Bucket: lc.uri(), Name: body.Data.Attributes.Name, DryRun: true}.HumanLine()) + return nil + } + render([]renderer.ResponseData{plannedLifecycleRule(body)}) + return nil + } + + created, err := lc.create(ctx, body) + if err != nil { + return printErr(err) + } + if isHuman() { + fmt.Println(LifecycleMutation{Action: "create", Bucket: lc.uri(), ID: ruleID(*created), Name: ruleName(*created)}.HumanLine()) + return nil + } + render([]renderer.ResponseData{LifecycleRule{LifecycleRuleData: *created}}) + return nil +} diff --git a/cmd/s3/lifecycle_create_test.go b/cmd/s3/lifecycle_create_test.go new file mode 100644 index 0000000..68bb259 --- /dev/null +++ b/cmd/s3/lifecycle_create_test.go @@ -0,0 +1,109 @@ +package s3 + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func TestBuildLifecycleCreateRequest(t *testing.T) { + body, err := buildLifecycleCreateRequest(lifecycleCreateInput{Prefix: "tmp/", ExpirationDays: 7}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := body.Data.Attributes + if body.Data.Type != operations.PostStorageBucketLifecycleRulesTypeLifecycleRules { + t.Errorf("type = %q", body.Data.Type) + } + if a.Name != "expire-7d-tmp" { + t.Errorf("default name = %q, want expire-7d-tmp", a.Name) + } + if a.ExpirationDays != 7 { + t.Errorf("expiration_days = %d", a.ExpirationDays) + } + if a.Prefix == nil || *a.Prefix != "tmp/" { + t.Errorf("prefix = %v", a.Prefix) + } + if a.Enabled == nil || !*a.Enabled { + t.Errorf("enabled should default to true, got %v", a.Enabled) + } + if a.NoncurrentDays != nil || a.AbortMpuDaysAfterInitiation != nil { + t.Errorf("optional days must be omitted when zero: %v %v", a.NoncurrentDays, a.AbortMpuDaysAfterInitiation) + } + + // Explicit name, all optional attributes and --disabled. + body, err = buildLifecycleCreateRequest(lifecycleCreateInput{Name: " abort-mpu ", ExpirationDays: 3650, NoncurrentDays: 30, AbortMpuDays: 2, Disabled: true}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a = body.Data.Attributes + if a.Name != "abort-mpu" { + t.Errorf("name = %q", a.Name) + } + if a.Prefix != nil { + t.Errorf("prefix should be omitted, got %q", *a.Prefix) + } + if a.NoncurrentDays == nil || *a.NoncurrentDays != 30 { + t.Errorf("noncurrent_days = %v", a.NoncurrentDays) + } + if a.AbortMpuDaysAfterInitiation == nil || *a.AbortMpuDaysAfterInitiation != 2 { + t.Errorf("abort_mpu_days_after_initiation = %v", a.AbortMpuDaysAfterInitiation) + } + if a.Enabled == nil || *a.Enabled { + t.Errorf("enabled should be false with --disabled, got %v", a.Enabled) + } + + // The wire format uses the API attribute names. + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"type":"lifecycle_rules"`, `"expiration_days":3650`, `"noncurrent_days":30`, `"abort_mpu_days_after_initiation":2`, `"enabled":false`} { + if !strings.Contains(string(raw), want) { + t.Errorf("payload %s lacks %s", raw, want) + } + } +} + +func TestBuildLifecycleCreateRequestValidation(t *testing.T) { + cases := []struct { + name string + in lifecycleCreateInput + want string + }{ + {"missing expiration", lifecycleCreateInput{AbortMpuDays: 2}, "the API requires --expiration-days on every rule"}, + {"negative expiration", lifecycleCreateInput{ExpirationDays: -1}, "the API requires --expiration-days on every rule"}, + {"negative noncurrent", lifecycleCreateInput{ExpirationDays: 1, NoncurrentDays: -1}, "--noncurrent-days"}, + {"negative abort", lifecycleCreateInput{ExpirationDays: 1, AbortMpuDays: -5}, "--abort-incomplete-multipart-days"}, + } + for _, c := range cases { + _, err := buildLifecycleCreateRequest(c.in) + if err == nil { + t.Errorf("%s: expected error", c.name) + continue + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("%s: exit %d, want %d", c.name, exitcode.Of(err), exitcode.Usage) + } + if !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: error %q should mention %q", c.name, err.Error(), c.want) + } + } +} + +func TestPlannedLifecycleRule(t *testing.T) { + body, err := buildLifecycleCreateRequest(lifecycleCreateInput{Prefix: "tmp/", ExpirationDays: 7, AbortMpuDays: 2}) + if err != nil { + t.Fatal(err) + } + row := plannedLifecycleRule(body).TableRow() + if row["id"].Value != "" { + t.Errorf("dry-run rule must have no id, got %q", row["id"].Value) + } + if row["name"].Value != "expire-7d-tmp" || row["expiration_days"].Value != "7d" || row["abort_mpu_days"].Value != "2d" || row["noncurrent_days"].Value != emptyCell { // unset days render the shared "-" placeholder + t.Errorf("unexpected planned row: %+v", row) + } +} diff --git a/cmd/s3/lifecycle_delete.go b/cmd/s3/lifecycle_delete.go new file mode 100644 index 0000000..2c6d922 --- /dev/null +++ b/cmd/s3/lifecycle_delete.go @@ -0,0 +1,133 @@ +package s3 + +import ( + "context" + "fmt" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +const flagRuleAll = "all" + +// newLifecycleDeleteCmd builds `lsh s3 lifecycle delete s3://bucket `. +func newLifecycleDeleteCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "delete []", + Aliases: []string{"rm", "remove"}, + Short: "Delete a lifecycle rule (or all of them with --all)", + Long: `Delete one lifecycle rule, addressed by id (lifecycle_...) or by name, or +every rule of the bucket with --all. + +Deletion asks for confirmation; pass --yes in scripts (a non-interactive +session without --yes fails with exit 7 instead of hanging).`, + Example: ` lsh s3 lifecycle delete s3://logs expire-7d-tmp + lsh s3 lifecycle rm s3://logs lifecycle_9x4kQ --yes + lsh s3 lifecycle delete s3://logs --all --yes`, + Args: cobra.RangeArgs(1, 2), + RunE: runLifecycleDelete, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + addYesFlag(cmd) + cmd.Flags().Bool(flagRuleAll, false, "delete every lifecycle rule of the bucket") + return cmd +} + +func runLifecycleDelete(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + all, _ := cmd.Flags().GetBool(flagRuleAll) + yes, _ := cmd.Flags().GetBool(flagYes) + switch { + case all && len(args) == 2: + return printErr(exitcode.Errorf(exitcode.Usage, "pass either a rule () or --all, not both")) + case !all && len(args) == 1: + return printErr(exitcode.Errorf(exitcode.Usage, "missing lifecycle rule: pass , or --all to delete every rule of the bucket")) + } + + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + + var targets []components.LifecycleRuleData + if all { + targets, err = lc.list(ctx) + if err != nil { + return printErr(err) + } + if len(targets) == 0 { + objectstorage.Hintf("no lifecycle rules on %s; nothing to delete", lc.uri()) + if !isHuman() { + render(nil) + } + return nil + } + } else { + rule, err := lc.get(ctx, args[1]) + if err != nil { + return printErr(err) + } + targets = []components.LifecycleRuleData{*rule} + } + + if dryRun() { + rows := make([]renderer.ResponseData, 0, len(targets)) + for _, r := range targets { + m := LifecycleMutation{Action: "delete", Bucket: lc.uri(), ID: ruleID(r), Name: ruleName(r), DryRun: true} + if isHuman() { + fmt.Println(m.HumanLine()) + } + rows = append(rows, m) + } + if !isHuman() { + render(rows) + } + return nil + } + + question := fmt.Sprintf("Delete lifecycle rule %s (%s) from %s?", ruleName(targets[0]), ruleID(targets[0]), lc.uri()) + if all { + question = fmt.Sprintf("Delete all %d lifecycle rules from %s?", len(targets), lc.uri()) + } + if err := objectstorage.ConfirmOrRefuse(cmd, yes, question); err != nil { + return printErr(err) + } + + rows := make([]renderer.ResponseData, 0, len(targets)) + var failed int + var firstErr error + for _, r := range targets { + m := LifecycleMutation{Action: "delete", Bucket: lc.uri(), ID: ruleID(r), Name: ruleName(r)} + if err := lc.delete(ctx, m.ID); err != nil { + failed++ + if firstErr == nil { + firstErr = err + } + m.Error = err.Error() + if isHuman() { + objectstorage.Warnf("%s", m.HumanLine()) + } + } else if isHuman() { + fmt.Println(m.HumanLine()) + } + rows = append(rows, m) + } + if !isHuman() { + render(rows) + } + switch { + case failed == 0: + return nil + case failed == len(targets) && len(targets) == 1: + return printErr(firstErr) + case failed == len(targets): + return printErr(exitcode.Errorf(exitcode.Of(firstErr), "none of the %d lifecycle rules could be deleted: %v", failed, firstErr)) + } + return printErr(exitcode.Errorf(exitcode.Partial, "%d of %d lifecycle rules could not be deleted", failed, len(targets))) +} diff --git a/cmd/s3/lifecycle_get.go b/cmd/s3/lifecycle_get.go new file mode 100644 index 0000000..fbe271e --- /dev/null +++ b/cmd/s3/lifecycle_get.go @@ -0,0 +1,43 @@ +package s3 + +import ( + "context" + + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +// newLifecycleGetCmd builds `lsh s3 lifecycle get s3://bucket `. +func newLifecycleGetCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "get ", + Aliases: []string{"show", "describe"}, + Short: "Show one lifecycle rule", + Long: `Show one lifecycle rule of a bucket, addressed by id (lifecycle_...) or by +name. A name shared by several rules is rejected (exit 2): use the id.`, + Example: ` lsh s3 lifecycle get s3://logs expire-7d-tmp + lsh s3 lifecycle get s3://logs lifecycle_9x4kQ -o json`, + Args: cobra.ExactArgs(2), + RunE: runLifecycleGet, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + return cmd +} + +func runLifecycleGet(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + rule, err := lc.get(ctx, args[1]) + if err != nil { + return printErr(err) + } + render([]renderer.ResponseData{LifecycleRule{LifecycleRuleData: *rule}}) + return nil +} diff --git a/cmd/s3/lifecycle_list.go b/cmd/s3/lifecycle_list.go new file mode 100644 index 0000000..67b7334 --- /dev/null +++ b/cmd/s3/lifecycle_list.go @@ -0,0 +1,49 @@ +package s3 + +import ( + "context" + + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/spf13/cobra" +) + +// newLifecycleListCmd builds `lsh s3 lifecycle list s3://bucket`. +func newLifecycleListCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "list ", + Aliases: []string{"ls"}, + Short: "List the lifecycle rules of a bucket", + Long: `List the lifecycle rules of a bucket. + +A bucket without rules prints an empty list and exits 0.`, + Example: ` lsh s3 lifecycle list s3://logs + lsh s3 lifecycle ls logs -o json`, + Args: cobra.ExactArgs(1), + RunE: runLifecycleList, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + return cmd +} + +func runLifecycleList(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + rules, err := lc.list(ctx) + if err != nil { + return printErr(err) + } + if len(rules) == 0 && isHuman() { + // Empty is a normal outcome (exit 0): say so on stderr and keep stdout + // clean instead of printing the renderer's "No results found" box. + objectstorage.Hintf("no lifecycle rules on %s", lc.uri()) + return nil + } + render(lifecycleRuleRows(rules)) + return nil +} diff --git a/cmd/s3/lifecycle_test.go b/cmd/s3/lifecycle_test.go new file mode 100644 index 0000000..658f94f --- /dev/null +++ b/cmd/s3/lifecycle_test.go @@ -0,0 +1,159 @@ +package s3 + +import ( + "strings" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func ptrS(s string) *string { return &s } +func ptrI(i int64) *int64 { return &i } +func ptrB(b bool) *bool { return &b } + +func rule(id, name string, exp *int64) components.LifecycleRuleData { + return components.LifecycleRuleData{ + ID: ptrS(id), + Attributes: &components.LifecycleRuleDataAttributes{ + Name: ptrS(name), + ExpirationDays: exp, + Enabled: ptrB(true), + }, + } +} + +func fixtureRules() []components.LifecycleRuleData { + return []components.LifecycleRuleData{ + rule("lifecycle_a", "expire-7d-tmp", ptrI(7)), + rule("lifecycle_b", "abort-mpu", ptrI(3650)), + rule("lifecycle_c", "dup", ptrI(1)), + rule("lifecycle_d", "dup", ptrI(2)), + rule("lifecycle_e", "lifecycle_a", ptrI(9)), // a name that looks like an id + } +} + +func TestFindLifecycleRule(t *testing.T) { + rules := fixtureRules() + + cases := []struct { + token string + wantID string + wantCode int + }{ + {"lifecycle_a", "lifecycle_a", 0}, // id wins over the rule named lifecycle_a + {"lifecycle_b", "lifecycle_b", 0}, + {"expire-7d-tmp", "lifecycle_a", 0}, + {" abort-mpu ", "lifecycle_b", 0}, + {"dup", "", exitcode.Usage}, + {"missing", "", exitcode.NotFound}, + {"", "", exitcode.Usage}, + } + for _, c := range cases { + got, err := findLifecycleRule(rules, c.token) + if c.wantCode != 0 { + if err == nil { + t.Errorf("findLifecycleRule(%q): expected error", c.token) + continue + } + if exitcode.Of(err) != c.wantCode { + t.Errorf("findLifecycleRule(%q): exit %d, want %d (%v)", c.token, exitcode.Of(err), c.wantCode, err) + } + if c.token == "dup" && (!strings.Contains(err.Error(), "lifecycle_c") || !strings.Contains(err.Error(), "lifecycle_d")) { + t.Errorf("ambiguous error should list the candidate ids, got %q", err.Error()) + } + continue + } + if err != nil { + t.Errorf("findLifecycleRule(%q): unexpected error %v", c.token, err) + continue + } + if ruleID(*got) != c.wantID { + t.Errorf("findLifecycleRule(%q) = %s, want %s", c.token, ruleID(*got), c.wantID) + } + } + + if _, err := findLifecycleRule(nil, "anything"); exitcode.Of(err) != exitcode.NotFound { + t.Errorf("empty rule list: exit %d, want %d", exitcode.Of(err), exitcode.NotFound) + } +} + +func TestDefaultLifecycleRuleName(t *testing.T) { + cases := []struct { + days int64 + prefix string + want string + }{ + {7, "", "expire-7d"}, + {7, "tmp/", "expire-7d-tmp"}, + {30, "Logs/2026/", "expire-30d-logs-2026"}, + {1, "a_b c", "expire-1d-a-b-c"}, + {365, "///", "expire-365d"}, + {14, "Ünïcode/ok", "expire-14d-n-code-ok"}, + } + for _, c := range cases { + if got := defaultLifecycleRuleName(c.days, c.prefix); got != c.want { + t.Errorf("defaultLifecycleRuleName(%d, %q) = %q, want %q", c.days, c.prefix, got, c.want) + } + } +} + +func TestLifecycleRuleTableRow(t *testing.T) { + r := LifecycleRule{LifecycleRuleData: components.LifecycleRuleData{ + ID: ptrS("lifecycle_9x4kQ"), + Attributes: &components.LifecycleRuleDataAttributes{ + Name: ptrS("expire-7d-tmp"), + Prefix: ptrS("tmp/"), + ExpirationDays: ptrI(7), + Enabled: ptrB(true), + }, + }} + row := r.TableRow() + want := map[string]string{ + "id": "lifecycle_9x4kQ", + "name": "expire-7d-tmp", + "prefix": "tmp/", + "expiration_days": "7d", + "noncurrent_days": "-", + "abort_mpu_days": "-", + "enabled": "true", + } + for k, v := range want { + if row[k].Value != v { + t.Errorf("row[%q] = %q, want %q", k, row[k].Value, v) + } + } + + // A rule with no attributes must not panic and renders the shared + // placeholder ("-", the same one every s3 table uses for empty cells). + empty := LifecycleRule{LifecycleRuleData: components.LifecycleRuleData{ID: ptrS("lifecycle_x")}} + for _, k := range []string{"enabled", "prefix", "expiration_days"} { + if got := empty.TableRow()[k].Value; got != emptyCell { + t.Errorf("empty rule %s = %q, want %q", k, got, emptyCell) + } + } + // An empty prefix string is also a placeholder, not a blank cell. + blank := LifecycleRule{LifecycleRuleData: components.LifecycleRuleData{Attributes: &components.LifecycleRuleDataAttributes{Prefix: ptrS("")}}} + if got := blank.TableRow()["prefix"].Value; got != emptyCell { + t.Errorf("blank prefix = %q, want %q", got, emptyCell) + } +} + +func TestLifecycleMutationHumanLine(t *testing.T) { + cases := []struct { + m LifecycleMutation + want string + }{ + {LifecycleMutation{Action: "create", Bucket: "s3://logs", ID: "lifecycle_1", Name: "expire-7d"}, "lifecycle: created lifecycle_1 (expire-7d)"}, + {LifecycleMutation{Action: "update", Bucket: "s3://logs", ID: "lifecycle_1", Name: "expire-7d"}, "lifecycle: updated lifecycle_1 (expire-7d)"}, + {LifecycleMutation{Action: "delete", Bucket: "s3://logs", ID: "lifecycle_1", Name: "expire-7d"}, "lifecycle: deleted lifecycle_1 (expire-7d)"}, + {LifecycleMutation{Action: "create", Bucket: "s3://logs", Name: "expire-7d", DryRun: true}, "(dryrun) lifecycle: create expire-7d on s3://logs"}, + {LifecycleMutation{Action: "delete", Bucket: "s3://logs", ID: "lifecycle_1", Name: "n", DryRun: true}, "(dryrun) lifecycle: delete lifecycle_1 (n) on s3://logs"}, + {LifecycleMutation{Action: "delete", Bucket: "s3://logs", ID: "lifecycle_1", Name: "n", Error: "boom"}, "lifecycle: failed to delete lifecycle_1 (n): boom"}, + } + for _, c := range cases { + if got := c.m.HumanLine(); got != c.want { + t.Errorf("HumanLine() = %q, want %q", got, c.want) + } + } +} diff --git a/cmd/s3/lifecycle_update.go b/cmd/s3/lifecycle_update.go new file mode 100644 index 0000000..832e246 --- /dev/null +++ b/cmd/s3/lifecycle_update.go @@ -0,0 +1,210 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +const ( + flagRuleEnable = "enable" + flagRuleDisable = "disable" +) + +// lifecycleUpdateInput holds only the attributes the user asked to change +// (nil = untouched), so the PATCH carries just those. +type lifecycleUpdateInput struct { + Name *string + Prefix *string + ExpirationDays *int64 + NoncurrentDays *int64 + AbortMpuDays *int64 + Enabled *bool +} + +func (in lifecycleUpdateInput) empty() bool { + return in.Name == nil && in.Prefix == nil && in.ExpirationDays == nil && + in.NoncurrentDays == nil && in.AbortMpuDays == nil && in.Enabled == nil +} + +// buildLifecycleUpdateRequest builds the PATCH body from the current rule and +// the changed attributes. The API requires Name on every PATCH, so the current +// name is sent when --name was not given. +func buildLifecycleUpdateRequest(current components.LifecycleRuleData, in lifecycleUpdateInput) (operations.PatchStorageBucketLifecycleRuleRequestBody, error) { + var body operations.PatchStorageBucketLifecycleRuleRequestBody + if in.empty() { + return body, exitcode.Errorf(exitcode.Usage, "nothing to update: pass at least one of --%s, --%s, --%s, --%s, --%s, --%s or --%s", + flagRuleName, flagRulePrefix, flagExpirationDays, flagNoncurrentDays, flagAbortMpuDays, flagRuleEnable, flagRuleDisable) + } + name := ruleName(current) + if in.Name != nil { + name = strings.TrimSpace(*in.Name) + } + if name == "" { + return body, exitcode.Errorf(exitcode.Usage, "the rule needs a name: pass --%s", flagRuleName) + } + if in.ExpirationDays != nil && *in.ExpirationDays <= 0 { + return body, exitcode.Errorf(exitcode.Usage, "the API requires --expiration-days on every rule (a positive number of days after which objects are deleted)") + } + if in.NoncurrentDays != nil && *in.NoncurrentDays < 0 { + return body, exitcode.Errorf(exitcode.Usage, "--%s must be a positive number of days", flagNoncurrentDays) + } + if in.AbortMpuDays != nil && *in.AbortMpuDays < 0 { + return body, exitcode.Errorf(exitcode.Usage, "--%s must be a positive number of days", flagAbortMpuDays) + } + body.Data = operations.PatchStorageBucketLifecycleRuleData{ + Type: operations.PatchStorageBucketLifecycleRuleTypeLifecycleRules, + Attributes: operations.PatchStorageBucketLifecycleRuleAttributes{ + Name: name, + Enabled: in.Enabled, + Prefix: in.Prefix, + ExpirationDays: in.ExpirationDays, + NoncurrentDays: in.NoncurrentDays, + AbortMpuDaysAfterInitiation: in.AbortMpuDays, + }, + } + return body, nil +} + +// plannedLifecycleUpdate merges the PATCH into the current rule for --dry-run. +func plannedLifecycleUpdate(current components.LifecycleRuleData, body operations.PatchStorageBucketLifecycleRuleRequestBody) LifecycleRule { + out := current + attrs := components.LifecycleRuleDataAttributes{} + if current.Attributes != nil { + attrs = *current.Attributes + } + a := body.Data.Attributes + name := a.Name + attrs.Name = &name + if a.Enabled != nil { + attrs.Enabled = a.Enabled + } + if a.Prefix != nil { + attrs.Prefix = a.Prefix + } + if a.ExpirationDays != nil { + attrs.ExpirationDays = a.ExpirationDays + } + if a.NoncurrentDays != nil { + attrs.NoncurrentDays = a.NoncurrentDays + } + if a.AbortMpuDaysAfterInitiation != nil { + attrs.AbortMpuDaysAfterInitiation = a.AbortMpuDaysAfterInitiation + } + out.Attributes = &attrs + return LifecycleRule{LifecycleRuleData: out} +} + +// newLifecycleUpdateCmd builds `lsh s3 lifecycle update s3://bucket `. +func newLifecycleUpdateCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "update ", + Aliases: []string{"edit", "set"}, + Short: "Update a lifecycle rule", + Long: `Update a lifecycle rule, addressed by id (lifecycle_...) or by name. + +Only the flags you pass are changed; --enable/--disable toggle the rule.`, + Example: ` lsh s3 lifecycle update s3://logs expire-7d-tmp --expiration-days 14 + lsh s3 lifecycle update s3://logs lifecycle_9x4kQ --disable + lsh s3 lifecycle update s3://logs expire-7d-tmp --name expire-14d-tmp --prefix tmp/`, + Args: cobra.ExactArgs(2), + RunE: runLifecycleUpdate, + }) + addProjectFlag(cmd, true, "project of the bucket (disambiguates names shared across projects)") + addBucketFilterFlags(cmd) + addLifecycleRuleFlags(cmd) + cmd.Flags().Bool(flagRuleEnable, false, "enable the rule") + cmd.Flags().Bool(flagRuleDisable, false, "disable the rule") + return cmd +} + +// lifecycleUpdateInputFromFlags collects only the flags that were set. +func lifecycleUpdateInputFromFlags(cmd *cobra.Command) lifecycleUpdateInput { + in := lifecycleUpdateInput{} + f := cmd.Flags() + if f.Changed(flagRuleName) { + v, _ := f.GetString(flagRuleName) + in.Name = &v + } + if f.Changed(flagRulePrefix) { + v, _ := f.GetString(flagRulePrefix) + in.Prefix = &v + } + if v, ok := expirationDaysFlag(cmd); ok { + in.ExpirationDays = &v + } + if f.Changed(flagNoncurrentDays) { + v, _ := f.GetInt64(flagNoncurrentDays) + in.NoncurrentDays = &v + } + if f.Changed(flagAbortMpuDays) { + v, _ := f.GetInt64(flagAbortMpuDays) + in.AbortMpuDays = &v + } + if v, _ := f.GetBool(flagRuleEnable); v { + t := true + in.Enabled = &t + } + if v, _ := f.GetBool(flagRuleDisable); v { + fl := false + in.Enabled = &fl + } + return in +} + +func runLifecycleUpdate(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + if en, _ := cmd.Flags().GetBool(flagRuleEnable); en { + if dis, _ := cmd.Flags().GetBool(flagRuleDisable); dis { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s and --%s are mutually exclusive", flagRuleEnable, flagRuleDisable)) + } + } + in := lifecycleUpdateInputFromFlags(cmd) + if in.empty() { + // Fail before any API call; the builder produces the same message. + _, err := buildLifecycleUpdateRequest(components.LifecycleRuleData{}, in) + return printErr(err) + } + lc, err := openLifecycleAPI(ctx, cmd, args[0]) + if err != nil { + return printErr(err) + } + current, err := lc.get(ctx, args[1]) + if err != nil { + return printErr(err) + } + body, err := buildLifecycleUpdateRequest(*current, in) + if err != nil { + return printErr(err) + } + id := ruleID(*current) + + if dryRun() { + if isHuman() { + fmt.Println(LifecycleMutation{Action: "update", Bucket: lc.uri(), ID: id, Name: body.Data.Attributes.Name, DryRun: true}.HumanLine()) + return nil + } + render([]renderer.ResponseData{plannedLifecycleUpdate(*current, body)}) + return nil + } + + updated, err := lc.update(ctx, id, body) + if err != nil { + return printErr(err) + } + if isHuman() { + fmt.Println(LifecycleMutation{Action: "update", Bucket: lc.uri(), ID: ruleID(*updated), Name: ruleName(*updated)}.HumanLine()) + return nil + } + render([]renderer.ResponseData{LifecycleRule{LifecycleRuleData: *updated}}) + return nil +} diff --git a/cmd/s3/lifecycle_update_test.go b/cmd/s3/lifecycle_update_test.go new file mode 100644 index 0000000..fa26742 --- /dev/null +++ b/cmd/s3/lifecycle_update_test.go @@ -0,0 +1,112 @@ +package s3 + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func TestBuildLifecycleUpdateRequestPartial(t *testing.T) { + current := components.LifecycleRuleData{ + ID: ptrS("lifecycle_a"), + Attributes: &components.LifecycleRuleDataAttributes{ + Name: ptrS("expire-7d-tmp"), + Prefix: ptrS("tmp/"), + ExpirationDays: ptrI(7), + Enabled: ptrB(true), + }, + } + + // Only --expiration-days changed: the name is filled from the rule and + // nothing else is sent. + body, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{ExpirationDays: ptrI(14)}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := body.Data.Attributes + if a.Name != "expire-7d-tmp" { + t.Errorf("name = %q, want the current name", a.Name) + } + if a.ExpirationDays == nil || *a.ExpirationDays != 14 { + t.Errorf("expiration_days = %v", a.ExpirationDays) + } + if a.Prefix != nil || a.Enabled != nil || a.NoncurrentDays != nil || a.AbortMpuDaysAfterInitiation != nil { + t.Errorf("untouched attributes must stay nil: %+v", a) + } + raw, _ := json.Marshal(body) + if strings.Contains(string(raw), `"prefix"`) || strings.Contains(string(raw), `"enabled"`) { + t.Errorf("payload should only carry name and expiration_days: %s", raw) + } + + // --disable plus a new name and an empty prefix (clears the filter). + body, err = buildLifecycleUpdateRequest(current, lifecycleUpdateInput{Name: ptrS(" new-name "), Prefix: ptrS(""), Enabled: ptrB(false)}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a = body.Data.Attributes + if a.Name != "new-name" { + t.Errorf("name = %q", a.Name) + } + if a.Enabled == nil || *a.Enabled { + t.Errorf("enabled = %v, want false", a.Enabled) + } + if a.Prefix == nil || *a.Prefix != "" { + t.Errorf("prefix = %v, want empty string", a.Prefix) + } + raw, _ = json.Marshal(body) + if !strings.Contains(string(raw), `"prefix":""`) || !strings.Contains(string(raw), `"enabled":false`) { + t.Errorf("payload should carry prefix and enabled: %s", raw) + } +} + +func TestBuildLifecycleUpdateRequestValidation(t *testing.T) { + current := components.LifecycleRuleData{ID: ptrS("lifecycle_a"), Attributes: &components.LifecycleRuleDataAttributes{Name: ptrS("n")}} + + if _, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{}); err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "nothing to update") { + t.Errorf("empty input: got %v", err) + } + if _, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{ExpirationDays: ptrI(0)}); err == nil || !strings.Contains(err.Error(), "the API requires --expiration-days on every rule") { + t.Errorf("zero expiration: got %v", err) + } + if _, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{NoncurrentDays: ptrI(-1)}); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("negative noncurrent: got %v", err) + } + // A rule without a name needs --name. + nameless := components.LifecycleRuleData{ID: ptrS("lifecycle_b"), Attributes: &components.LifecycleRuleDataAttributes{}} + if _, err := buildLifecycleUpdateRequest(nameless, lifecycleUpdateInput{Enabled: ptrB(true)}); err == nil || !strings.Contains(err.Error(), "--name") { + t.Errorf("nameless rule: got %v", err) + } + if body, err := buildLifecycleUpdateRequest(nameless, lifecycleUpdateInput{Name: ptrS("fixed"), Enabled: ptrB(true)}); err != nil || body.Data.Attributes.Name != "fixed" { + t.Errorf("nameless rule with --name: %v %+v", err, body) + } +} + +func TestPlannedLifecycleUpdate(t *testing.T) { + current := components.LifecycleRuleData{ + ID: ptrS("lifecycle_a"), + Attributes: &components.LifecycleRuleDataAttributes{ + Name: ptrS("expire-7d-tmp"), + Prefix: ptrS("tmp/"), + ExpirationDays: ptrI(7), + Enabled: ptrB(true), + }, + } + body, err := buildLifecycleUpdateRequest(current, lifecycleUpdateInput{ExpirationDays: ptrI(14), Enabled: ptrB(false)}) + if err != nil { + t.Fatal(err) + } + row := plannedLifecycleUpdate(current, body).TableRow() + if row["id"].Value != "lifecycle_a" || row["name"].Value != "expire-7d-tmp" || row["prefix"].Value != "tmp/" { + t.Errorf("planned update lost current values: %+v", row) + } + if row["expiration_days"].Value != "14d" || row["enabled"].Value != "false" { + t.Errorf("planned update did not apply the patch: %+v", row) + } + // The original must not be mutated. + if *current.Attributes.ExpirationDays != 7 || !*current.Attributes.Enabled { + t.Errorf("plannedLifecycleUpdate mutated the current rule") + } +} diff --git a/cmd/s3/ls.go b/cmd/s3/ls.go new file mode 100644 index 0000000..b6e0323 --- /dev/null +++ b/cmd/s3/ls.go @@ -0,0 +1,454 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/cli" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/pagination" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" +) + +// Local flag names of `ls`. +const ( + flagRecursive = "recursive" + flagHumanReadable = "human-readable" + flagSummarize = "summarize" + flagVersions = "versions" + flagStartingToken = "starting-token" + flagStream = "stream" + flagAllProjects = "all-projects" + flagStorageClass = "storage-class" +) + +// maxListPageSize is the largest max-keys S3 accepts per request. +const maxListPageSize = 1000 + +// NewLsCmd builds `lsh s3 list [s3://bucket[/prefix]]`. +func NewLsCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "list [s3://bucket[/prefix]]", + Aliases: []string{"ls"}, + GroupID: groupBuckets, + Short: "List buckets, or the objects under a prefix (alias: ls)", + Long: `List buckets, or the objects under a prefix. + +Without an argument, lists buckets through the Latitude API. Pick a project +with --project (or LSH_PROJECT), list every project with --all-projects, or — +in a terminal — choose from the project picker (which includes "All projects"). +--storage-class and --site narrow the listing. With an +argument, lists the objects and common prefixes under s3://[/] +straight from the bucket's S3 endpoint: the prefix is +matched literally ('s3://b/2026' matches every key starting with "2026"), +sub-prefixes are shown as PRE entries unless --recursive is given, and an empty +listing prints nothing and exits 0. + +Pagination follows the global flags: --page-size (default and maximum 1000 +keys per request), --max-items to cap the total number of entries and +--no-paginate to stop after one page; the token to resume from is printed on +stderr and accepted by --starting-token. + +Structured output (-o json|yaml|csv) emits one row per object or prefix; +--stream with -o json prints one JSON object per line as the listing arrives +instead of buffering everything.`, + Example: ` lsh s3 list + lsh s3 list --project my-project --storage-class high_performance + lsh s3 list s3://backups/2026/09/ + lsh s3 list s3://backups --recursive --human-readable --summarize + lsh s3 list s3://backups/logs/ --versions + lsh s3 list s3://backups --recursive -o json --stream | jq -r .key + lsh s3 list s3://backups --no-paginate --page-size 100 + lsh s3 list s3://backups --starting-token `, + Args: cobra.MaximumNArgs(1), + RunE: runLs, + }) + + addProjectFlag(cmd, true, "only list buckets of this project (ID or slug)") + cmd.Flags().Bool(flagAllProjects, false, "list buckets across every project (skip the project picker)") + cmd.Flags().StringP(flagStorageClass, "c", "", "only list buckets of this storage class (standard or high_performance)") + cmd.Flags().String(flagSite, "", "only list buckets in this Latitude site (e.g. DAL, TYO4)") + cmd.Flags().BoolP(flagRecursive, "r", false, "list every object under the prefix instead of stopping at the next '/'") + cmd.Flags().BoolP(flagHumanReadable, "H", false, "print sizes in human readable units (KiB, MiB…)") + cmd.Flags().Bool(flagSummarize, false, "append the total number of objects and their size") + cmd.Flags().Bool(flagVersions, false, "list every object version (versioned buckets)") + cmd.Flags().String(flagStartingToken, "", "resume the listing from the token printed by a previous --no-paginate or --max-items run") + cmd.Flags().Bool(flagStream, false, "with -o json, print one JSON object per line as the listing arrives (NDJSON)") + unsupportedAWSBoolFlags(cmd, map[string]string{ + "request-payer": "there is no requester-pays billing on Latitude object storage", + }) + rejectRegionFlag(cmd) + + return cmd +} + +func runLs(cmd *cobra.Command, args []string) error { + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + if len(args) == 0 { + return runLsBuckets(ctx, cmd) + } + return runLsObjects(ctx, cmd, args[0]) +} + +// --------------------------------------------------------------------------- +// Buckets +// --------------------------------------------------------------------------- + +func runLsBuckets(ctx context.Context, cmd *cobra.Command) error { + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "listing buckets needs the Latitude API and cannot be combined with --endpoint-url; pass s3:// to list objects on that endpoint")) + } + classFlag, _ := cmd.Flags().GetString(flagStorageClass) + // Shared alias table (standard|std|wasabi, high_performance|high-performance| + // hp|high|vast); an empty value means "no filter". + class, err := objectstorage.ParseStorageClass(classFlag) + if err != nil { + return printErr(fmt.Errorf("--%s: %w", flagStorageClass, err)) + } + + site, _ := cmd.Flags().GetString(flagSite) + + // Pick the project the same way as the rest of the CLI: --project / + // LSH_PROJECT, or --all-projects, otherwise an interactive picker with an + // "All projects" entry (and a usage error in a non-interactive session). + project, _, err := cli.PickProjectForList(cmd) + if err != nil { + return printErr(err) + } + + r := newResolver(cmd) + r.Project = project // "" when the user chose all projects + data, err := r.ListBuckets(ctx) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + data = filterBucketsByClass(data, class) + + // The SDK model drops the site slug, so fetch it best-effort for the Site + // column (and for the --site filter), scoped to the same --project filter + // as the listing. + sites, siteErr := objectstorage.RawBucketSitesForProject(ctx, "", project) + if siteErr != nil { + sites = nil + } + if s := strings.TrimSpace(site); s != "" { + // The filter depends entirely on that lookup (the SDK model drops the + // slug), so a failure would silently drop every bucket and exit 0. + if siteErr != nil { + return printErr(exitcode.Errorf(exitcode.Generic, "--site %s needs the sites of the listed buckets, which could not be fetched: %v", s, siteErr)) + } + data = filterBucketsBySite(data, s, sites) + } + + // Render through the shared renderer, exactly like every other `list` + // command (servers, volume, filesystems…): the default table, or + // -o json|yaml|csv|text with --query. + render(BucketRows(data, sites)) + return nil +} + +// filterBucketsBySite keeps the buckets in one Latitude site ("" keeps all). +// The site slug comes from the SDK model when present, otherwise from the +// site map fetched alongside the listing. +func filterBucketsBySite(data []components.ObjectStorageData, site string, sites map[string]string) []components.ObjectStorageData { + out := make([]components.ObjectStorageData, 0, len(data)) + for _, d := range data { + s := objectstorage.BucketFromData(d).Site + if s == "" && d.ID != nil { + s = sites[*d.ID] + } + if strings.EqualFold(s, site) { + out = append(out, d) + } + } + return out +} + +// filterBucketsByClass keeps the buckets of one storage class ("" keeps all). +func filterBucketsByClass(data []components.ObjectStorageData, class string) []components.ObjectStorageData { + if class == "" { + return data + } + out := make([]components.ObjectStorageData, 0, len(data)) + for _, d := range data { + if strings.EqualFold(objectstorage.BucketFromData(d).StorageClass, class) { + out = append(out, d) + } + } + return out +} + +// --------------------------------------------------------------------------- +// Objects +// --------------------------------------------------------------------------- + +// listOptions controls listObjects. +type listOptions struct { + // Prefix is matched literally (no trailing slash is added). + Prefix string + // Recursive drops the "/" delimiter. + Recursive bool + // Versions lists object versions instead of current objects. + Versions bool + // PageSize is the max-keys sent per request (1..1000; 0 means 1000). + PageSize int + // MaxItems caps the total number of entries (0 = unlimited). + MaxItems int64 + // NoPaginate stops after the first page. + NoPaginate bool + // StartingToken resumes a ListObjectsV2 listing from a continuation token. + StartingToken string + // OnEntry, when set, receives every entry as it arrives instead of it + // being accumulated in listResult.Entries (streaming output). + OnEntry func(renderer.ResponseData) +} + +// listResult is the outcome of listObjects. +type listResult struct { + // Entries holds prefixes and objects in listing order (empty when + // listOptions.OnEntry streamed them). + Entries []renderer.ResponseData + // Count and Bytes summarize the listed objects (prefixes excluded). + Count int + Bytes int64 + // Total is the number of entries emitted (prefixes included). + Total int64 + // NextToken is the continuation token to resume from when the listing + // stopped before the end ("" when exhausted). + NextToken string +} + +func (r *listResult) emit(opts listOptions, e renderer.ResponseData) { + r.Total++ + if o, ok := e.(objectstorage.Object); ok { + r.Count++ + r.Bytes += o.Size + } + if opts.OnEntry != nil { + opts.OnEntry(e) + return + } + r.Entries = append(r.Entries, e) +} + +// listObjects lists the objects (or versions) of bucketName under the prefix, +// paginating with continuation tokens and honoring the page-size, max-items, +// no-paginate and starting-token controls. It only issues GET requests. +func listObjects(ctx context.Context, client *minio.Client, bucketName string, opts listOptions) (listResult, error) { + if opts.PageSize <= 0 || opts.PageSize > maxListPageSize { + opts.PageSize = maxListPageSize + } + if opts.Versions { + return listVersions(ctx, client, bucketName, opts) + } + + var res listResult + delimiter := "/" + if opts.Recursive { + delimiter = "" + } + core := minio.Core{Client: client} + token := opts.StartingToken + for { + if err := ctx.Err(); err != nil { + return res, err + } + maxKeys := opts.PageSize + if opts.MaxItems > 0 { + remaining := opts.MaxItems - res.Total + if remaining <= 0 { + return res, nil + } + if remaining < int64(maxKeys) { + maxKeys = int(remaining) + } + } + page, err := listPage(ctx, core, bucketName, opts.Prefix, token, delimiter, maxKeys) + if err != nil { + return res, err + } + for _, cp := range page.CommonPrefixes { + res.emit(opts, objectstorage.NewPrefix(cp.Prefix)) + } + for _, info := range page.Contents { + res.emit(opts, objectstorage.ObjectFromInfo(info, false)) + } + next := page.NextContinuationToken + if !page.IsTruncated || next == "" { + res.NextToken = "" + return res, nil + } + res.NextToken = next + if opts.NoPaginate { + return res, nil + } + if opts.MaxItems > 0 && res.Total >= opts.MaxItems { + return res, nil + } + token = next + } +} + +// listPage issues one ListObjectsV2 request. minio.Core.ListObjectsV2 takes no +// context (it is the only entry point that exposes continuation tokens, which +// --starting-token / --no-paginate need), so the call runs in a goroutine and +// the caller stops waiting as soon as ctx is cancelled: Ctrl-C then returns +// ctx.Err(), which Humanize maps to exit 130. An abandoned request finishes in +// the background and its result is dropped (the channel is buffered). +func listPage(ctx context.Context, core minio.Core, bucketName, prefix, token, delimiter string, maxKeys int) (minio.ListBucketV2Result, error) { + type pageResult struct { + page minio.ListBucketV2Result + err error + } + done := make(chan pageResult, 1) + go func() { + page, err := core.ListObjectsV2(bucketName, prefix, "", token, delimiter, maxKeys) + done <- pageResult{page: page, err: err} + }() + select { + case <-ctx.Done(): + return minio.ListBucketV2Result{}, ctx.Err() + case r := <-done: + return r.page, r.err + } +} + +// listVersions lists object versions (ListObjectVersions) through the minio +// iterator. Versions listings have no continuation token to expose, so +// --no-paginate and --starting-token do not apply; --max-items does. +func listVersions(ctx context.Context, client *minio.Client, bucketName string, opts listOptions) (listResult, error) { + var res listResult + ctx, cancel := context.WithCancel(ctx) + defer cancel() + ch := client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{ + Prefix: opts.Prefix, + Recursive: opts.Recursive, + WithVersions: true, + MaxKeys: opts.PageSize, + }) + for info := range ch { + if info.Err != nil { + return res, info.Err + } + if opts.MaxItems > 0 && res.Total >= opts.MaxItems { + cancel() + // Drain so the producer goroutine exits. + for range ch { + } + break + } + // minio reports common prefixes of a non-recursive listing as entries + // with a key ending in "/" and no version. + if !opts.Recursive && strings.HasSuffix(info.Key, "/") && info.VersionID == "" && info.Size == 0 { + res.emit(opts, objectstorage.NewPrefix(info.Key)) + continue + } + res.emit(opts, objectstorage.ObjectFromInfo(info, true)) + } + return res, nil +} + +// streamJSON writes one entry as a single JSON line (NDJSON). +func streamJSON(w io.Writer, e renderer.ResponseData) error { + b, err := json.Marshal(e) + if err != nil { + return err + } + _, err = fmt.Fprintln(w, string(b)) + return err +} + +// resolvePageSize returns the max-keys per request: --page-size when given +// explicitly (capped at 1000), else 1000. +func resolvePageSize(cmd *cobra.Command, resolved pagination.Options) int { + if !cmd.Flags().Changed("page-size") { + return maxListPageSize + } + if resolved.PageSize <= 0 || resolved.PageSize > maxListPageSize { + return maxListPageSize + } + return int(resolved.PageSize) +} + +func runLsObjects(ctx context.Context, cmd *cobra.Command, arg string) error { + ref, err := objectstorage.ParseRemote(arg) + if err != nil { + return printErr(err) + } + recursive, _ := cmd.Flags().GetBool(flagRecursive) + human, _ := cmd.Flags().GetBool(flagHumanReadable) + summarize, _ := cmd.Flags().GetBool(flagSummarize) + versions, _ := cmd.Flags().GetBool(flagVersions) + stream, _ := cmd.Flags().GetBool(flagStream) + startingToken, _ := cmd.Flags().GetString(flagStartingToken) + + pg := pagination.Resolve() + opts := listOptions{ + Prefix: ref.Key, + Recursive: recursive, + Versions: versions, + PageSize: resolvePageSize(cmd, pg), + MaxItems: pg.MaxItems, + NoPaginate: pg.NoPaginate, + StartingToken: startingToken, + } + if versions && (pg.NoPaginate || startingToken != "") { + objectstorage.Warnf("--no-paginate and --starting-token do not apply to --versions listings; ignoring them") + opts.NoPaginate = false + opts.StartingToken = "" + } + + format := renderer.ResolveFormat() + if stream && format != renderer.FormatJSON { + objectstorage.Warnf("--stream only applies to -o json; ignoring it") + stream = false + } + + if dryRun() { + objectstorage.Hintf("(dryrun) ls is read-only; listing %s", ref) + } + + sess, err := openBucket(ctx, cmd, ref.Bucket, false) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + + // Objects render through the shared renderer like every other list command + // (default table, or -o json|yaml|csv|text with --query). --stream keeps + // the NDJSON fast path for -o json over very large listings. + if stream { + opts.OnEntry = func(e renderer.ResponseData) { + if err := streamJSON(os.Stdout, e); err != nil { + objectstorage.Warnf("could not write entry: %v", err) + } + } + } + + res, err := listObjects(ctx, sess.Client, sess.Bucket.BucketName, opts) + if err != nil { + return printErr(sess.humanize(err)) + } + + if !stream { + render(res.Entries) + if summarize && renderer.ResolveFormat() == renderer.FormatTable { + fmt.Print(objectstorage.LsSummary(res.Count, res.Bytes, human)) + fmt.Println() + } + } + + if res.NextToken != "" { + objectstorage.Hintf("Next token: %s (use --starting-token)", res.NextToken) + } + return nil +} diff --git a/cmd/s3/ls_test.go b/cmd/s3/ls_test.go new file mode 100644 index 0000000..9d20f3d --- /dev/null +++ b/cmd/s3/ls_test.go @@ -0,0 +1,485 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" + "github.com/latitudesh/lsh/internal/renderer" +) + +const lsTestBucket = "backups-7f3a" + +// newLsTestSession builds a Session against the fake server. +func newLsTestSession(t *testing.T, srv *s3test.Server, bucket string) *Session { + t.Helper() + b := &objectstorage.Bucket{ + ID: "bkt_1", + Name: "backups", + BucketName: bucket, + Endpoint: srv.URL(), + StorageClass: "standard", + SigningRegion: "us-east-1", + } + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +// seedLsBucket fills the bucket with objects under two prefixes plus one at +// the root and forces two-key pages so continuation tokens are exercised. +func seedLsBucket(t *testing.T) (*s3test.Server, *Session) { + t.Helper() + srv := s3test.New() + t.Cleanup(srv.Close) + srv.MaxListKeys = 2 + srv.CreateBucket(lsTestBucket) + srv.AddObject(lsTestBucket, "2026/09/dump.sql", make([]byte, 1258291), "application/sql") + srv.AddObject(lsTestBucket, "2026/09/notes.txt", []byte("notes"), "text/plain") + srv.AddObject(lsTestBucket, "logs/app.log", []byte("log line"), "text/plain") + srv.AddObject(lsTestBucket, "readme.md", []byte("# hi"), "text/markdown") + srv.ResetRequests() + return srv, newLsTestSession(t, srv, lsTestBucket) +} + +func entryNames(entries []renderer.ResponseData) []string { + out := make([]string, 0, len(entries)) + for _, e := range entries { + switch v := e.(type) { + case objectstorage.Prefix: + out = append(out, "PRE "+v.Prefix) + case objectstorage.Object: + out = append(out, v.Key) + } + } + return out +} + +func assertNoWrites(t *testing.T, srv *s3test.Server) { + t.Helper() + if w := srv.WriteRequests(); len(w) != 0 { + t.Fatalf("ls issued %d write requests: %+v", len(w), w) + } +} + +func listRequests(srv *s3test.Server) []s3test.Request { + var out []s3test.Request + for _, r := range srv.Requests() { + if r.Method == http.MethodGet && r.Query.Get("list-type") == "2" { + out = append(out, r) + } + } + return out +} + +func TestListObjectsDelimiter(t *testing.T) { + srv, sess := seedLsBucket(t) + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + got := strings.Join(entryNames(res.Entries), ",") + want := "PRE 2026/,PRE logs/,readme.md" + if got != want { + t.Fatalf("entries = %q, want %q", got, want) + } + if res.NextToken != "" { + t.Fatalf("NextToken = %q, want empty (listing exhausted)", res.NextToken) + } + if res.Count != 1 || res.Bytes != 4 { + t.Fatalf("summary = %d objects / %d bytes, want 1 / 4", res.Count, res.Bytes) + } + reqs := listRequests(srv) + if len(reqs) < 2 { + t.Fatalf("expected the listing to paginate with tokens, got %d list requests", len(reqs)) + } + if reqs[1].Query.Get("continuation-token") == "" { + t.Fatalf("second page did not carry a continuation token: %v", reqs[1].Query) + } + for _, r := range reqs { + if r.Query.Get("delimiter") != "/" { + t.Fatalf("delimiter = %q, want /", r.Query.Get("delimiter")) + } + if r.Query.Get("max-keys") != "1000" { + t.Fatalf("max-keys = %q, want 1000 by default", r.Query.Get("max-keys")) + } + } + assertNoWrites(t, srv) +} + +func TestListObjectsRecursive(t *testing.T) { + srv, sess := seedLsBucket(t) + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + got := strings.Join(entryNames(res.Entries), ",") + want := "2026/09/dump.sql,2026/09/notes.txt,logs/app.log,readme.md" + if got != want { + t.Fatalf("entries = %q, want %q", got, want) + } + if res.Count != 4 || res.Bytes != 1258291+5+8+4 { + t.Fatalf("summary = %d objects / %d bytes", res.Count, res.Bytes) + } + for _, r := range listRequests(srv) { + if r.Query.Get("delimiter") != "" { + t.Fatalf("recursive listing sent delimiter %q", r.Query.Get("delimiter")) + } + } + assertNoWrites(t, srv) +} + +func TestListObjectsLiteralPrefix(t *testing.T) { + _, sess := seedLsBucket(t) + // No trailing slash is added: "2026" matches keys starting with "2026". + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Prefix: "2026"}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if got := strings.Join(entryNames(res.Entries), ","); got != "PRE 2026/" { + t.Fatalf("entries = %q, want PRE 2026/", got) + } + res, err = listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Prefix: "2026/09/"}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if got := strings.Join(entryNames(res.Entries), ","); got != "2026/09/dump.sql,2026/09/notes.txt" { + t.Fatalf("entries = %q", got) + } +} + +func TestListObjectsMaxItemsAndStartingToken(t *testing.T) { + srv, sess := seedLsBucket(t) + first, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true, MaxItems: 3}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if len(first.Entries) != 3 { + t.Fatalf("got %d entries, want 3 (max-items)", len(first.Entries)) + } + if first.NextToken == "" { + t.Fatalf("expected a NextToken when max-items stops early") + } + // max-items never over-fetches: the last request asks only for what is left. + reqs := listRequests(srv) + if last := reqs[len(reqs)-1]; last.Query.Get("max-keys") != "1" { + t.Fatalf("last max-keys = %q, want 1", last.Query.Get("max-keys")) + } + + rest, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true, StartingToken: first.NextToken}) + if err != nil { + t.Fatalf("listObjects (resume): %v", err) + } + if got := strings.Join(entryNames(rest.Entries), ","); got != "readme.md" { + t.Fatalf("resumed entries = %q, want readme.md", got) + } + if rest.NextToken != "" { + t.Fatalf("NextToken after the last page = %q", rest.NextToken) + } + assertNoWrites(t, srv) +} + +func TestListObjectsNoPaginate(t *testing.T) { + srv, sess := seedLsBucket(t) + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true, NoPaginate: true, PageSize: 2}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if len(res.Entries) != 2 { + t.Fatalf("got %d entries, want one page of 2", len(res.Entries)) + } + if res.NextToken == "" { + t.Fatalf("expected a NextToken after one page") + } + if reqs := listRequests(srv); len(reqs) != 1 { + t.Fatalf("--no-paginate issued %d list requests, want 1", len(reqs)) + } +} + +func TestListObjectsStreamsEntries(t *testing.T) { + _, sess := seedLsBucket(t) + var buf bytes.Buffer + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{ + Recursive: true, + OnEntry: func(e renderer.ResponseData) { + if err := streamJSON(&buf, e); err != nil { + t.Fatalf("streamJSON: %v", err) + } + }, + }) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if len(res.Entries) != 0 { + t.Fatalf("streamed entries must not be accumulated, got %d", len(res.Entries)) + } + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + if len(lines) != 4 { + t.Fatalf("got %d NDJSON lines, want 4:\n%s", len(lines), buf.String()) + } + var first map[string]interface{} + if err := json.Unmarshal([]byte(lines[0]), &first); err != nil { + t.Fatalf("line 0 is not JSON: %v", err) + } + if first["key"] != "2026/09/dump.sql" || first["type"] != "object" { + t.Fatalf("unexpected first line: %v", first) + } + if res.Count != 4 { + t.Fatalf("Count = %d, want 4 even when streaming", res.Count) + } +} + +func TestListObjectsErrorsSurface(t *testing.T) { + srv, sess := seedLsBucket(t) + srv.DenyAll = true + _, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{}) + if err == nil { + t.Fatalf("expected an error from a denied listing") + } + herr := sess.humanize(err) + if !strings.Contains(herr.Error(), "access denied") { + t.Fatalf("humanized error = %q", herr) + } +} + +func TestListObjectsVersions(t *testing.T) { + srv := s3test.New() + t.Cleanup(srv.Close) + b := srv.CreateBucket(lsTestBucket) + b.Versioned = true + srv.AddObject(lsTestBucket, "cfg/app.yaml", []byte("v1"), "text/yaml") + srv.AddObject(lsTestBucket, "cfg/app.yaml", []byte("v2!"), "text/yaml") + sess := newLsTestSession(t, srv, lsTestBucket) + + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Prefix: "cfg/", Recursive: true, Versions: true}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if len(res.Entries) != 2 { + t.Fatalf("got %d version entries, want 2: %v", len(res.Entries), entryNames(res.Entries)) + } + // The fake server assigns version IDs before archiving the previous + // version, so only the ordering, sizes and IsLatest are asserted here. + latest, ok := res.Entries[0].(objectstorage.Object) + if !ok || latest.Size != 3 || latest.VersionID == "" || latest.IsLatest == nil || !*latest.IsLatest { + t.Fatalf("first entry should be the latest version (3 bytes): %+v", res.Entries[0]) + } + older := res.Entries[1].(objectstorage.Object) + if older.Size != 2 || older.VersionID == "" || older.IsLatest == nil || *older.IsLatest { + t.Fatalf("second entry should be the older version (2 bytes): %+v", older) + } + + // --max-items applies to version listings too. + capped, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Recursive: true, Versions: true, MaxItems: 1}) + if err != nil { + t.Fatalf("listObjects (capped): %v", err) + } + if len(capped.Entries) != 1 { + t.Fatalf("got %d entries, want 1", len(capped.Entries)) + } + assertNoWrites(t, srv) +} + +// TestLsObjectEntries lists a seeded prefix and checks the renderable entries +// (one PRE prefix + the objects with their sizes), which the shared renderer +// then prints as the table / json / csv the rest of the CLI uses. +func TestLsObjectEntries(t *testing.T) { + _, sess := seedLsBucket(t) + res, err := listObjects(context.Background(), sess.Client, lsTestBucket, listOptions{Prefix: "2026/09/"}) + if err != nil { + t.Fatalf("listObjects: %v", err) + } + if res.Count != 2 || res.Bytes != 1258296 { + t.Fatalf("count=%d bytes=%d, want 2 / 1258296", res.Count, res.Bytes) + } + var sizes []int64 + for _, e := range res.Entries { + if o, ok := e.(objectstorage.Object); ok { + sizes = append(sizes, o.Size) + } + } + if len(sizes) != 2 || sizes[0] != 1258291 { + t.Fatalf("object sizes = %v", sizes) + } +} + +// TestLsSummary checks the --summarize footer helper (used only in table mode). +func TestLsSummary(t *testing.T) { + if got := objectstorage.LsSummary(3, 3*1024*1024, true); got != "\nTotal Objects: 3\n Total Size: 3.0 MiB" { + t.Fatalf("human summary = %q", got) + } + if got := objectstorage.LsSummary(2, 1258296, false); got != "\nTotal Objects: 2\n Total Size: 1258296" { + t.Fatalf("byte summary = %q", got) + } +} + +func TestBucketRowsSortedAndFiltered(t *testing.T) { + created := time.Date(2026, 9, 7, 15, 12, 1, 0, time.Local) + mk := func(id, name string, class components.StorageClass) components.ObjectStorageData { + n := name + i := id + c := class + t := created + return components.ObjectStorageData{ID: &i, Attributes: &components.ObjectStorageDataAttributes{Name: &n, StorageClass: &c, CreatedAt: &t}} + } + data := []components.ObjectStorageData{ + mk("bkt_2", "logs", components.StorageClass("high_performance")), + mk("bkt_1", "backups", components.StorageClass("standard")), + mk("bkt_3", "media", components.StorageClass("standard")), + } + // BucketRows renders through the shared renderer; the row for a bucket keeps + // its name so the table shows it. + rows := BucketRows(data, nil) + if len(rows) != 3 || rows[0].TableRow()["name"].Value != "logs" { + t.Fatalf("unexpected rows: %d", len(rows)) + } + + // --storage-class goes through the shared alias table so "high-performance" + // (and hp, vast…) mean the same thing here as in mb and access-keys. + class, err := objectstorage.ParseStorageClass("high-performance") + if err != nil || class != objectstorage.ClassHighPerformance { + t.Fatalf("ParseStorageClass = %q, %v", class, err) + } + filtered := filterBucketsByClass(data, class) + if len(filtered) != 1 || *filtered[0].Attributes.Name != "logs" { + t.Fatalf("filtered = %v", entryBucketNames(filtered)) + } + // An empty value is "no filter", not an error. + if class, err := objectstorage.ParseStorageClass(""); err != nil || class != "" { + t.Fatalf("ParseStorageClass(\"\") = %q, %v; want no filter", class, err) + } + if len(filterBucketsByClass(data, "")) != len(data) { + t.Fatalf("an empty class must keep every bucket") + } + if _, err := objectstorage.ParseStorageClass("glacier"); exitcode.Of(err) != exitcode.Usage { + t.Fatalf("expected a usage error for an unknown storage class, got %v", err) + } +} + +// TestListObjectsCancelBetweenPages cancels the context while the first page +// is being consumed: the loop must stop before requesting the second page and +// surface context.Canceled, which the session maps to exit 130. +func TestListObjectsCancelBetweenPages(t *testing.T) { + srv, sess := seedLsBucket(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var seen int + res, err := listObjects(ctx, sess.Client, lsTestBucket, listOptions{ + Recursive: true, + OnEntry: func(renderer.ResponseData) { + seen++ + cancel() + }, + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if seen == 0 || seen > srv.MaxListKeys { + t.Fatalf("saw %d entries, want only the first page (<= %d)", seen, srv.MaxListKeys) + } + if got := len(listRequests(srv)); got != 1 { + t.Fatalf("issued %d list requests after cancellation, want 1", got) + } + if res.Total != int64(seen) { + t.Fatalf("Total = %d, want %d", res.Total, seen) + } + if code := exitcode.Of(sess.humanize(err)); code != exitcode.Interrupted { + t.Fatalf("humanized exit code = %d, want %d (interrupted)", code, exitcode.Interrupted) + } +} + +// TestListObjectsCancelDuringRequest cancels the context while a list request +// is still in flight: minio.Core.ListObjectsV2 has no context, so listObjects +// must stop waiting on its own instead of hanging until the server answers. +func TestListObjectsCancelDuringRequest(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + var once bool + blocking := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !once { + once = true + close(started) + } + <-release + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(blocking.Close) + + b := &objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: lsTestBucket, Endpoint: blocking.URL, SigningRegion: "us-east-1"} + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + sess := &Session{Bucket: b, Cred: cred, Client: client} + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-started + cancel() + }() + done := make(chan error, 1) + go func() { + _, err := listObjects(ctx, sess.Client, lsTestBucket, listOptions{Recursive: true}) + done <- err + }() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if code := exitcode.Of(sess.humanize(err)); code != exitcode.Interrupted { + t.Fatalf("humanized exit code = %d, want %d (interrupted)", code, exitcode.Interrupted) + } + case <-time.After(5 * time.Second): + t.Fatalf("listObjects did not return after the context was cancelled while the request was in flight") + } + // Let the abandoned request finish so the server can shut down cleanly. + close(release) +} + +// TestLsInvalidBackendNameExitsUsage drives the real openBucket path with an +// endpoint override: a backend bucket name S3 rejects ("b" is shorter than the +// three characters the protocol requires) must be a usage error (exit 2) +// before any request or credential lookup happens. +func TestLsInvalidBackendNameExitsUsage(t *testing.T) { + srv := s3test.New() + t.Cleanup(srv.Close) + t.Setenv(objectstorage.EnvEndpointURL, srv.URL()) + + cmd := NewLsCmd() + err := runLsObjects(context.Background(), cmd, "s3://b/") + if err == nil { + t.Fatalf("expected an error for the one-letter backend bucket name") + } + if code := exitcode.Of(err); code != exitcode.Usage { + t.Fatalf("exit code = %d, want %d; err=%v", code, exitcode.Usage, err) + } + if !strings.Contains(err.Error(), "invalid bucket name") { + t.Fatalf("error should name the invalid bucket: %v", err) + } + if reqs := srv.Requests(); len(reqs) != 0 { + t.Fatalf("no request should reach the endpoint, got %d", len(reqs)) + } +} + +func entryBucketNames(data []components.ObjectStorageData) []string { + out := make([]string, 0, len(data)) + for _, d := range data { + out = append(out, objectstorage.BucketFromData(d).Name) + } + return out +} diff --git a/cmd/s3/mb.go b/cmd/s3/mb.go new file mode 100644 index 0000000..f6b8eed --- /dev/null +++ b/cmd/s3/mb.go @@ -0,0 +1,614 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "strings" + "time" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + cobra "github.com/spf13/cobra" +) + +// Flag names of mb. +const ( + flagMbName = "name" + flagMbRegion = "region" + flagMbStorageClass = "storage-class" + flagMbVersioning = "versioning" + flagMbLocking = "locking" + flagMbRetentionMode = "retention-mode" + flagMbRetentionDays = "retention-days" + flagMbCreateAccessKey = "create-access-key" + flagMbNoAccessKey = "no-access-key" +) + +// mbOptions are the validated inputs of `mb`. +type mbOptions struct { + Name string + Project string + Region string + StorageClass string + // Versioning is the --versioning value; VersioningSet records whether the + // flag was given explicitly (so --locking --versioning=false is caught). + Versioning bool + VersioningSet bool + Locking bool + RetentionMode string + RetentionDays int64 +} + +// NewMbCmd builds `lsh s3 create-bucket s3://bucket`. +func NewMbCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "create-bucket s3://bucket", + Aliases: []string{"mb", "create"}, + GroupID: groupBuckets, + Short: "Create a bucket (alias: mb)", + Long: `Create an object storage bucket through the Latitude API. + +The bucket is created in a Latitude site (--region DAL, NYC, TYO4…). The +storage class picks the backend tier: standard (default) or +high_performance (select sites only). Versioning, object lock and the default +retention cannot be changed after creation. + +In an interactive session missing --project and --region are asked for, and +right after the bucket is created the CLI offers to create and save an S3 +access key so copy/list/delete work immediately; use --create-access-key or +--no-access-key in scripts.`, + Example: ` lsh s3 create-bucket s3://backups --region DAL --project my-project + lsh s3 create-bucket s3://fast-cache --region TYO4 --storage-class high_performance + lsh s3 create-bucket s3://audit --region NYC --locking --retention-mode COMPLIANCE --retention-days 30 + lsh s3 create-bucket s3://ci-artifacts --region DAL --create-access-key + lsh s3 create-bucket s3://backups --region DAL -o json`, + Args: cobra.MaximumNArgs(1), + RunE: runMb, + }) + // Registered as required so the root pre-run opens the shared project + // picker (or fails with exit 2 in a non-interactive session), like every + // other project-scoped command. + addProjectFlag(cmd, false, "project ID or slug that owns the bucket (required; a picker opens when omitted in a terminal)") + cmd.Flags().String(flagMbName, "", "bucket name (legacy alternative to the positional s3://bucket)") + _ = cmd.Flags().MarkHidden(flagMbName) + cmd.Flags().String(flagMbRegion, "", "Latitude site slug where the bucket lives (e.g. DAL, NYC, TYO4); see 'lsh regions list'") + cmd.Flags().StringP(flagMbStorageClass, "c", objectstorage.ClassStandard, "storage class: standard or high_performance (aliases: std, hp)") + cmd.Flags().Bool(flagMbVersioning, false, "enable object versioning") + cmd.Flags().Bool(flagMbLocking, false, "enable object lock (WORM); implies --versioning and cannot be added later") + cmd.Flags().String(flagMbRetentionMode, "", "default object lock retention mode: NONE, GOVERNANCE or COMPLIANCE (requires --locking)") + cmd.Flags().Int64(flagMbRetentionDays, 0, "default object lock retention period in days (requires --locking)") + cmd.Flags().Bool(flagMbCreateAccessKey, false, "create and save an access key for the new bucket's class without asking") + cmd.Flags().Bool(flagMbNoAccessKey, false, "never offer to create an access key") + unsupportedAWSFlags(cmd, map[string]string{ + "tags": "buckets have no tags; use the bucket name or project to organise them", + }) + return cmd +} + +func runMb(cmd *cobra.Command, args []string) error { + o, err := mbOptionsFromCmd(cmd, args) + if err != nil { + return printErr(err) + } + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "mb creates buckets through the Latitude API; unset --endpoint-url / %s", objectstorage.EnvEndpointURL)) + } + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + if o.Region == "" { + region, err := pickRegion(ctx, cmd) + if err != nil { + return printErr(err) + } + o.Region = region + } + req, err := buildCreateBucketRequest(o) + if err != nil { + return printErr(err) + } + a := req.Data.Attributes + + if dryRun() { + if isHuman() { + fmt.Printf("(dryrun) make_bucket: s3://%s\n", o.Name) + fmt.Println(" " + describeCreateRequest(a)) + } else { + render([]renderer.ResponseData{mbPlan{Attributes: a, DryRun: true}}) + } + return nil + } + + if o.Locking { + objectstorage.Warnf("object lock, versioning and the default retention cannot be changed after the bucket is created") + if strings.EqualFold(o.RetentionMode, string(operations.RetentionModeCompliance)) && objectstorage.CanPrompt(cmd) { + ok, err := objectstorage.Confirm(fmt.Sprintf("COMPLIANCE retention of %d days cannot be shortened or bypassed by anyone. Create bucket %s?", o.RetentionDays, o.Name)) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + if !ok { + return printErr(exitcode.Errorf(exitcode.Refused, "cancelled")) + } + } + } + + client := apiClient() + resp, err := client.ObjectStorage.PostStorageBuckets(ctx, req, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return printErr(objectstorage.Humanize(err, nil, nil)) + } + if resp == nil || resp.Object == nil || resp.Object.Data == nil { + return printErr(exitcode.Errorf(exitcode.Generic, "the API returned no bucket for %s", o.Name)) + } + b := objectstorage.BucketFromData(*resp.Object.Data) + if b.Name == "" { + b.Name = o.Name + } + if b.Site == "" { + b.Site = strings.ToUpper(o.Region) + } + if b.StorageClass == "" { + b.StorageClass = string(*a.StorageClass) + } + if b.ProjectID == "" && b.ProjectSlug == "" { + b.ProjectSlug = o.Project + } + + if isHuman() { + fmt.Printf("make_bucket: s3://%s\n", b.Name) + for _, line := range describeCreatedBucket(b) { + fmt.Println(" " + line) + } + } else { + render([]renderer.ResponseData{NewBucketRow(b)}) + } + + offerAccessKey(ctx, cmd, b, o) + if isHuman() { + objectstorage.Hintf("Try: lsh s3 copy ./file s3://%s/", b.Name) + } + return nil +} + +// mbOptionsFromCmd reads the flags and positional argument. +func mbOptionsFromCmd(cmd *cobra.Command, args []string) (mbOptions, error) { + f := cmd.Flags() + o := mbOptions{Project: projectFlag(cmd)} + legacy, _ := f.GetString(flagMbName) + switch { + case len(args) == 1 && legacy != "" && !sameBucketToken(args[0], legacy): + return o, exitcode.Errorf(exitcode.Usage, "bucket given twice (%q and --name %q); pass only one", args[0], legacy) + case len(args) == 1: + ref, err := objectstorage.ParseBucketOnly(args[0]) + if err != nil { + return o, err + } + o.Name = ref.Bucket + case legacy != "": + ref, err := objectstorage.ParseBucketOnly(legacy) + if err != nil { + return o, err + } + o.Name = ref.Bucket + default: + return o, exitcode.Errorf(exitcode.Usage, "missing bucket: usage 'lsh s3 create-bucket s3:// --region '") + } + o.Region, _ = f.GetString(flagMbRegion) + o.StorageClass, _ = f.GetString(flagMbStorageClass) + o.Versioning, _ = f.GetBool(flagMbVersioning) + o.VersioningSet = f.Changed(flagMbVersioning) + o.Locking, _ = f.GetBool(flagMbLocking) + o.RetentionMode, _ = f.GetString(flagMbRetentionMode) + o.RetentionDays, _ = f.GetInt64(flagMbRetentionDays) + return o, nil +} + +func sameBucketToken(a, b string) bool { + ra, errA := objectstorage.ParseBucketOnly(a) + rb, errB := objectstorage.ParseBucketOnly(b) + return errA == nil && errB == nil && ra.Bucket == rb.Bucket +} + +// buildCreateBucketRequest validates the options and builds the API request. +// It is pure so the rules can be unit-tested without cobra or the API. +func buildCreateBucketRequest(o mbOptions) (operations.PostStorageBucketsRequestBody, error) { + var req operations.PostStorageBucketsRequestBody + name := strings.TrimSpace(o.Name) + if name == "" { + return req, exitcode.Errorf(exitcode.Usage, "missing bucket name: usage 'lsh s3 create-bucket s3:// --region '") + } + if strings.ContainsAny(name, " /\\") { + return req, exitcode.Errorf(exitcode.Usage, "invalid bucket name %q: it cannot contain spaces or slashes", name) + } + project := strings.TrimSpace(o.Project) + if project == "" { + return req, exitcode.Errorf(exitcode.Usage, "--project is required (or set LSH_PROJECT): the bucket must belong to a project") + } + region := strings.TrimSpace(o.Region) + if region == "" { + return req, exitcode.Errorf(exitcode.Usage, "--region is required; regions are Latitude sites such as DAL, NYC or TYO4 (run 'lsh regions list')") + } + if looksLikeAWSRegion(region) { + return req, exitcode.Errorf(exitcode.Usage, "%q is not a Latitude site; --region takes a site slug such as DAL, NYC or TYO4 (run 'lsh regions list')", region) + } + + // The shared alias table (std, hp, high-performance…) applies here too. + class, err := objectstorage.ParseStorageClass(o.StorageClass) + if err != nil { + return req, exitcode.Errorf(exitcode.Usage, "invalid --storage-class %q: use standard or high_performance", o.StorageClass) + } + if class == "" { + class = objectstorage.ClassStandard + } + var storageClass operations.StorageClass + switch class { + case objectstorage.ClassHighPerformance: + storageClass = operations.StorageClassHighPerformance + default: + storageClass = operations.StorageClassStandard + } + + mode := strings.ToUpper(strings.TrimSpace(o.RetentionMode)) + var retentionMode operations.RetentionMode + switch mode { + case "", string(operations.RetentionModeNone): + retentionMode = operations.RetentionModeNone + case string(operations.RetentionModeGovernance): + retentionMode = operations.RetentionModeGovernance + case string(operations.RetentionModeCompliance): + retentionMode = operations.RetentionModeCompliance + default: + return req, exitcode.Errorf(exitcode.Usage, "invalid --retention-mode %q: use NONE, GOVERNANCE or COMPLIANCE", o.RetentionMode) + } + if o.RetentionDays < 0 { + return req, exitcode.Errorf(exitcode.Usage, "--retention-days must be a positive number of days") + } + hasRetention := retentionMode != operations.RetentionModeNone || o.RetentionDays > 0 + if hasRetention && !o.Locking { + return req, exitcode.Errorf(exitcode.Usage, "--retention-mode and --retention-days require --locking (object lock must be enabled at creation)") + } + if retentionMode != operations.RetentionModeNone && o.RetentionDays == 0 { + return req, exitcode.Errorf(exitcode.Usage, "--retention-mode %s requires --retention-days ", retentionMode) + } + if o.RetentionDays > 0 && retentionMode == operations.RetentionModeNone { + return req, exitcode.Errorf(exitcode.Usage, "--retention-days requires --retention-mode GOVERNANCE or COMPLIANCE") + } + + versioning := o.Versioning + if o.Locking { + if o.VersioningSet && !o.Versioning { + return req, exitcode.Errorf(exitcode.Usage, "--locking requires versioning; drop --versioning=false") + } + versioning = true + } + + attrs := operations.PostStorageBucketsAttributes{ + Project: project, + Name: name, + Region: strings.ToUpper(region), + StorageClass: &storageClass, + Versioning: &versioning, + Locking: mbPtr(o.Locking), + RetentionMode: &retentionMode, + } + if o.RetentionDays > 0 { + attrs.RetentionPeriod = mbPtr(o.RetentionDays) + } + req.Data = operations.PostStorageBucketsData{ + Type: operations.PostStorageBucketsTypeObjects, + Attributes: attrs, + } + return req, nil +} + +func mbPtr[T any](v T) *T { return &v } + +// describeCreateRequest renders the request for dry-run output. +func describeCreateRequest(a operations.PostStorageBucketsAttributes) string { + parts := []string{ + "project: " + a.Project, + "region: " + a.Region, + "class: " + string(*a.StorageClass), + "versioning: " + yesNo(a.Versioning != nil && *a.Versioning), + } + if a.Locking != nil && *a.Locking { + lock := "yes" + if a.RetentionMode != nil && *a.RetentionMode != operations.RetentionModeNone { + lock = string(*a.RetentionMode) + if a.RetentionPeriod != nil { + lock = fmt.Sprintf("%s (%dd)", lock, *a.RetentionPeriod) + } + } + parts = append(parts, "locking: "+lock) + } else { + parts = append(parts, "locking: no") + } + return strings.Join(parts, " ") +} + +// describeCreatedBucket renders the detail lines under `make_bucket:`. +func describeCreatedBucket(b *objectstorage.Bucket) []string { + first := []string{} + if b.ID != "" { + first = append(first, "id: "+b.ID) + } + if b.BucketName != "" { + first = append(first, "bucket name on the endpoint: "+b.BucketName) + } + if b.StorageClass != "" { + first = append(first, "class: "+b.StorageClass) + } + if b.Site != "" { + first = append(first, "site: "+b.Site) + } + if b.Endpoint != "" { + first = append(first, "endpoint: "+b.Endpoint) + } + lines := []string{strings.Join(first, " ")} + if b.Versioning || b.Locking { + lines = append(lines, fmt.Sprintf("versioning: %s locking: %s", yesNo(b.Versioning), lockingLabel(b))) + } + return lines +} + +// mbPlan is the structured dry-run row of mb. +type mbPlan struct { + Attributes operations.PostStorageBucketsAttributes + DryRun bool +} + +func (p mbPlan) TableRow() table.Row { + a := p.Attributes + op := "make_bucket" + if p.DryRun { + op = "(dryrun) make_bucket" + } + retention := "" + if a.RetentionMode != nil && *a.RetentionMode != operations.RetentionModeNone { + retention = string(*a.RetentionMode) + if a.RetentionPeriod != nil { + retention = fmt.Sprintf("%s (%dd)", retention, *a.RetentionPeriod) + } + } + return table.Row{ + "op": {Label: "Op", Value: op}, + "name": {Label: "Name", Value: a.Name}, + "project": {Label: "Project", Value: a.Project}, + "region": {Label: "Site", Value: a.Region}, + "storage_class": {Label: "Class", Value: string(*a.StorageClass)}, + "versioning": {Label: "Versioning", Value: yesNo(a.Versioning != nil && *a.Versioning)}, + "locking": {Label: "Locking", Value: yesNo(a.Locking != nil && *a.Locking)}, + "retention": {Label: "Retention", Value: retention}, + } +} + +// MarshalJSON emits the plan as a flat document. +func (p mbPlan) MarshalJSON() ([]byte, error) { + a := p.Attributes + doc := map[string]interface{}{ + "op": "make_bucket", + "dry_run": p.DryRun, + "name": a.Name, + "project": a.Project, + "region": a.Region, + "storage_class": string(*a.StorageClass), + "versioning": a.Versioning != nil && *a.Versioning, + "locking": a.Locking != nil && *a.Locking, + } + if a.RetentionMode != nil && *a.RetentionMode != operations.RetentionModeNone { + doc["retention_mode"] = string(*a.RetentionMode) + if a.RetentionPeriod != nil { + doc["retention_days"] = *a.RetentionPeriod + } + } + return json.Marshal(doc) +} + +// pickRegion asks the user for a site when --region is missing, or fails +// with a usage error in non-interactive sessions. +func pickRegion(ctx context.Context, cmd *cobra.Command) (string, error) { + if !objectstorage.CanPrompt(cmd) { + return "", exitcode.Errorf(exitcode.Usage, "--region is required; regions are Latitude sites such as DAL, NYC or TYO4 (run 'lsh regions list')") + } + sites, err := mbListSites(ctx) + if err != nil { + return "", err + } + if len(sites) == 0 { + return "", exitcode.Errorf(exitcode.Usage, "--region is required and no sites were returned by the API (run 'lsh regions list')") + } + options := make([]string, 0, len(sites)) + for _, s := range sites { + options = append(options, s.label()) + } + idx, err := objectstorage.Choose("Which site should host the bucket? (--region)", options, 0) + if err != nil { + return "", objectstorage.Humanize(err, nil, nil) + } + if idx < 0 { + return "", exitcode.Errorf(exitcode.Refused, "no site selected; pass --region ") + } + return sites[idx].Slug, nil +} + +type mbSite struct { + Slug string + Name string +} + +func (s mbSite) label() string { + if s.Name == "" || strings.EqualFold(s.Name, s.Slug) { + return s.Slug + } + return s.Slug + " - " + s.Name +} + +// mbListSites fetches the site slugs from the API (custom storage-only sites +// included), sorted by slug. +func mbListSites(ctx context.Context) ([]mbSite, error) { + client := apiClient() + resp, err := client.Regions.Get(ctx, operations.GetRegionsRequest{ + IncludeCustom: mbPtr(true), + PageSize: mbPtr(int64(100)), + }, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return nil, objectstorage.Humanize(err, nil, nil) + } + var out []mbSite + if resp != nil && resp.Regions != nil { + for _, r := range resp.Regions.Data { + if r.Attributes == nil || r.Attributes.Slug == nil || *r.Attributes.Slug == "" { + continue + } + s := mbSite{Slug: *r.Attributes.Slug} + if r.Attributes.Name != nil { + s.Name = *r.Attributes.Name + } + out = append(out, s) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Slug < out[j].Slug }) + return out, nil +} + +// offerAccessKey implements the post-creation access key offer (J1). Any +// failure here is a warning: the bucket already exists and the command +// succeeds. +func offerAccessKey(ctx context.Context, cmd *cobra.Command, b *objectstorage.Bucket, o mbOptions) { + f := cmd.Flags() + noKey, _ := f.GetBool(flagMbNoAccessKey) + createKey, _ := f.GetBool(flagMbCreateAccessKey) + if noKey || endpointOverride(cmd) != "" { + return + } + if !isHuman() { + // The key would have to be printed to stay usable, and the structured + // document of mb describes a bucket; say so instead of doing nothing. + if createKey { + objectstorage.Warnf("--create-access-key is only offered with human output; create the key with: lsh s3 access-keys create --bucket %s --save", b.Name) + } + return + } + if !createKey && !objectstorage.CanPrompt(cmd) { + return + } + _, profileName, profile, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + objectstorage.Warnf("not saving an access key: %v", err) + return + } + keys := profile.ObjectStorageKeys() + if name, _, ok := objectstorage.SelectKey(keys, profile.DefaultObjectStorageKey(), b, true); ok { + objectstorage.Hintf("Saved access key %q already covers this bucket; the CLI will use it automatically.", name) + return + } + + scope := config.ScopeFullAccess + if !createKey { + allLabel := fmt.Sprintf("key for all %s buckets of project %s", b.StorageClass, b.ProjectRef()) + if b.StorageClass == objectstorage.ClassHighPerformance && b.Site != "" { + allLabel += " in site " + b.Site + } + idx, err := objectstorage.Choose( + "To upload and download files the CLI needs an S3 access key (separate from your API token). Create one now?", + []string{allLabel + " (recommended)", "key for this bucket only (rw)", "skip"}, 0) + if err != nil { + objectstorage.Warnf("could not read the answer: %v", err) + return + } + switch idx { + case 0: + scope = config.ScopeFullAccess + case 1: + scope = config.ScopeLimitedAccess + default: + objectstorage.Hintf("Skipped. Create a key later with: lsh s3 configure (or: lsh s3 access-keys create --bucket %s --save)", b.Name) + return + } + } + + if b.StorageClass == objectstorage.ClassHighPerformance && b.Site == "" { + if err := newResolver(cmd).FillSite(ctx, b); err != nil { + objectstorage.Warnf("could not determine the bucket's site: %v", err) + } + } + // The name is generated the way the dashboard does it (see petname.go); a + // clash with a saved key is handled by saveNewKey when storing. + req := accessKeyRequest{ + Project: firstNonEmptyStr(b.ProjectID, b.ProjectSlug, o.Project), + StorageClass: b.StorageClass, + Site: b.Site, + Scope: scope, + } + var perms map[string]string + switch scope { + case config.ScopeLimitedAccess: + perms = map[string]string{b.ID: config.PermissionRW} + req.Buckets = perms + } + req.Name = generateKeyName(b.StorageClass) + + created, err := createAccessKeyRetrying(ctx, cmd, req, true) + if err != nil { + objectstorage.Warnf("bucket created, but the access key was not: %v", err) + objectstorage.Hintf("Create one later with: lsh s3 access-keys create --bucket %s --save", b.Name) + return + } + stored := created.stored(b.ProjectID, perms, config.KeySourceMakeBkt) + if stored.StorageClass == "" { + stored.StorageClass = b.StorageClass + } + if stored.Site == "" && b.StorageClass == objectstorage.ClassHighPerformance { + stored.Site = b.Site + } + if stored.Scope == "" { + stored.Scope = scope + } + stored.CreatedAt = time.Now().UTC() + name := created.Name + if name == "" { + name = req.Name + } + finalName, savedProfile, err := saveNewKey(cmd, name, stored) + if err != nil { + mbReportUnsavedKey(ctx, os.Stderr, created, req, b, name, err) + return + } + if savedProfile == "" { + savedProfile = profileName + } + objectstorage.Hintf("Saved to %s (0600) as %q in profile %s. The CLI uses it automatically.", mbConfigPathForHint(), finalName, savedProfile) + objectstorage.Hintf("For apps or CI create a separate key: lsh s3 access-keys create --bucket %s", b.Name) +} + +// mbReportUnsavedKey handles a key that was created but could not be stored in +// the profile. mb creates the key on the user's behalf, so the key is removed +// again rather than leaving a credential whose secret would have to be printed; +// the shared helper only prints when the removal fails. +func mbReportUnsavedKey(ctx context.Context, w io.Writer, created *createdAccessKey, req accessKeyRequest, b *objectstorage.Bucket, name string, saveErr error) { + created.Endpoint = firstNonEmptyStr(created.Endpoint, b.Endpoint) + created.SigningRegion = firstNonEmptyStr(created.SigningRegion, b.SigningRegion) + plan := &createPlan{Request: req} + if req.Scope == config.ScopeLimitedAccess { + plan.Buckets = []scopedBucket{{Bucket: b, Permission: config.PermissionRW}} + created.Buckets = map[string]string{b.Name: config.PermissionRW} + } + reportUnsavedKey(ctx, w, req, created, plan, name, false, saveErr) +} + +// mbConfigPathForHint renders the config location for messages. +func mbConfigPathForHint() string { + if p := os.Getenv("LSH_CONFIG_PATH"); p != "" { + return p + } + return "~/.config/lsh/config.json" +} diff --git a/cmd/s3/mb_test.go b/cmd/s3/mb_test.go new file mode 100644 index 0000000..526edd2 --- /dev/null +++ b/cmd/s3/mb_test.go @@ -0,0 +1,379 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cli" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func baseMbOptions() mbOptions { + return mbOptions{Name: "backups", Project: "my-project", Region: "DAL"} +} + +func TestBuildCreateBucketRequest_Defaults(t *testing.T) { + req, err := buildCreateBucketRequest(baseMbOptions()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := req.Data.Attributes + if req.Data.Type != operations.PostStorageBucketsTypeObjects { + t.Errorf("type = %q, want objects", req.Data.Type) + } + if a.Name != "backups" || a.Project != "my-project" || a.Region != "DAL" { + t.Errorf("attributes = %+v", a) + } + if a.StorageClass == nil || *a.StorageClass != operations.StorageClassStandard { + t.Errorf("storage class = %v, want standard", a.StorageClass) + } + if a.Versioning == nil || *a.Versioning { + t.Errorf("versioning should default to false") + } + if a.Locking == nil || *a.Locking { + t.Errorf("locking should default to false") + } + if a.RetentionMode == nil || *a.RetentionMode != operations.RetentionModeNone { + t.Errorf("retention mode = %v, want NONE", a.RetentionMode) + } + if a.RetentionPeriod != nil { + t.Errorf("retention period should be nil, got %d", *a.RetentionPeriod) + } +} + +func TestBuildCreateBucketRequest_RegionUpperCased(t *testing.T) { + o := baseMbOptions() + o.Region = "tyo4" + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if req.Data.Attributes.Region != "TYO4" { + t.Errorf("region = %q, want TYO4", req.Data.Attributes.Region) + } +} + +func TestBuildCreateBucketRequest_RequiredFields(t *testing.T) { + cases := []struct { + name string + mutate func(*mbOptions) + want string + }{ + {"missing name", func(o *mbOptions) { o.Name = "" }, "missing bucket name"}, + {"missing project", func(o *mbOptions) { o.Project = "" }, "--project is required"}, + {"missing region", func(o *mbOptions) { o.Region = "" }, "--region is required"}, + {"bad class", func(o *mbOptions) { o.StorageClass = "glacier" }, "invalid --storage-class"}, + {"bad retention mode", func(o *mbOptions) { o.Locking = true; o.RetentionMode = "FOREVER"; o.RetentionDays = 1 }, "invalid --retention-mode"}, + {"name with slash", func(o *mbOptions) { o.Name = "a/b" }, "cannot contain spaces or slashes"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + o := baseMbOptions() + tc.mutate(&o) + _, err := buildCreateBucketRequest(o) + if err == nil { + t.Fatalf("expected an error") + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("exit code = %d, want %d (usage)", exitcode.Of(err), exitcode.Usage) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error %q does not mention %q", err.Error(), tc.want) + } + }) + } +} + +func TestBuildCreateBucketRequest_AWSRegionDetected(t *testing.T) { + for _, region := range []string{"us-east-1", "eu-central-2", "ap-southeast-1"} { + o := baseMbOptions() + o.Region = region + _, err := buildCreateBucketRequest(o) + if err == nil { + t.Fatalf("%s: expected an error", region) + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("%s: exit code = %d, want usage", region, exitcode.Of(err)) + } + if !strings.Contains(err.Error(), "is not a Latitude site") || !strings.Contains(err.Error(), "DAL") { + t.Errorf("%s: error %q should explain that regions are Latitude sites", region, err.Error()) + } + } + // Latitude slugs with digits are not mistaken for AWS regions. + for _, region := range []string{"DAL", "TYO4", "SAO2", "nyc"} { + o := baseMbOptions() + o.Region = region + if _, err := buildCreateBucketRequest(o); err != nil { + t.Errorf("%s: unexpected error %v", region, err) + } + } +} + +func TestBuildCreateBucketRequest_LockingImpliesVersioning(t *testing.T) { + o := baseMbOptions() + o.Locking = true + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := req.Data.Attributes + if a.Locking == nil || !*a.Locking { + t.Errorf("locking should be true") + } + if a.Versioning == nil || !*a.Versioning { + t.Errorf("versioning should be implied by --locking") + } + + o.VersioningSet = true + o.Versioning = false + _, err = buildCreateBucketRequest(o) + if err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("--locking --versioning=false should be a usage error, got %v", err) + } + if !strings.Contains(err.Error(), "--locking requires versioning") { + t.Errorf("unexpected message %q", err.Error()) + } +} + +func TestBuildCreateBucketRequest_RetentionOnlyWithLocking(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*mbOptions) + }{ + {"mode without locking", func(o *mbOptions) { o.RetentionMode = "GOVERNANCE"; o.RetentionDays = 7 }}, + {"days without locking", func(o *mbOptions) { o.RetentionDays = 7 }}, + } { + t.Run(tc.name, func(t *testing.T) { + o := baseMbOptions() + tc.mutate(&o) + _, err := buildCreateBucketRequest(o) + if err == nil || exitcode.Of(err) != exitcode.Usage { + t.Fatalf("expected a usage error, got %v", err) + } + if !strings.Contains(err.Error(), "require --locking") { + t.Errorf("unexpected message %q", err.Error()) + } + }) + } + + // Mode and days must come together. + o := baseMbOptions() + o.Locking = true + o.RetentionMode = "compliance" + if _, err := buildCreateBucketRequest(o); err == nil || !strings.Contains(err.Error(), "requires --retention-days") { + t.Errorf("mode without days should be rejected, got %v", err) + } + o = baseMbOptions() + o.Locking = true + o.RetentionDays = 30 + if _, err := buildCreateBucketRequest(o); err == nil || !strings.Contains(err.Error(), "requires --retention-mode") { + t.Errorf("days without mode should be rejected, got %v", err) + } + o = baseMbOptions() + o.Locking = true + o.RetentionMode = "GOVERNANCE" + o.RetentionDays = -1 + if _, err := buildCreateBucketRequest(o); err == nil || !strings.Contains(err.Error(), "positive") { + t.Errorf("negative days should be rejected, got %v", err) + } +} + +func TestBuildCreateBucketRequest_RetentionMapsToPeriod(t *testing.T) { + o := baseMbOptions() + o.Locking = true + o.RetentionMode = "compliance" + o.RetentionDays = 30 + o.StorageClass = "high_performance" + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + a := req.Data.Attributes + if a.RetentionMode == nil || *a.RetentionMode != operations.RetentionModeCompliance { + t.Errorf("retention mode = %v, want COMPLIANCE", a.RetentionMode) + } + if a.RetentionPeriod == nil || *a.RetentionPeriod != 30 { + t.Errorf("retention period = %v, want 30", a.RetentionPeriod) + } + if a.StorageClass == nil || *a.StorageClass != operations.StorageClassHighPerformance { + t.Errorf("storage class = %v, want high_performance", a.StorageClass) + } + + body, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, want := range []string{`"retention_period":30`, `"retention_mode":"COMPLIANCE"`, `"locking":true`, `"versioning":true`, `"storage_class":"high_performance"`} { + if !strings.Contains(string(body), want) { + t.Errorf("request body %s lacks %s", body, want) + } + } +} + +func TestDescribeCreateRequest(t *testing.T) { + o := baseMbOptions() + o.Locking = true + o.RetentionMode = "GOVERNANCE" + o.RetentionDays = 7 + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Fatal(err) + } + got := describeCreateRequest(req.Data.Attributes) + for _, want := range []string{"project: my-project", "region: DAL", "class: standard", "versioning: yes", "locking: GOVERNANCE (7d)"} { + if !strings.Contains(got, want) { + t.Errorf("%q lacks %q", got, want) + } + } +} + +func TestMbPlanJSON(t *testing.T) { + req, err := buildCreateBucketRequest(baseMbOptions()) + if err != nil { + t.Fatal(err) + } + body, err := json.Marshal(mbPlan{Attributes: req.Data.Attributes, DryRun: true}) + if err != nil { + t.Fatal(err) + } + var doc map[string]interface{} + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + if doc["op"] != "make_bucket" || doc["dry_run"] != true || doc["name"] != "backups" || doc["region"] != "DAL" { + t.Errorf("plan = %s", body) + } + if _, has := doc["retention_mode"]; has { + t.Errorf("retention_mode should be omitted when NONE: %s", body) + } +} + +// Default key names and the clash policy now come from the shared helpers +// (defaultKeyName in accesskeys_shared.go, saveNewKey in s3.go), which are +// covered by their own tests; the mb-specific copies were removed. + +func TestBuildCreateBucketRequest_StorageClassAliases(t *testing.T) { + cases := map[string]operations.StorageClass{ + "": operations.StorageClassStandard, + "standard": operations.StorageClassStandard, + "std": operations.StorageClassStandard, + "high_performance": operations.StorageClassHighPerformance, + "high-performance": operations.StorageClassHighPerformance, + "HP": operations.StorageClassHighPerformance, + } + for in, want := range cases { + o := baseMbOptions() + o.StorageClass = in + req, err := buildCreateBucketRequest(o) + if err != nil { + t.Errorf("%q: unexpected error %v", in, err) + continue + } + if got := *req.Data.Attributes.StorageClass; got != want { + t.Errorf("%q: storage class = %q, want %q", in, got, want) + } + } +} + +func TestNewMbCmd_ProjectFlagRequiresRootPicker(t *testing.T) { + cmd := NewMbCmd() + if cmd.Flags().Lookup(flagProject) == nil { + t.Fatalf("mb must register --project") + } + // mb needs exactly one project, so --project is NOT marked optional: the + // root pre-run resolves it the same way as every other command (LSH_PROJECT, + // then the interactive project picker in a terminal, else an error). + if cmd.Annotations[cli.ProjectOptionalAnnotation] == "true" { + t.Errorf("mb must not carry %s: the root pre-run should own the project prompt", cli.ProjectOptionalAnnotation) + } + // buildCreateBucketRequest still guards against an empty project as a safety net. + o := baseMbOptions() + o.Project = "" + _, err := buildCreateBucketRequest(o) + if err == nil || exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--project is required") { + t.Errorf("missing project must stay a usage error, got %v", err) + } +} + +// mbUnsavedKeyFixture builds the state of a key that was created for a new +// bucket but could not be stored in the profile. +func mbUnsavedKeyFixture() (*createdAccessKey, accessKeyRequest, *objectstorage.Bucket) { + created := &createdAccessKey{ + Name: "lsh-me-backups", AccessKeyID: "AKIAEXAMPLE", SecretAccessKey: "s3cr3t/value", + Username: "me+lsh-me-backups@latitude.sh", + StorageClass: "standard", Project: "my-project", Scope: config.ScopeLimitedAccess, + } + req := accessKeyRequest{Project: "my-project", StorageClass: "standard", Name: "lsh-me-backups", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": config.PermissionRW}} + b := &objectstorage.Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a", Endpoint: "https://s3.us-central-1.storage.sh", SigningRegion: "us-central-1"} + return created, req, b +} + +// stubDiscard replaces the API deletion for the duration of a test. +func stubDiscard(t *testing.T, err error) *int { + t.Helper() + calls := 0 + saved := discardKey + discardKey = func(context.Context, accessKeyRequest, *createdAccessKey) error { + calls++ + return err + } + t.Cleanup(func() { discardKey = saved }) + return &calls +} + +// TestMbReportUnsavedKey_RemovesInsteadOfPrinting covers the preferred +// recovery: a key that cannot be saved is deleted again, so no one-time secret +// is printed and no credential is left live on the API. +func TestMbReportUnsavedKey_RemovesInsteadOfPrinting(t *testing.T) { + created, req, b := mbUnsavedKeyFixture() + calls := stubDiscard(t, nil) + + var out bytes.Buffer + mbReportUnsavedKey(context.Background(), &out, created, req, b, "lsh-me-backups", errors.New("read-only file system")) + text := out.String() + if *calls != 1 { + t.Errorf("the key must be deleted exactly once, got %d calls", *calls) + } + if strings.Contains(text, "s3cr3t/value") { + t.Errorf("no secret may be printed when the key was removed:\n%s", text) + } + for _, want := range []string{"could not be saved", "read-only file system", "It was deleted again", "Retry once the profile is writable"} { + if !strings.Contains(text, want) { + t.Errorf("report lacks %q:\n%s", want, text) + } + } +} + +// TestMbReportUnsavedKey_PrintsSecretOnceWhenRemovalFails covers the fallback: +// the key is live and unusable unless its secret is shown, so it is printed — +// exactly once, with the command that stores it. +func TestMbReportUnsavedKey_PrintsSecretOnceWhenRemovalFails(t *testing.T) { + created, req, b := mbUnsavedKeyFixture() + stubDiscard(t, errors.New("403 forbidden")) + + var out bytes.Buffer + mbReportUnsavedKey(context.Background(), &out, created, req, b, "lsh-me-backups", errors.New("read-only file system")) + text := out.String() + for _, want := range []string{ + "could not be saved", "read-only file system", "nor deleted again", + "Access Key ID:", "AKIAEXAMPLE", + "Secret Access Key:", "s3cr3t/value", + "https://s3.us-central-1.storage.sh", "us-central-1", + "backups-7f3a (rw)", + "lsh s3 access-keys import --name lsh-me-backups --access-key-id AKIAEXAMPLE --project my-project --storage-class standard --bucket backups=rw", + } { + if !strings.Contains(text, want) { + t.Errorf("report lacks %q:\n%s", want, text) + } + } + if strings.Count(text, "s3cr3t/value") != 1 { + t.Errorf("the secret must be printed exactly once:\n%s", text) + } +} diff --git a/cmd/s3/metrics.go b/cmd/s3/metrics.go new file mode 100644 index 0000000..9d37521 --- /dev/null +++ b/cmd/s3/metrics.go @@ -0,0 +1,371 @@ +package s3 + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/spf13/cobra" +) + +// optAllProjects is --all-projects (same spelling as ls). +const optAllProjects = "all-projects" + +// metricsConcurrency bounds the parallel GetStorageBucketMetrics calls. +const metricsConcurrency = 8 + +// MetricsRow is one bucket's consumption and estimated cost for the current +// billing period, as returned by GET /storage/buckets/{id}/metrics. +type MetricsRow struct { + Bucket string `json:"bucket"` + ID string `json:"id"` + Project string `json:"project,omitempty"` + StorageClass string `json:"storage_class,omitempty"` + Site string `json:"site,omitempty"` + CurrentGB *int64 `json:"current_gb"` + ConsumedGB *int64 `json:"consumed_gb"` + Unit string `json:"unit,omitempty"` + CostAmount *float64 `json:"estimated_cost"` + Currency string `json:"currency,omitempty"` + PeriodStart *time.Time `json:"period_start,omitempty"` + PeriodEnd *time.Time `json:"period_end,omitempty"` + // Total marks the aggregated row appended in table mode. + Total bool `json:"-"` + // costs is the per-currency sum of a TOTAL row (several currencies may mix). + costs map[string]float64 +} + +// newMetricsRow converts an API payload for bucket b. +func newMetricsRow(b *objectstorage.Bucket, site string, data *operations.GetStorageBucketMetricsData) MetricsRow { + row := MetricsRow{Bucket: b.Name, ID: b.ID, Project: b.ProjectRef(), StorageClass: b.StorageClass, Site: site} + if row.Site == "" { + row.Site = b.Site + } + if data == nil || data.Attributes == nil { + return row + } + a := data.Attributes + if a.Storage != nil { + row.CurrentGB = a.Storage.Current + row.ConsumedGB = a.Storage.Consumed + if a.Storage.Unit != nil { + row.Unit = string(*a.Storage.Unit) + } + } + if a.EstimatedCost != nil { + row.CostAmount = a.EstimatedCost.Amount + if a.EstimatedCost.Currency != nil { + row.Currency = *a.EstimatedCost.Currency + } + } + if a.Period != nil { + row.PeriodStart = a.Period.Start + row.PeriodEnd = a.Period.End + } + return row +} + +func (m MetricsRow) TableRow() table.Row { + return table.Row{ + "id": {Label: "ID", Value: m.ID}, + "bucket": {Label: "Bucket", Value: m.Bucket}, + "project": {Label: "Project", Value: m.Project}, + "storage_class": {Label: "Class", Value: m.StorageClass}, + "region": {Label: "Site", Value: m.Site}, + "current_gb": {Label: "Current (GB)", Value: gbLabel(m.CurrentGB)}, + "consumed_gb": {Label: "Consumed (GB)", Value: gbLabel(m.ConsumedGB)}, + "estimated_cost": {Label: "Est. Cost", Value: m.costLabel()}, + "period_start": {Label: "Period Start", Value: dateLabel(m.PeriodStart)}, + "period_end": {Label: "Period End", Value: dateLabel(m.PeriodEnd)}, + } +} + +// costLabel renders "12.30 USD" (or the per-currency sums of a TOTAL row). +func (m MetricsRow) costLabel() string { + if m.Total && len(m.costs) > 0 { + currencies := make([]string, 0, len(m.costs)) + for c := range m.costs { + currencies = append(currencies, c) + } + sort.Strings(currencies) + parts := make([]string, 0, len(currencies)) + for _, c := range currencies { + parts = append(parts, formatCost(m.costs[c], c)) + } + return strings.Join(parts, " + ") + } + if m.CostAmount == nil { + return emptyCell + } + return formatCost(*m.CostAmount, m.Currency) +} + +func formatCost(amount float64, currency string) string { + return strings.TrimSpace(fmt.Sprintf("%.2f %s", amount, currency)) +} + +// gbLabel renders the GB figure, or the shared table placeholder when unset. +func gbLabel(v *int64) string { + if v == nil { + return emptyCell + } + return fmt.Sprintf("%d", *v) +} + +func dateLabel(t *time.Time) string { + if t == nil || t.IsZero() { + return "" + } + // The API reports billing-period boundaries in UTC; rendering them in the + // local zone shifts them a day west of Greenwich. + return t.UTC().Format("2006-01-02") +} + +// costOf returns the estimated cost or 0 when unknown. +func (m MetricsRow) costOf() float64 { + if m.CostAmount == nil { + return 0 + } + return *m.CostAmount +} + +// sortMetricsRows orders rows by estimated cost (desc), then consumed GB +// (desc), then bucket name so the output is stable. +func sortMetricsRows(rows []MetricsRow) { + sort.SliceStable(rows, func(i, j int) bool { + ci, cj := rows[i].costOf(), rows[j].costOf() + if ci != cj { + return ci > cj + } + gi, gj := int64Of(rows[i].ConsumedGB), int64Of(rows[j].ConsumedGB) + if gi != gj { + return gi > gj + } + return rows[i].Bucket < rows[j].Bucket + }) +} + +func int64Of(v *int64) int64 { + if v == nil { + return 0 + } + return *v +} + +// metricsTotal sums current/consumed GB and the cost per currency across rows. +// Period columns are filled when every row shares the same period. +func metricsTotal(rows []MetricsRow) MetricsRow { + total := MetricsRow{Bucket: "TOTAL", Total: true, costs: map[string]float64{}} + var current, consumed int64 + var haveCurrent, haveConsumed bool + var currency string + var samePeriod = true + for i, r := range rows { + if r.CurrentGB != nil { + current += *r.CurrentGB + haveCurrent = true + } + if r.ConsumedGB != nil { + consumed += *r.ConsumedGB + haveConsumed = true + } + if r.CostAmount != nil { + total.costs[r.Currency] += *r.CostAmount + } + if r.Unit != "" && total.Unit == "" { + total.Unit = r.Unit + } + if i == 0 { + total.PeriodStart, total.PeriodEnd = r.PeriodStart, r.PeriodEnd + } else if !sameTime(total.PeriodStart, r.PeriodStart) || !sameTime(total.PeriodEnd, r.PeriodEnd) { + samePeriod = false + } + } + if haveCurrent { + total.CurrentGB = ¤t + } + if haveConsumed { + total.ConsumedGB = &consumed + } + if len(total.costs) == 1 { + for c, amount := range total.costs { + currency = c + a := amount + total.CostAmount = &a + } + total.Currency = currency + } + if !samePeriod { + total.PeriodStart, total.PeriodEnd = nil, nil + } + return total +} + +func sameTime(a, b *time.Time) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return a.Equal(*b) +} + +// metricsResult pairs a bucket with its metrics or the error fetching them. +type metricsResult struct { + Bucket *objectstorage.Bucket + Row MetricsRow + Err error +} + +// fetchMetrics calls GetStorageBucketMetrics for every bucket with bounded +// concurrency. Per-bucket failures do not stop the others; results keep the +// input order. +func fetchMetrics(ctx context.Context, api *sdk.Latitudesh, buckets []*objectstorage.Bucket, sites map[string]string, opts []operations.Option) []metricsResult { + results := make([]metricsResult, len(buckets)) + sem := make(chan struct{}, metricsConcurrency) + var wg sync.WaitGroup + for i, b := range buckets { + wg.Add(1) + go func(i int, b *objectstorage.Bucket) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + res := metricsResult{Bucket: b} + resp, err := api.ObjectStorage.GetStorageBucketMetrics(ctx, b.ID, opts...) + if err != nil { + res.Err = objectstorage.HumanizeAPI(err, fmt.Sprintf("metrics for bucket %s", b.Display())) + } else { + var data *operations.GetStorageBucketMetricsData + if resp.Object != nil { + data = resp.Object.Data + } + res.Row = newMetricsRow(b, sites[b.ID], data) + } + results[i] = res + }(i, b) + } + wg.Wait() + return results +} + +// NewMetricsCmd builds `lsh s3 metrics [s3://bucket]`. +func NewMetricsCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "metrics [s3://bucket]", + GroupID: groupReports, + Short: "Current size and estimated cost of a bucket", + Long: `Show the storage consumed in the current billing period and the estimated +cost, per bucket. Without a bucket every bucket of the project (or of the team +with --all-projects) is listed, sorted by estimated cost, with a TOTAL row in +table output. Structured output (-o json|yaml|csv) stays one record per bucket.`, + Example: ` lsh s3 metrics s3://backups + lsh s3 metrics --project my-project + lsh s3 metrics --all-projects -o json --query '[].{bucket:bucket,cost:estimated_cost}'`, + Args: cobra.MaximumNArgs(1), + RunE: runMetrics, + }) + addProjectFlag(cmd, true, "only buckets of this project (ID or slug)") + addBucketFilterFlags(cmd) + cmd.Flags().Bool(optAllProjects, false, "buckets of every project of the team") + return cmd +} + +func runMetrics(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "metrics come from the Latitude API; unset --endpoint-url / %s to use this command", objectstorage.EnvEndpointURL)) + } + allProjects, _ := cmd.Flags().GetBool(optAllProjects) + if allProjects && cmd.Flags().Changed(flagProject) { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s and --%s are mutually exclusive", flagProject, optAllProjects)) + } + + var buckets []*objectstorage.Bucket + var sites map[string]string + if len(args) == 1 { + ref, err := objectstorage.ParseBucketOnly(args[0]) + if err != nil { + return printErr(err) + } + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + if s, err := objectstorage.RawBucketSites(ctx, b.ID); err == nil { + sites = s + } else { + lsh.LogDebugf("metrics: could not fetch site for %s: %v", b.ID, err) + } + buckets = []*objectstorage.Bucket{b} + } else { + r := newResolver(cmd) + if allProjects { + r.Project = "" + } + list, err := r.ListBuckets(ctx) + if err != nil { + return printErr(err) + } + // Scope the site lookup to the project when one is known; only + // --all-projects needs the team-wide listing. + if s, err := objectstorage.RawBucketSitesForProject(ctx, "", r.Project); err == nil { + sites = s + } else { + lsh.LogDebugf("metrics: could not fetch bucket sites: %v", err) + } + // --storage-class/--site also narrow the multi-bucket listing, not just + // a single ambiguous name. + list = filterBucketsByClass(list, r.ClassFilter) + for _, d := range list { + b := objectstorage.BucketFromData(d) + if r.SiteFilter != "" { + site := b.Site + if site == "" && d.ID != nil { + site = sites[*d.ID] + } + if !strings.EqualFold(site, r.SiteFilter) { + continue + } + } + buckets = append(buckets, b) + } + } + + results := fetchMetrics(ctx, apiClient(), buckets, sites, []operations.Option{operations.WithRetries(lsh.RetryConfig())}) + rows := make([]MetricsRow, 0, len(results)) + var failed int + var firstErr error + for _, res := range results { + if res.Err != nil { + failed++ + if firstErr == nil { + firstErr = res.Err + } + objectstorage.Warnf("metrics unavailable for %s: %v", res.Bucket.Display(), res.Err) + continue + } + rows = append(rows, res.Row) + } + if failed > 0 && len(rows) == 0 { + return printErr(firstErr) + } + + sortMetricsRows(rows) + if isHuman() && len(rows) > 1 { + rows = append(rows, metricsTotal(rows)) + } + render(objectstorage.AsResponseData(rows)) + if failed > 0 { + return printErr(exitcode.Errorf(exitcode.Partial, "metrics unavailable for %d of %d buckets", failed, len(results))) + } + return nil +} diff --git a/cmd/s3/metrics_test.go b/cmd/s3/metrics_test.go new file mode 100644 index 0000000..f2b33fb --- /dev/null +++ b/cmd/s3/metrics_test.go @@ -0,0 +1,249 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func ptrF(f float64) *float64 { return &f } + +func metricsFixture() []MetricsRow { + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2026, 9, 30, 23, 59, 59, 0, time.UTC) + return []MetricsRow{ + {Bucket: "cheap", ID: "bkt_1", CurrentGB: ptrI(10), ConsumedGB: ptrI(12), Unit: "GB", CostAmount: ptrF(1.5), Currency: "USD", PeriodStart: &start, PeriodEnd: &end}, + {Bucket: "big", ID: "bkt_2", CurrentGB: ptrI(500), ConsumedGB: ptrI(480), Unit: "GB", CostAmount: ptrF(30), Currency: "USD", PeriodStart: &start, PeriodEnd: &end}, + {Bucket: "free", ID: "bkt_3", CurrentGB: ptrI(0), ConsumedGB: ptrI(0), Unit: "GB", CostAmount: ptrF(0), Currency: "USD", PeriodStart: &start, PeriodEnd: &end}, + {Bucket: "brl", ID: "bkt_4", CurrentGB: ptrI(100), ConsumedGB: ptrI(90), Unit: "GB", CostAmount: ptrF(30), Currency: "BRL", PeriodStart: &start, PeriodEnd: &end}, + {Bucket: "unknown", ID: "bkt_5"}, // metrics payload without attributes + } +} + +func TestSortMetricsRows(t *testing.T) { + rows := metricsFixture() + sortMetricsRows(rows) + var got []string + for _, r := range rows { + got = append(got, r.Bucket) + } + // Cost desc; ties (30 USD vs 30 BRL) by consumed desc; zero-cost rows by + // consumed, then name. + want := []string{"big", "brl", "cheap", "free", "unknown"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("order = %v, want %v", got, want) + } +} + +func TestMetricsTotal(t *testing.T) { + rows := metricsFixture() + total := metricsTotal(rows) + if total.Bucket != "TOTAL" || !total.Total { + t.Errorf("total row not marked: %+v", total) + } + if total.CurrentGB == nil || *total.CurrentGB != 610 { + t.Errorf("current_gb = %v, want 610", total.CurrentGB) + } + if total.ConsumedGB == nil || *total.ConsumedGB != 582 { + t.Errorf("consumed_gb = %v, want 582", total.ConsumedGB) + } + if total.costs["USD"] != 31.5 || total.costs["BRL"] != 30 { + t.Errorf("costs = %v", total.costs) + } + // Mixed currencies: no single amount, label lists both (sorted). + if total.CostAmount != nil { + t.Errorf("mixed currencies must not collapse into one amount: %v", *total.CostAmount) + } + if got := total.TableRow()["estimated_cost"].Value; got != "30.00 BRL + 31.50 USD" { + t.Errorf("cost label = %q", got) + } + // Same period on every row (rows without a period do not count). + if total.PeriodStart != nil && total.TableRow()["period_start"].Value == "" { + t.Errorf("period should be kept when uniform") + } + + // Single currency: amount and currency are filled. + single := metricsTotal(rows[:3]) + if single.CostAmount == nil || *single.CostAmount != 31.5 || single.Currency != "USD" { + t.Errorf("single-currency total = %+v", single) + } + if got := single.TableRow()["estimated_cost"].Value; got != "31.50 USD" { + t.Errorf("single-currency label = %q", got) + } + if single.TableRow()["period_start"].Value == "" || single.TableRow()["period_end"].Value == "" { + t.Errorf("uniform period should be shown on the TOTAL row") + } + + // Rows with no metrics at all yield a total without numbers. + empty := metricsTotal([]MetricsRow{{Bucket: "x"}, {Bucket: "y"}}) + if empty.CurrentGB != nil || empty.ConsumedGB != nil || empty.CostAmount != nil { + t.Errorf("empty total should have nil numbers: %+v", empty) + } + // Empty numbers render the shared "-" placeholder used by every s3 table. + if got := empty.TableRow()["estimated_cost"].Value; got != emptyCell { + t.Errorf("empty cost label = %q, want %q", got, emptyCell) + } + if got := empty.TableRow()["current_gb"].Value; got != emptyCell { + t.Errorf("empty current_gb label = %q, want %q", got, emptyCell) + } +} + +func TestMetricsRowJSONShape(t *testing.T) { + rows := metricsFixture() + raw, err := json.Marshal(rows[0]) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"bucket":"cheap"`, `"id":"bkt_1"`, `"current_gb":10`, `"consumed_gb":12`, `"estimated_cost":1.5`, `"currency":"USD"`, `"period_start":`} { + if !strings.Contains(string(raw), want) { + t.Errorf("json %s lacks %s", raw, want) + } + } + // The TOTAL marker never leaks into structured output. + total := metricsTotal(rows) + raw, _ = json.Marshal(total) + if strings.Contains(string(raw), "Total") || strings.Contains(string(raw), "costs") { + t.Errorf("total internals leaked: %s", raw) + } +} + +// fakeMetricsAPI serves GET /storage/buckets/{id}/metrics; bkt_err fails +// with 500 and bkt_missing with 404. It records the peak concurrency. +func fakeMetricsAPI(t *testing.T, peak *int32) *httptest.Server { + var inflight int32 + var mu sync.Mutex + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cur := atomic.AddInt32(&inflight, 1) + defer atomic.AddInt32(&inflight, -1) + mu.Lock() + if cur > *peak { + *peak = cur + } + mu.Unlock() + time.Sleep(20 * time.Millisecond) + + if r.Header.Get("Authorization") == "" { + t.Errorf("missing Authorization header") + } + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(parts) != 4 || parts[0] != "storage" || parts[1] != "buckets" || parts[3] != "metrics" { + http.Error(w, "unexpected path "+r.URL.Path, http.StatusNotFound) + return + } + id := parts[2] + w.Header().Set("Content-Type", "application/vnd.api+json") + switch id { + case "bkt_err": + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"errors":[{"status":"500","title":"Internal Server Error","detail":"boom"}]}`) + return + case "bkt_missing": + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, `{"errors":[{"status":"404","title":"Not Found","detail":"Couldn't find bucket"}]}`) + return + } + n := strings.TrimPrefix(id, "bkt_") + fmt.Fprintf(w, `{"data":{"id":%q,"type":"object_storage_metrics","attributes":{"period":{"start":"2026-09-01T00:00:00Z","end":"2026-09-30T23:59:59Z"},"storage":{"consumed":%s0,"current":%s,"unit":"GB"},"estimated_cost":{"amount":%s.5,"currency":"USD"}}}}`, id, n, n, n) + })) +} + +func TestFetchMetricsFanOut(t *testing.T) { + var peak int32 + srv := fakeMetricsAPI(t, &peak) + defer srv.Close() + + api := sdk.New(sdk.WithServerURL(srv.URL), sdk.WithSecurity("test-token")) + var buckets []*objectstorage.Bucket + for i := 1; i <= 20; i++ { + buckets = append(buckets, &objectstorage.Bucket{ID: fmt.Sprintf("bkt_%d", i), Name: fmt.Sprintf("b%d", i), StorageClass: "standard", ProjectSlug: "proj"}) + } + buckets = append(buckets, + &objectstorage.Bucket{ID: "bkt_err", Name: "broken"}, + &objectstorage.Bucket{ID: "bkt_missing", Name: "gone"}, + ) + sites := map[string]string{"bkt_1": "DAL", "bkt_2": "TYO4"} + + results := fetchMetrics(context.Background(), api, buckets, sites, nil) + if len(results) != len(buckets) { + t.Fatalf("got %d results, want %d", len(results), len(buckets)) + } + if peak > metricsConcurrency { + t.Errorf("peak concurrency %d exceeds %d", peak, metricsConcurrency) + } + if peak < 2 { + t.Errorf("expected parallel requests, peak was %d", peak) + } + + var failed int + for i, res := range results { + if res.Bucket != buckets[i] { + t.Errorf("result %d out of order: %s", i, res.Bucket.ID) + } + switch res.Bucket.ID { + case "bkt_err": + failed++ + if res.Err == nil || exitcode.Of(res.Err) != exitcode.Generic { + t.Errorf("bkt_err: err = %v", res.Err) + } + case "bkt_missing": + failed++ + if res.Err == nil || exitcode.Of(res.Err) != exitcode.NotFound { + t.Errorf("bkt_missing: err = %v (exit %d)", res.Err, exitcode.Of(res.Err)) + } else if !strings.Contains(res.Err.Error(), "gone (bkt_missing)") || !strings.Contains(res.Err.Error(), "Couldn't find bucket") { + t.Errorf("bkt_missing: message should name the bucket and carry the API detail: %v", res.Err) + } + default: + if res.Err != nil { + t.Errorf("%s: unexpected error %v", res.Bucket.ID, res.Err) + continue + } + n := strings.TrimPrefix(res.Bucket.ID, "bkt_") + if res.Row.Bucket != res.Bucket.Name || res.Row.ID != res.Bucket.ID || res.Row.Project != "proj" { + t.Errorf("row identity wrong: %+v", res.Row) + } + if res.Row.CurrentGB == nil || fmt.Sprint(*res.Row.CurrentGB) != n { + t.Errorf("%s current_gb = %v", res.Bucket.ID, res.Row.CurrentGB) + } + if res.Row.CostAmount == nil || res.Row.Currency != "USD" { + t.Errorf("%s cost = %v %s", res.Bucket.ID, res.Row.CostAmount, res.Row.Currency) + } + if res.Row.PeriodStart == nil || res.Row.PeriodEnd == nil { + t.Errorf("%s period missing", res.Bucket.ID) + } + } + } + if failed != 2 { + t.Errorf("failed = %d, want 2", failed) + } + if results[0].Row.Site != "DAL" || results[1].Row.Site != "TYO4" || results[2].Row.Site != "" { + t.Errorf("site mapping wrong: %q %q %q", results[0].Row.Site, results[1].Row.Site, results[2].Row.Site) + } +} + +// TestDateLabelIsUTC covers the billing-period boundary fix: the API reports +// them in UTC, so rendering them in the local zone moved them a day west of +// Greenwich (UTC-3 showed the previous month's last day). +func TestDateLabelIsUTC(t *testing.T) { + restore := time.Local + time.Local = time.FixedZone("UTC-3", -3*60*60) + defer func() { time.Local = restore }() + + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + if got := dateLabel(&start); got != "2026-09-01" { + t.Errorf("dateLabel = %q, want 2026-09-01 (the UTC date the API reported)", got) + } + if got := dateLabel(nil); got != "" { + t.Errorf("dateLabel(nil) = %q, want empty", got) + } +} diff --git a/cmd/s3/mv.go b/cmd/s3/mv.go new file mode 100644 index 0000000..1df96d4 --- /dev/null +++ b/cmd/s3/mv.go @@ -0,0 +1,37 @@ +package s3 + +import ( + "github.com/spf13/cobra" +) + +// NewMvCmd builds `lsh s3 move `. +func NewMvCmd() *cobra.Command { + var f transferFlags + cmd := newCmd(&cobra.Command{ + Use: "move ", + Aliases: []string{"mv"}, + GroupID: groupObjects, + Short: "Move files and objects (alias: mv)", + Long: `Move a local file into a bucket, an object to the local disk, or an object to +another key (server-side, same endpoint; buckets that resolve different access +keys are streamed through this machine). Each source is deleted only after +its copy succeeded; a failed copy leaves the source untouched. Both operands +local, or source and destination naming the same object, is refused. + +` + destinationRulesHelp + ` + +All cp flags apply (--recursive, --exclude/--include, --content-type, +--metadata, --no-overwrite, --quiet, ...). Output lines use the "move:" +prefix. --dry-run (or --dry-run) prints the plan without copying or deleting.`, + Example: ` lsh s3 move ./dump.sql s3://backups/2026/09/ + lsh s3 move s3://backups/tmp/report.pdf ./reports/ + lsh s3 move s3://backups/tmp/ s3://backups/archive/2026/ --recursive + lsh s3 move ./logs s3://logs/host-1/ --recursive --exclude "*" --include "*.log" --dry-run`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return runCopyCommand(cmd, args, &f, true) + }, + }) + addTransferFlags(cmd, &f, true) + return cmd +} diff --git a/cmd/s3/petname.go b/cmd/s3/petname.go new file mode 100644 index 0000000..fec0047 --- /dev/null +++ b/cmd/s3/petname.go @@ -0,0 +1,56 @@ +package s3 + +import ( + "crypto/rand" + "math/big" + + "github.com/latitudesh/lsh/internal/objectstorage" +) + +// Access key names follow the same "pet name" pattern the dashboard uses when +// it auto-suggests a name: key---, e.g. key-brave-otter-std. +// There is no random numeric segment; uniqueness is enforced per project, so a +// candidate that collides is simply re-rolled (see createAccessKeyRetrying). +// The names satisfy the API rules (lowercase, 3-25 chars, start/end alphanumeric). + +var petAdjectives = []string{ + "amber", "azure", "bold", "brave", "brisk", "calm", "clever", "cosmic", + "crimson", "daring", "deep", "eager", "electric", "gentle", "golden", "happy", + "jolly", "keen", "lively", "lucky", "mellow", "merry", "mighty", "noble", + "plum", "proud", "quiet", "rapid", "royal", "shiny", "silent", "sleek", + "smart", "snappy", "solar", "spry", "stellar", "sturdy", "sunny", "swift", + "teal", "tidy", "vivid", "warm", "wise", "witty", "young", "zesty", +} + +var petNouns = []string{ + "otter", "falcon", "harbor", "forest", "river", "meadow", "comet", "canyon", + "maple", "willow", "cedar", "cobra", "lynx", "heron", "raven", "badger", + "marlin", "puma", "bison", "koala", "gecko", "panda", "tiger", "walrus", + "zephyr", "summit", "delta", "ember", "glacier", "harvest", "island", "jungle", + "lagoon", "monsoon", "nebula", "orchid", "pebble", "quartz", "ridge", "sequoia", + "tundra", "vortex", "wave", "beacon", "boulder", "cascade", "dune", "fjord", +} + +// tierAbbr maps a storage class to the dashboard's short suffix. +func tierAbbr(storageClass string) string { + if storageClass == objectstorage.ClassHighPerformance { + return "hp" + } + return "std" +} + +// generateKeyName returns a fresh pet name for the given storage class, capped +// at the API's 25-character limit. +func generateKeyName(storageClass string) string { + return capKeyName("key-" + pick(petAdjectives) + "-" + pick(petNouns) + "-" + tierAbbr(storageClass)) +} + +// pick returns a uniformly random element of list using crypto/rand, falling +// back to the first element only if the source ever fails. +func pick(list []string) string { + n, err := rand.Int(rand.Reader, big.NewInt(int64(len(list)))) + if err != nil { + return list[0] + } + return list[n.Int64()] +} diff --git a/cmd/s3/presign.go b/cmd/s3/presign.go new file mode 100644 index 0000000..befddfa --- /dev/null +++ b/cmd/s3/presign.go @@ -0,0 +1,220 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/spf13/cobra" +) + +// presignMaxExpiry is the SigV4 ceiling for pre-signed URLs (7 days). +const presignMaxExpiry = 7 * 24 * time.Hour + +// presignOptions are the parsed flags of `lsh s3 presign`. +type presignOptions struct { + Method string + Expires time.Duration + VersionID string + ContentType string +} + +// NewPresignCmd builds `lsh s3 presign s3://bucket/key`. +func NewPresignCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "presign s3://bucket/key", + Aliases: []string{"presigned-url", "share"}, + GroupID: groupObjects, + Short: "Generate a pre-signed URL for an object", + Long: `Generate a pre-signed URL that grants temporary access to one object +without sharing credentials. + +The URL is signed locally with the access key the CLI selects for the bucket +(the only network call is the bucket lookup). --expires-in accepts seconds +(3600) or a duration with an s/m/h/d suffix (15m, 1h, 7d); the maximum is 7 +days (604800 seconds). --method PUT signs an upload URL and requires a key +with write permission. In the default output only the URL is printed, so the +command composes with $(...) and curl.`, + Example: ` lsh s3 presign s3://backups/report.pdf + lsh s3 presign s3://backups/report.pdf --expires-in 15m + curl -o report.pdf "$(lsh s3 presign s3://backups/report.pdf)" + lsh s3 presign s3://uploads/in.bin --method PUT --expires-in 1h + lsh s3 presign s3://backups/report.pdf -o json`, + Args: cobra.ExactArgs(1), + }) + f := cmd.Flags() + f.String("expires-in", "3600", "how long the URL stays valid: seconds or a duration (15m, 1h, 7d); max 7d") + f.String("method", "GET", "HTTP method to sign: GET, PUT or HEAD") + f.String("version-id", "", "sign a URL for this specific version (GET/HEAD only)") + f.String("content-type", "", "content type the uploader will send (PUT only; informational, see notes)") + addProjectFlag(cmd, true, "project the bucket belongs to (disambiguates buckets with the same name)") + addBucketFilterFlags(cmd) + rejectRegionFlag(cmd) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + expiresIn, _ := f.GetString("expires-in") + method, _ := f.GetString("method") + versionID, _ := f.GetString("version-id") + contentType, _ := f.GetString("content-type") + + // ObjectRef's shared message points at --recursive, a flag only rm and + // the transfer commands have; presign signs exactly one object. + if r, parseErr := objectstorage.ParseRemote(args[0]); parseErr == nil && strings.HasSuffix(r.Key, "/") { + return printErr(objectstorage.ErrUsagef("key %q ends with '/'; presign signs a single object (use 'lsh s3 list %s' to list the prefix)", r.Key, r)) + } + ref, err := objectstorage.ObjectRef(args[0], false) + if err != nil { + return printErr(err) + } + opts, err := presignParseOptions(method, expiresIn, versionID, contentType) + if err != nil { + return printErr(err) + } + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + sess, err := openBucket(ctx, cmd, ref.Bucket, opts.Method == http.MethodPut) + if err != nil { + return printErr(err) + } + res, err := presignObject(ctx, sess, ref.Key, opts) + if err != nil { + return printErr(err) + } + if isHuman() { + presignWrite(os.Stdout, res) + return nil + } + render([]renderer.ResponseData{res}) + return nil + } + return cmd +} + +// presignParseOptions validates the flag values. +func presignParseOptions(method, expiresIn, versionID, contentType string) (presignOptions, error) { + opts := presignOptions{VersionID: versionID, ContentType: contentType} + opts.Method = strings.ToUpper(strings.TrimSpace(method)) + switch opts.Method { + case http.MethodGet, http.MethodPut, http.MethodHead: + case "": + opts.Method = http.MethodGet + default: + return opts, objectstorage.ErrUsagef("--method must be GET, PUT or HEAD (got %q)", method) + } + d, err := parseExpiresIn(expiresIn) + if err != nil { + return opts, err + } + opts.Expires = d + if opts.VersionID != "" && opts.Method == http.MethodPut { + return opts, objectstorage.ErrUsagef("--version-id only applies to GET and HEAD URLs") + } + if opts.ContentType != "" && opts.Method != http.MethodPut { + return opts, objectstorage.ErrUsagef("--content-type only applies to --method PUT") + } + return opts, nil +} + +var expiresInRe = regexp.MustCompile(`^(\d+)\s*([smhd])$`) + +// parseExpiresIn accepts a plain integer (seconds) or a duration with an +// s/m/h/d suffix (15m, 1h, 7d; composite Go durations such as 1h30m also +// work). Values above 7 days are a usage error rather than being clipped. +func parseExpiresIn(s string) (time.Duration, error) { + v := strings.TrimSpace(s) + if v == "" { + return 0, objectstorage.ErrUsagef("--expires-in cannot be empty") + } + var d time.Duration + switch { + case isDigits(v): + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return 0, objectstorage.ErrUsagef("invalid --expires-in %q: %v", s, err) + } + d = time.Duration(n) * time.Second + default: + if m := expiresInRe.FindStringSubmatch(v); m != nil { + n, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + return 0, objectstorage.ErrUsagef("invalid --expires-in %q: %v", s, err) + } + unit := map[string]time.Duration{"s": time.Second, "m": time.Minute, "h": time.Hour, "d": 24 * time.Hour}[m[2]] + d = time.Duration(n) * unit + } else { + parsed, err := time.ParseDuration(v) + if err != nil { + return 0, objectstorage.ErrUsagef("invalid --expires-in %q: use seconds (3600) or a duration such as 15m, 1h or 7d", s) + } + d = parsed + } + } + if d < time.Second { + return 0, objectstorage.ErrUsagef("--expires-in must be at least 1 second (got %q)", s) + } + if d > presignMaxExpiry { + return 0, objectstorage.ErrUsagef("--expires-in %q exceeds the 7 day maximum (604800 seconds) for pre-signed URLs", s) + } + return d, nil +} + +func isDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// presignObject signs the URL locally for the given method. +func presignObject(ctx context.Context, sess *Session, key string, opts presignOptions) (objectstorage.PresignResult, error) { + if opts.ContentType != "" && opts.Method == http.MethodPut { + objectstorage.Warnf("--content-type is not part of the signature; the uploader must send Content-Type: %s itself (a mismatch is accepted by most backends but not enforced)", opts.ContentType) + } + params := url.Values{} + if opts.VersionID != "" { + params.Set("versionId", opts.VersionID) + } + expiresAt := time.Now().Add(opts.Expires) + var ( + u *url.URL + err error + ) + switch opts.Method { + case http.MethodPut: + u, err = sess.Client.PresignedPutObject(ctx, sess.Bucket.BucketName, key, opts.Expires) + case http.MethodHead: + u, err = sess.Client.PresignedHeadObject(ctx, sess.Bucket.BucketName, key, opts.Expires, params) + default: + u, err = sess.Client.PresignedGetObject(ctx, sess.Bucket.BucketName, key, opts.Expires, params) + } + if err != nil { + return objectstorage.PresignResult{}, sess.humanize(err) + } + if u == nil { + return objectstorage.PresignResult{}, exitcode.Errorf(exitcode.Generic, "presigning returned no URL") + } + return objectstorage.PresignResult{URL: u.String(), Method: opts.Method, ExpiresAt: expiresAt}, nil +} + +// presignWrite prints the human output: only the URL, so the command composes +// with $(...) and pipes. +func presignWrite(w io.Writer, res objectstorage.PresignResult) { + fmt.Fprintln(w, res.URL) +} diff --git a/cmd/s3/presign_test.go b/cmd/s3/presign_test.go new file mode 100644 index 0000000..59f460a --- /dev/null +++ b/cmd/s3/presign_test.go @@ -0,0 +1,261 @@ +package s3 + +import ( + "bytes" + "context" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" +) + +func presignTestSession(t *testing.T, srv *s3test.Server, region string) *Session { + t.Helper() + b := &objectstorage.Bucket{ID: "bkt_1", Name: "b", BucketName: "backups-7f3a", Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: region} + cred := objectstorage.NewCredential("AKIAEXAMPLE", "topsecret", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +func TestParseExpiresIn(t *testing.T) { + cases := []struct { + in string + want time.Duration + wantErr string + }{ + {in: "3600", want: time.Hour}, + {in: "1", want: time.Second}, + {in: "604800", want: 7 * 24 * time.Hour}, + {in: "15m", want: 15 * time.Minute}, + {in: "90s", want: 90 * time.Second}, + {in: "1h", want: time.Hour}, + {in: "7d", want: 7 * 24 * time.Hour}, + {in: "1h30m", want: 90 * time.Minute}, + {in: " 2h ", want: 2 * time.Hour}, + {in: "8d", wantErr: "exceeds the 7 day maximum"}, + {in: "604801", wantErr: "exceeds the 7 day maximum"}, + {in: "0", wantErr: "at least 1 second"}, + {in: "junk", wantErr: "invalid --expires-in"}, + {in: "10x", wantErr: "invalid --expires-in"}, + {in: "-5", wantErr: "invalid --expires-in"}, + {in: "", wantErr: "cannot be empty"}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + got, err := parseExpiresIn(tc.in) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("expected error, got %v", got) + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("exit code = %d, want %d", exitcode.Of(err), exitcode.Usage) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("error %q does not mention %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("got %v, want %v", got, tc.want) + } + }) + } +} + +func TestPresignParseOptions(t *testing.T) { + if o, err := presignParseOptions("get", "3600", "", ""); err != nil || o.Method != http.MethodGet || o.Expires != time.Hour { + t.Errorf("lower-case get: %+v %v", o, err) + } + if o, err := presignParseOptions("", "60", "", ""); err != nil || o.Method != http.MethodGet { + t.Errorf("empty method defaults to GET: %+v %v", o, err) + } + if o, err := presignParseOptions("head", "60", "v1", ""); err != nil || o.Method != http.MethodHead || o.VersionID != "v1" { + t.Errorf("head with version: %+v %v", o, err) + } + if _, err := presignParseOptions("POST", "60", "", ""); exitcode.Of(err) != exitcode.Usage { + t.Errorf("POST must be a usage error, got %v", err) + } + if _, err := presignParseOptions("PUT", "60", "v1", ""); exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--version-id") { + t.Errorf("PUT with version-id must be a usage error, got %v", err) + } + if _, err := presignParseOptions("GET", "60", "", "text/plain"); exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--content-type") { + t.Errorf("GET with content-type must be a usage error, got %v", err) + } + if o, err := presignParseOptions("put", "60", "", "text/plain"); err != nil || o.ContentType != "text/plain" { + t.Errorf("PUT with content-type: %+v %v", o, err) + } + if _, err := presignParseOptions("GET", "8d", "", ""); exitcode.Of(err) != exitcode.Usage { + t.Errorf("8d must be a usage error, got %v", err) + } +} + +func TestPresignGetURL(t *testing.T) { + srv := s3test.New() + defer srv.Close() + sess := presignTestSession(t, srv, "eu-west-2") + + before := time.Now() + res, err := presignObject(context.Background(), sess, "2026/09/report.pdf", presignOptions{Method: http.MethodGet, Expires: 15 * time.Minute}) + if err != nil { + t.Fatal(err) + } + if res.Method != http.MethodGet { + t.Errorf("method = %q", res.Method) + } + if res.ExpiresAt.Before(before.Add(15*time.Minute-time.Second)) || res.ExpiresAt.After(time.Now().Add(15*time.Minute+time.Second)) { + t.Errorf("expires_at = %v", res.ExpiresAt) + } + u, err := url.Parse(res.URL) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(res.URL, srv.URL()) { + t.Errorf("url %q not on endpoint %s", res.URL, srv.URL()) + } + if u.Path != "/backups-7f3a/2026/09/report.pdf" { + t.Errorf("path = %q, want path-style //", u.Path) + } + q := u.Query() + if q.Get("X-Amz-Expires") != "900" { + t.Errorf("X-Amz-Expires = %q", q.Get("X-Amz-Expires")) + } + if cred := q.Get("X-Amz-Credential"); !strings.Contains(cred, "/eu-west-2/s3/aws4_request") || !strings.HasPrefix(cred, "AKIAEXAMPLE/") { + t.Errorf("X-Amz-Credential = %q", cred) + } + if q.Get("X-Amz-Algorithm") != "AWS4-HMAC-SHA256" || q.Get("X-Amz-Signature") == "" { + t.Errorf("missing SigV4 query parameters: %v", q) + } + if q.Has("versionId") { + t.Error("versionId must be absent when not requested") + } + if strings.Contains(res.URL, "topsecret") { + t.Error("URL leaked the secret") + } + // Presigning is local: the bucket is never contacted. + if n := len(srv.Requests()); n != 0 { + t.Errorf("%d requests sent while presigning, want 0", n) + } + + var out bytes.Buffer + presignWrite(&out, res) + if out.String() != res.URL+"\n" { + t.Errorf("human output = %q", out.String()) + } +} + +func TestPresignVersionIDAndHead(t *testing.T) { + srv := s3test.New() + defer srv.Close() + sess := presignTestSession(t, srv, "us-east-1") + + res, err := presignObject(context.Background(), sess, "k", presignOptions{Method: http.MethodHead, Expires: time.Hour, VersionID: "3HL4kqtJlcpXroDTDmJ"}) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(res.URL) + if err != nil { + t.Fatal(err) + } + if u.Query().Get("versionId") != "3HL4kqtJlcpXroDTDmJ" { + t.Errorf("versionId missing: %s", res.URL) + } + if u.Query().Get("X-Amz-Expires") != "3600" { + t.Errorf("X-Amz-Expires = %q", u.Query().Get("X-Amz-Expires")) + } + if res.Method != http.MethodHead { + t.Errorf("method = %q", res.Method) + } +} + +func TestPresignPutURL(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + sess := presignTestSession(t, srv, "us-east-1") + + res, err := presignObject(context.Background(), sess, "in/upload.bin", presignOptions{Method: http.MethodPut, Expires: 7 * 24 * time.Hour}) + if err != nil { + t.Fatal(err) + } + if res.Method != http.MethodPut { + t.Errorf("method = %q", res.Method) + } + u, err := url.Parse(res.URL) + if err != nil { + t.Fatal(err) + } + if u.Path != "/backups-7f3a/in/upload.bin" || u.Query().Get("X-Amz-Expires") != "604800" { + t.Errorf("url = %s", res.URL) + } + if len(srv.Requests()) != 0 { + t.Error("presigning must not contact the backend") + } + + // The URL actually works against the fake server without credentials. + req, err := http.NewRequest(http.MethodPut, res.URL, strings.NewReader("payload")) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("PUT with presigned URL = %d", resp.StatusCode) + } + if o := srv.Object("backups-7f3a", "in/upload.bin"); o == nil || string(o.Data) != "payload" { + t.Errorf("uploaded object = %+v", o) + } +} + +// TestPresignCommandRejectsRegionFlag guards the aws-compat shim on presign: +// --region must fail with exit 2 and point to --signing-region, before the +// bucket is looked up. +func TestPresignCommandRejectsRegionFlag(t *testing.T) { + cmd := NewPresignCmd() + Finalize(cmd) // production installs this in build_s3.go + cmd.SetArgs([]string{"s3://backups/report.pdf", "--region", "us-east-1"}) + var err error + _, stderr := rmCaptureOutput(t, func() { err = cmd.Execute() }) + if exitcode.Of(err) != exitcode.Usage { + t.Fatalf("exit code = %d, want %d (%v)", exitcode.Of(err), exitcode.Usage, err) + } + if !strings.Contains(stderr, "--region is not supported") || !strings.Contains(stderr, "--signing-region") { + t.Errorf("stderr lacks the directed --region explanation:\n%s", stderr) + } +} + +// TestPresignRejectsPrefixWithoutRecursiveHint covers the message fix: the +// shared object parser points at --recursive, a flag presign does not have. +func TestPresignRejectsPrefixWithoutRecursiveHint(t *testing.T) { + cmd := NewPresignCmd() + Finalize(cmd) + cmd.SetArgs([]string{"s3://backups/2026/"}) + var errBuf bytes.Buffer + cmd.SetErr(&errBuf) + err := cmd.Execute() + if err == nil { + t.Fatal("expected a usage error for a prefix") + } + if code := exitcode.Of(err); code != exitcode.Usage { + t.Errorf("exit code = %d, want %d", code, exitcode.Usage) + } + if strings.Contains(err.Error(), "--recursive") { + t.Errorf("presign has no --recursive; message was %q", err) + } + if !strings.Contains(err.Error(), "single object") { + t.Errorf("message %q should say presign signs one object", err) + } +} diff --git a/cmd/s3/rb.go b/cmd/s3/rb.go new file mode 100644 index 0000000..918658a --- /dev/null +++ b/cmd/s3/rb.go @@ -0,0 +1,359 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + cobra "github.com/spf13/cobra" +) + +// Flag names of rb. +const ( + flagRbForce = "force" + flagRbVersions = "versions" + flagRbBypass = "bypass-governance-retention" + flagRbMaxDelete = "max-delete" +) + +// rbDeleteBatch caps the objects sent per multi-delete request. +const rbDeleteBatch = 1000 + +// rbDeleteBucketAPI deletes the bucket record through the Latitude API. It is +// a variable so tests can stub the control plane. +var rbDeleteBucketAPI = func(ctx context.Context, b *objectstorage.Bucket) error { + client := apiClient() + _, err := client.ObjectStorage.DeleteStorageBuckets(ctx, b.ID, operations.WithRetries(lsh.RetryConfig())) + if err != nil { + return rbHumanizeAPIDelete(objectstorage.Humanize(err, nil, nil), b) + } + return nil +} + +// rbHumanizeAPIDelete adds the --force hint to "not empty" refusals from the +// API (409/422 mentioning not empty, or an S3 BucketNotEmpty passthrough). +func rbHumanizeAPIDelete(err error, b *objectstorage.Bucket) error { + if err == nil { + return nil + } + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "not empty") || strings.Contains(msg, "bucketnotempty") { + return exitcode.Errorf(exitcode.Refused, "bucket %s is not empty; re-run with --force to delete its objects first (add --versions for versioned buckets)", b.Display()) + } + return err +} + +// rbOptions are the inputs of the --force flow. +type rbOptions struct { + Versions bool + Bypass bool + Yes bool + MaxDelete int64 + DryRun bool + Human bool +} + +// NewRbCmd builds `lsh s3 delete-bucket s3://bucket`. +func NewRbCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "delete-bucket s3://bucket", + Aliases: []string{"rb"}, + GroupID: groupBuckets, + Short: "Delete a bucket (alias: rb)", + Long: `Delete a bucket through the Latitude API. + +Without --force the bucket must be empty; the API refuses otherwise (exit 7). +With --force the CLI lists the objects, asks for confirmation (or --yes), +deletes them and then removes the bucket. Versioned buckets also need +--versions so every object version and delete marker is removed. Buckets with COMPLIANCE object lock cannot be +force-deleted; GOVERNANCE retention needs --bypass-governance-retention. + +--force never skips the confirmation: pass --yes in scripts. Without a TTY and +without --yes the command fails fast with exit 7.`, + Example: ` lsh s3 delete-bucket s3://backups + lsh s3 delete-bucket s3://backups --force + lsh s3 delete-bucket s3://backups --force --versions --yes + lsh s3 delete-bucket s3://backups --force --max-delete 100 --dry-run`, + Args: cobra.ExactArgs(1), + RunE: runRb, + }) + addProjectFlag(cmd, true, "project ID or slug to disambiguate the bucket name") + addYesFlag(cmd) + addBucketFilterFlags(cmd) + cmd.Flags().Bool(flagRbForce, false, "delete all objects in the bucket first") + cmd.Flags().Bool(flagRbVersions, false, "with --force: also delete every object version and delete marker") + cmd.Flags().Bool(flagRbBypass, false, "with --force: bypass GOVERNANCE object lock retention") + cmd.Flags().Int64(flagRbMaxDelete, 0, "with --force: refuse when more than N objects would be deleted (0 = no limit)") + return cmd +} + +func runRb(cmd *cobra.Command, args []string) error { + ref, err := objectstorage.ParseBucketOnly(args[0]) + if err != nil { + return printErr(err) + } + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "rb deletes the bucket through the Latitude API and cannot work with --endpoint-url / %s; unset it and address the bucket by name or bkt_ ID", objectstorage.EnvEndpointURL)) + } + f := cmd.Flags() + force, _ := f.GetBool(flagRbForce) + yes, _ := f.GetBool(flagYes) + o := rbOptions{Yes: yes, DryRun: dryRun(), Human: isHuman()} + o.Versions, _ = f.GetBool(flagRbVersions) + o.Bypass, _ = f.GetBool(flagRbBypass) + o.MaxDelete, _ = f.GetInt64(flagRbMaxDelete) + if o.MaxDelete < 0 { + return printErr(exitcode.Errorf(exitcode.Usage, "--max-delete must be zero or a positive number")) + } + if !force && (o.Versions || o.Bypass || o.MaxDelete > 0) { + return printErr(exitcode.Errorf(exitcode.Usage, "--versions, --bypass-governance-retention and --max-delete only apply with --force")) + } + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + + if !force { + row := objectstorage.DeleteResult{Bucket: b.Name, DryRun: o.DryRun} + if !o.DryRun { + if err := rbDeleteBucketAPI(ctx, b); err != nil { + return printErr(err) + } + row.Deleted = true + } + rbEmitResults(o.Human, os.Stdout, []objectstorage.DeleteResult{row}) + return nil + } + + sess, err := openResolved(cmd, b, true) + if err != nil { + return printErr(err) + } + rows, err := forceRemoveBucket(ctx, cmd, sess, o, os.Stdout) + if !o.Human && len(rows) > 0 { + render(objectstorage.AsResponseData(rows)) + } + if err != nil { + return printErr(err) + } + return nil +} + +// forceRemoveBucket empties the bucket and deletes it (or prints the plan in +// dry-run mode). In human mode the aws-style lines are written to w as they +// happen; otherwise the rows are returned for the caller to render. The +// returned rows are also populated on partial failure so structured output +// still lists what was deleted. +func forceRemoveBucket(ctx context.Context, cmd *cobra.Command, sess *Session, o rbOptions, w io.Writer) ([]objectstorage.DeleteResult, error) { + b := sess.Bucket + if b.Locking && strings.EqualFold(b.RetentionMode, string(operations.RetentionModeCompliance)) { + return nil, exitcode.Errorf(exitcode.Refused, "bucket %s has COMPLIANCE object lock: its objects cannot be deleted until their retention expires, so the bucket cannot be force-deleted", b.Display()) + } + if b.Versioning && !o.Versions { + return nil, exitcode.Errorf(exitcode.Refused, "bucket %s is versioned; add --versions to delete all object versions and delete markers as well", b.Display()) + } + if b.Locking && !o.Bypass && !o.DryRun { + objectstorage.Hintf("note: bucket %s has %s object lock; objects still under retention will fail to delete unless you pass --bypass-governance-retention", b.Display(), strings.ToUpper(b.RetentionMode)) + } + + objects, err := rbListAllObjects(ctx, sess.Client, b.BucketName, o.Versions) + if err != nil { + return nil, sess.humanize(err) + } + if o.MaxDelete > 0 && int64(len(objects)) > o.MaxDelete { + return nil, exitcode.Errorf(exitcode.Refused, "bucket %s holds %s %s, more than --max-delete %d; nothing was deleted", b.Display(), rbCommaInt(int64(len(objects))), rbNoun(o.Versions, len(objects)), o.MaxDelete) + } + + if o.DryRun { + rows := make([]objectstorage.DeleteResult, 0, len(objects)+1) + for _, obj := range objects { + rows = append(rows, rbDeleteRow(b.Name, obj, o.Versions, true)) + } + rows = append(rows, objectstorage.DeleteResult{Bucket: b.Name, DryRun: true}) + if o.Human { + rbEmitResults(true, w, rows) + } + return rows, nil + } + + question := fmt.Sprintf("Delete bucket %s (%s, project %s, backend %s) and its %s %s?", + b.Name, b.ID, b.ProjectRef(), b.BucketName, rbCommaInt(int64(len(objects))), rbNoun(o.Versions, len(objects))) + if len(objects) == 0 { + question = fmt.Sprintf("Delete bucket %s (%s, project %s, backend %s)? It is empty.", b.Name, b.ID, b.ProjectRef(), b.BucketName) + } + if err := objectstorage.ConfirmOrRefuse(cmd, o.Yes, question); err != nil { + return nil, err + } + + rows, deleted, failed := rbDeleteObjects(ctx, sess, objects, o, w) + // An interrupt aborts the batches mid-flight and inflates failed; report + // the signal instead of a partial run (same contract as rm). + if ctx.Err() != nil { + return rows, objectstorage.Humanize(ctx.Err(), sess.Bucket, &sess.Cred) + } + if failed > 0 { + fmt.Fprintf(os.Stderr, "%d deleted, %d failed; bucket %s was not removed\n", deleted, failed, b.Display()) + return rows, exitcode.Errorf(exitcode.Partial, "could not delete every object of bucket %s (%d deleted, %d failed)", b.Display(), deleted, failed) + } + + if err := rbDeleteBucketAPI(ctx, b); err != nil { + return rows, err + } + final := objectstorage.DeleteResult{Bucket: b.Name, Deleted: true} + rows = append(rows, final) + if o.Human { + fmt.Fprintln(w, final.HumanLine()) + } + return rows, nil +} + +// rbListAllObjects enumerates every object (or every version and delete marker +// when withVersions is set) in the bucket. +func rbListAllObjects(ctx context.Context, client *minio.Client, bucket string, withVersions bool) ([]minio.ObjectInfo, error) { + var out []minio.ObjectInfo + for info := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Recursive: true, WithVersions: withVersions}) { + if info.Err != nil { + return nil, info.Err + } + out = append(out, info) + } + return out, nil +} + +// rbDeleteObjects removes the objects in batches. It stops at the first batch +// with a failure. Human mode prints one line per object to w. +func rbDeleteObjects(ctx context.Context, sess *Session, objects []minio.ObjectInfo, o rbOptions, w io.Writer) (rows []objectstorage.DeleteResult, deleted, failed int) { + b := sess.Bucket + for start := 0; start < len(objects); start += rbDeleteBatch { + end := start + rbDeleteBatch + if end > len(objects) { + end = len(objects) + } + batch := objects[start:end] + if ctx.Err() != nil { + return rows, deleted, failed + } + + ch := make(chan minio.ObjectInfo, len(batch)) + for _, obj := range batch { + item := minio.ObjectInfo{Key: obj.Key} + if o.Versions && obj.VersionID != "" && obj.VersionID != "null" { + item.VersionID = obj.VersionID + } + ch <- item + } + close(ch) + + results := sess.Client.RemoveObjectsWithResult(ctx, b.BucketName, ch, minio.RemoveObjectsOptions{GovernanceBypass: o.Bypass}) + seen := map[string]bool{} + for res := range results { + row := objectstorage.DeleteResult{Bucket: b.Name, Key: res.ObjectName} + if o.Versions { + row.VersionID = res.ObjectVersionID + } + if res.Err != nil { + row.Error = sess.humanize(res.Err).Error() + failed++ + } else { + row.Deleted = true + deleted++ + } + seen[row.Key+"\x00"+row.VersionID] = true + rows = append(rows, row) + if o.Human { + fmt.Fprintln(w, row.HumanLine()) + } + } + // Backends in quiet mode report nothing for successes: account for the + // objects of a batch that produced no result as deleted. + for _, obj := range batch { + vid := "" + if o.Versions && obj.VersionID != "" && obj.VersionID != "null" { + vid = obj.VersionID + } + if seen[obj.Key+"\x00"+vid] { + continue + } + row := objectstorage.DeleteResult{Bucket: b.Name, Key: obj.Key, VersionID: vid, Deleted: true} + deleted++ + rows = append(rows, row) + if o.Human { + fmt.Fprintln(w, row.HumanLine()) + } + } + if failed > 0 { + return rows, deleted, failed + } + } + return rows, deleted, failed +} + +// rbDeleteRow builds the row for one listed object. +func rbDeleteRow(bucket string, obj minio.ObjectInfo, withVersions, dry bool) objectstorage.DeleteResult { + row := objectstorage.DeleteResult{Bucket: bucket, Key: obj.Key, DryRun: dry} + if withVersions && obj.VersionID != "" && obj.VersionID != "null" { + row.VersionID = obj.VersionID + } + return row +} + +// rbEmitResults prints human lines to w, or renders the rows. +func rbEmitResults(human bool, w io.Writer, rows []objectstorage.DeleteResult) { + if human { + for _, r := range rows { + fmt.Fprintln(w, r.HumanLine()) + } + return + } + out := make([]renderer.ResponseData, 0, len(rows)) + for _, r := range rows { + out = append(out, r) + } + render(out) +} + +func rbNoun(versions bool, n int) string { + switch { + case versions && n == 1: + return "object version" + case versions: + return "object versions" + case n == 1: + return "object" + default: + return "objects" + } +} + +// rbCommaInt renders 1204 as "1,204". +func rbCommaInt(n int64) string { + s := strconv.FormatInt(n, 10) + neg := strings.HasPrefix(s, "-") + if neg { + s = s[1:] + } + var sb strings.Builder + for i, r := range s { + if i > 0 && (len(s)-i)%3 == 0 { + sb.WriteByte(',') + } + sb.WriteRune(r) + } + if neg { + return "-" + sb.String() + } + return sb.String() +} diff --git a/cmd/s3/rb_test.go b/cmd/s3/rb_test.go new file mode 100644 index 0000000..3753379 --- /dev/null +++ b/cmd/s3/rb_test.go @@ -0,0 +1,309 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" +) + +// newRbStatSession builds a Session against the fake S3 server for the rb and +// stat tests. +func newRbStatSession(t *testing.T, srv *s3test.Server, backendName string) *Session { + t.Helper() + b := &objectstorage.Bucket{ + ID: "bkt_1", + Name: "backups", + BucketName: backendName, + Endpoint: srv.URL(), + StorageClass: "standard", + SigningRegion: "us-east-1", + ProjectSlug: "my-project", + } + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +// stubRbDeleteAPI replaces the API delete for the test and reports calls. +func stubRbDeleteAPI(t *testing.T, fail error) *int { + t.Helper() + calls := 0 + prev := rbDeleteBucketAPI + rbDeleteBucketAPI = func(ctx context.Context, b *objectstorage.Bucket) error { + calls++ + return fail + } + t.Cleanup(func() { rbDeleteBucketAPI = prev }) + return &calls +} + +func rbSeedBucket(srv *s3test.Server, name string, n int) { + srv.CreateBucket(name) + for i := 0; i < n; i++ { + srv.AddObject(name, fmt.Sprintf("dir/file-%02d.txt", i), []byte("payload"), "text/plain") + } +} + +func TestForceRemoveBucket_DeletesObjectsThenBucket(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 3) + sess := newRbStatSession(t, srv, "backups-7f3a") + calls := stubRbDeleteAPI(t, nil) + + var out bytes.Buffer + rows, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true}, &out) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if *calls != 1 { + t.Errorf("API delete called %d times, want 1", *calls) + } + if keys := srv.Keys("backups-7f3a"); len(keys) != 0 { + t.Errorf("objects left on the backend: %v", keys) + } + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(lines) != 4 { + t.Fatalf("expected 3 delete lines + remove_bucket, got %d:\n%s", len(lines), out.String()) + } + for _, l := range lines[:3] { + if !strings.HasPrefix(l, "delete: s3://backups/dir/file-") { + t.Errorf("unexpected line %q", l) + } + } + if lines[3] != "remove_bucket: s3://backups" { + t.Errorf("last line = %q", lines[3]) + } + if len(rows) != 4 || !rows[3].Deleted || rows[3].Key != "" { + t.Errorf("rows = %+v", rows) + } + // Deletes go through multi-object POSTs, never one DELETE per object. + for _, r := range srv.WriteRequests() { + if r.Method == "DELETE" { + t.Errorf("unexpected single DELETE request %s", r.Path) + } + } +} + +func TestForceRemoveBucket_MaxDeleteRefuses(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 5) + sess := newRbStatSession(t, srv, "backups-7f3a") + calls := stubRbDeleteAPI(t, nil) + + var out bytes.Buffer + _, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true, MaxDelete: 2}, &out) + if err == nil { + t.Fatalf("expected a refusal") + } + if exitcode.Of(err) != exitcode.Refused { + t.Errorf("exit code = %d, want %d", exitcode.Of(err), exitcode.Refused) + } + if !strings.Contains(err.Error(), "--max-delete 2") || !strings.Contains(err.Error(), "5 objects") { + t.Errorf("message %q should mention the limit and the count", err.Error()) + } + if *calls != 0 { + t.Errorf("API delete must not be called, got %d", *calls) + } + if len(srv.WriteRequests()) != 0 { + t.Errorf("no write must reach the backend, got %d", len(srv.WriteRequests())) + } + if len(srv.Keys("backups-7f3a")) != 5 { + t.Errorf("objects were deleted despite the refusal") + } + if out.Len() != 0 { + t.Errorf("nothing should be printed on stdout, got %q", out.String()) + } +} + +func TestForceRemoveBucket_DryRunWritesNothing(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 2) + sess := newRbStatSession(t, srv, "backups-7f3a") + calls := stubRbDeleteAPI(t, nil) + + var out bytes.Buffer + // Yes is deliberately false: dry-run never prompts. + rows, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{DryRun: true, Human: true}, &out) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if *calls != 0 { + t.Errorf("API delete must not be called in dry-run, got %d", *calls) + } + if len(srv.WriteRequests()) != 0 { + t.Errorf("dry-run must not write, got %d write requests", len(srv.WriteRequests())) + } + if len(srv.Keys("backups-7f3a")) != 2 { + t.Errorf("dry-run deleted objects") + } + want := "(dryrun) delete: s3://backups/dir/file-00.txt\n(dryrun) delete: s3://backups/dir/file-01.txt\n(dryrun) remove_bucket: s3://backups\n" + if out.String() != want { + t.Errorf("output:\n%s\nwant:\n%s", out.String(), want) + } + if len(rows) != 3 || !rows[0].DryRun || !rows[2].DryRun { + t.Errorf("rows = %+v", rows) + } +} + +func TestForceRemoveBucket_VersionedNeedsVersionsFlag(t *testing.T) { + srv := s3test.New() + defer srv.Close() + bkt := srv.CreateBucket("backups-7f3a") + bkt.Versioned = true + // The fake server derives version IDs from the history length before + // appending, so pin distinct IDs explicitly. + srv.AddObject("backups-7f3a", "a.txt", []byte("v1"), "text/plain").VersionID = "v1" + srv.AddObject("backups-7f3a", "a.txt", []byte("v2"), "text/plain").VersionID = "v2" + sess := newRbStatSession(t, srv, "backups-7f3a") + sess.Bucket.Versioning = true + calls := stubRbDeleteAPI(t, nil) + + var out bytes.Buffer + _, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true}, &out) + if err == nil || exitcode.Of(err) != exitcode.Refused { + t.Fatalf("expected exit 7 refusal, got %v", err) + } + if !strings.Contains(err.Error(), "--versions") || !strings.Contains(err.Error(), "delete markers") { + t.Errorf("message %q should point to --versions", err.Error()) + } + if *calls != 0 || len(srv.Requests()) != 0 { + t.Errorf("refusal must happen before any request (api=%d, s3=%d)", *calls, len(srv.Requests())) + } + + // With --versions every version goes away. + out.Reset() + rows, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true, Versions: true}, &out) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if *calls != 1 { + t.Errorf("API delete called %d times, want 1", *calls) + } + if len(srv.Keys("backups-7f3a")) != 0 { + t.Errorf("versions left: %v", srv.Keys("backups-7f3a")) + } + if len(rows) != 3 { // v1, v2 and the bucket + t.Errorf("rows = %+v", rows) + } + if !strings.Contains(out.String(), "(version v1)") || !strings.Contains(out.String(), "(version v2)") { + t.Errorf("output should list versions:\n%s", out.String()) + } +} + +func TestForceRemoveBucket_ComplianceLockRefused(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 1) + sess := newRbStatSession(t, srv, "backups-7f3a") + sess.Bucket.Locking = true + sess.Bucket.Versioning = true + sess.Bucket.RetentionMode = "COMPLIANCE" + stubRbDeleteAPI(t, nil) + + _, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Versions: true, Human: true}, &bytes.Buffer{}) + if err == nil || exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "COMPLIANCE") { + t.Fatalf("expected COMPLIANCE refusal, got %v", err) + } +} + +func TestForceRemoveBucket_NonInteractiveWithoutYesRefuses(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 1) + sess := newRbStatSession(t, srv, "backups-7f3a") + calls := stubRbDeleteAPI(t, nil) + + _, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Human: true}, &bytes.Buffer{}) + if err == nil || exitcode.Of(err) != exitcode.Refused { + t.Fatalf("expected exit 7 without --yes in a non-TTY, got %v", err) + } + for _, want := range []string{"backups", "bkt_1", "my-project", "backups-7f3a", "1 object"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("question %q should include %q", err.Error(), want) + } + } + if *calls != 0 || len(srv.WriteRequests()) != 0 { + t.Errorf("nothing must be deleted before confirmation") + } +} + +func TestForceRemoveBucket_APIDeleteFailureKeepsExitCode(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 1) + sess := newRbStatSession(t, srv, "backups-7f3a") + stubRbDeleteAPI(t, exitcode.Errorf(exitcode.Permission, "your API token does not have permission")) + + var out bytes.Buffer + rows, err := forceRemoveBucket(context.Background(), nil, sess, rbOptions{Yes: true, Human: true}, &out) + if err == nil || exitcode.Of(err) != exitcode.Permission { + t.Fatalf("expected the API error to propagate, got %v", err) + } + if len(rows) != 1 || !rows[0].Deleted { + t.Errorf("object rows should still be returned: %+v", rows) + } + if strings.Contains(out.String(), "remove_bucket") { + t.Errorf("remove_bucket must not be printed when the API delete failed") + } +} + +func TestRbDeleteObjects_BackendFailureIsPartial(t *testing.T) { + srv := s3test.New() + defer srv.Close() + rbSeedBucket(srv, "backups-7f3a", 2) + sess := newRbStatSession(t, srv, "backups-7f3a") + objects, err := rbListAllObjects(context.Background(), sess.Client, "backups-7f3a", false) + if err != nil { + t.Fatal(err) + } + srv.FailNext = &s3test.ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: 403} + + var out bytes.Buffer + rows, deleted, failed := rbDeleteObjects(context.Background(), sess, objects, rbOptions{Human: true}, &out) + if deleted != 0 || failed == 0 { + t.Fatalf("deleted=%d failed=%d, want a failed batch", deleted, failed) + } + if len(rows) == 0 || rows[0].Error == "" { + t.Errorf("rows should carry the error: %+v", rows) + } + if !strings.Contains(out.String(), "delete failed:") { + t.Errorf("human output should report the failure:\n%s", out.String()) + } +} + +func TestRbHumanizeAPIDelete(t *testing.T) { + b := &objectstorage.Bucket{ID: "bkt_1", Name: "backups"} + err := rbHumanizeAPIDelete(exitcode.Errorf(exitcode.Usage, "the API rejected the request: bucket is not empty"), b) + if exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "--force") { + t.Errorf("not-empty should become exit 7 with a --force hint, got %v", err) + } + other := exitcode.Errorf(exitcode.NotFound, "bucket not found") + if got := rbHumanizeAPIDelete(other, b); !errors.Is(got, other) { + t.Errorf("unrelated errors must pass through, got %v", got) + } + if rbHumanizeAPIDelete(nil, b) != nil { + t.Errorf("nil must stay nil") + } +} + +func TestRbCommaInt(t *testing.T) { + cases := map[int64]string{0: "0", 7: "7", 999: "999", 1000: "1,000", 1204: "1,204", 1234567: "1,234,567", -1204: "-1,204"} + for in, want := range cases { + if got := rbCommaInt(in); got != want { + t.Errorf("rbCommaInt(%d) = %q, want %q", in, got, want) + } + } +} diff --git a/cmd/s3/rm.go b/cmd/s3/rm.go new file mode 100644 index 0000000..f184f49 --- /dev/null +++ b/cmd/s3/rm.go @@ -0,0 +1,436 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" +) + +// rmBatchSize is the S3 multi-delete limit. +const rmBatchSize = 1000 + +// rmOptions are the parsed flags of `lsh s3 delete`. +type rmOptions struct { + Recursive bool + All bool + MaxDelete int + Versions bool + VersionID string + BypassGovernance bool + Quiet bool + OnlyShowErrors bool + DryRun bool + Filters *objectstorage.Filters +} + +// NewRmCmd builds `lsh s3 delete s3://bucket/key`. +func NewRmCmd() *cobra.Command { + filters := &objectstorage.Filters{} + cmd := newCmd(&cobra.Command{ + Use: "delete s3://bucket/key", + Aliases: []string{"rm", "delete-object", "remove"}, + GroupID: groupObjects, + Short: "Delete objects — the bucket stays (alias: rm)", + Long: `Delete one object, or every object under a prefix with --recursive. + +A single object is deleted without confirmation and the command exits 0 even +when the key does not exist (S3 deletes are idempotent). With --recursive the +prefix is normalized to end with "/", the matching objects are enumerated and +counted, and you are asked to confirm (or pass --yes). Deleting every object +in a bucket (s3://bucket with no prefix) additionally requires --all. + +Filters (--exclude/--include) are shell globs evaluated against the key +relative to the prefix, in the order given; the last matching rule wins. + +Object lock: buckets in COMPLIANCE mode are refused; GOVERNANCE mode requires +--bypass-governance-retention. Use --dry-run to print the plan without +deleting anything.`, + Example: ` lsh s3 delete s3://backups/2026/09/dump.sql + lsh s3 delete s3://logs/tmp/ --recursive --dry-run + lsh s3 delete s3://logs/tmp/ --recursive --exclude "*" --include "*.log" --yes + lsh s3 delete s3://logs --recursive --all --max-delete 5000 --yes + lsh s3 delete s3://backups/dump.sql --version-id 3HL4kqtJlcpXroDTDmJ`, + Args: cobra.ExactArgs(1), + }) + f := cmd.Flags() + f.BoolP("recursive", "r", false, "delete every object under the prefix (asks for confirmation)") + f.Bool("all", false, "allow deleting every object in the bucket (required when no prefix is given)") + f.Int("max-delete", 0, "refuse to delete more than N objects (0 = unlimited)") + f.Bool("versions", false, "with --recursive: delete all versions and delete markers, not only the current objects") + f.String("version-id", "", "delete this specific version of the object") + f.Bool("bypass-governance-retention", false, "bypass GOVERNANCE object lock retention (requires a key allowed to do so)") + f.BoolP("quiet", "q", false, "do not print the deleted objects") + f.Bool("only-show-errors", false, "print only failures") + objectstorage.FilterFlags(f, filters) + addYesFlag(cmd) + addProjectFlag(cmd, true, "project the bucket belongs to (disambiguates buckets with the same name)") + addBucketFilterFlags(cmd) + unsupportedAWSBoolFlags(cmd, map[string]string{ + "request-payer": "Latitude buckets have no requester-pays mode", + }) + rejectRegionFlag(cmd) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + opts := rmOptions{Filters: filters, DryRun: dryRun()} + opts.Recursive, _ = f.GetBool("recursive") + opts.All, _ = f.GetBool("all") + opts.MaxDelete, _ = f.GetInt("max-delete") + opts.Versions, _ = f.GetBool("versions") + opts.VersionID, _ = f.GetString("version-id") + opts.BypassGovernance, _ = f.GetBool("bypass-governance-retention") + opts.Quiet, _ = f.GetBool("quiet") + opts.OnlyShowErrors, _ = f.GetBool("only-show-errors") + yes, _ := f.GetBool(flagYes) + + ref, err := rmParseArgs(args[0], opts) + if err != nil { + return printErr(err) + } + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + sess, err := openBucket(ctx, cmd, ref.Bucket, true) + if err != nil { + return printErr(err) + } + confirm := func(question string) error { + return objectstorage.ConfirmOrRefuse(cmd, yes, question) + } + rows, err := runRm(ctx, sess, ref, opts, isHuman(), os.Stdout, confirm) + if !isHuman() && len(rows) > 0 { + render(objectstorage.AsResponseData(rows)) + } + if err != nil { + return printErr(err) + } + return nil + } + return cmd +} + +// rmParseArgs parses the target and validates flag combinations. +func rmParseArgs(arg string, opts rmOptions) (objectstorage.Ref, error) { + if opts.MaxDelete < 0 { + return objectstorage.Ref{}, objectstorage.ErrUsagef("--max-delete must be zero or positive") + } + if !opts.Recursive { + if opts.All { + return objectstorage.Ref{}, objectstorage.ErrUsagef("--all only makes sense with --recursive") + } + if opts.Versions { + return objectstorage.Ref{}, objectstorage.ErrUsagef("--versions requires --recursive; use --version-id to delete one version of an object") + } + // ObjectRef rejects bucket-only targets and keys ending in "/" with + // the hint to add --recursive. A bare bucket gets a longer message: + // the legacy `storage-objects rm ` deleted the bucket, and + // that alias now lands here, so both destinations are spelled out. + ref, err := objectstorage.ObjectRef(arg, false) + if err != nil { + if bucket, bucketErr := objectstorage.ParseBucketOnly(arg); bucketErr == nil { + return ref, objectstorage.ErrUsagef("%s names a bucket, not an object; to delete its objects use 'lsh s3 delete s3://%s --recursive --all', to delete the bucket itself use 'lsh s3 delete-bucket s3://%s'", arg, bucket.Bucket, bucket.Bucket) + } + } + return ref, err + } + if opts.VersionID != "" { + return objectstorage.Ref{}, objectstorage.ErrUsagef("--version-id cannot be combined with --recursive; use --versions to delete every version under the prefix") + } + ref, err := objectstorage.ParseRemote(arg) + if err != nil { + return ref, err + } + ref.Key = objectstorage.NormalizePrefix(strings.TrimLeft(ref.Key, "/")) + if ref.Key == "" && !opts.All { + return ref, objectstorage.ErrUsagef("refusing to delete every object in s3://%s without --all", ref.Bucket) + } + return ref, nil +} + +// runRm executes rm against an open session. Human lines go to out; the +// structured rows are returned for the caller to render. Errors carry exit +// codes (2 usage, 6 partial, 7 refused). +func runRm(ctx context.Context, sess *Session, ref objectstorage.Ref, opts rmOptions, human bool, out io.Writer, confirm func(question string) error) ([]objectstorage.DeleteResult, error) { + if !opts.Recursive { + return rmSingle(ctx, sess, ref.Key, opts, human, out) + } + return rmRecursive(ctx, sess, ref.Key, opts, human, out, confirm) +} + +// rmBucketLabel is the bucket part of `s3:///` in the output: +// the display name, or the backend name when addressed by --endpoint-url. +func rmBucketLabel(b *objectstorage.Bucket) string { + if b.Name != "" && !b.EndpointOverride { + return b.Name + } + return b.BucketName +} + +// rmCheckObjectLock enforces the object lock rules before any write. +func rmCheckObjectLock(b *objectstorage.Bucket, bypass bool) error { + if !b.Locking { + return nil + } + switch strings.ToUpper(b.RetentionMode) { + case "COMPLIANCE": + return exitcode.Errorf(exitcode.Refused, "bucket %s has object lock in COMPLIANCE mode; objects cannot be deleted until their retention expires", b.Display()) + case "GOVERNANCE": + if !bypass { + return exitcode.Errorf(exitcode.Refused, "bucket %s has object lock in GOVERNANCE mode; re-run with --bypass-governance-retention to delete retained objects", b.Display()) + } + } + return nil +} + +// rmSingle deletes one object (or one version) without HEAD and without a +// prompt. A missing key is not an error. +func rmSingle(ctx context.Context, sess *Session, key string, opts rmOptions, human bool, out io.Writer) ([]objectstorage.DeleteResult, error) { + if opts.VersionID != "" { + // Deleting a specific version is permanent, so the lock rules apply; + // without --version-id a versioned bucket only gets a delete marker. + if err := rmCheckObjectLock(sess.Bucket, opts.BypassGovernance); err != nil { + return nil, err + } + } + res := objectstorage.DeleteResult{Bucket: rmBucketLabel(sess.Bucket), Key: key, VersionID: opts.VersionID, DryRun: opts.DryRun} + if opts.DryRun { + rmEmit(res, opts, human, out) + return []objectstorage.DeleteResult{res}, nil + } + err := sess.Client.RemoveObject(ctx, sess.Bucket.BucketName, key, minio.RemoveObjectOptions{ + VersionID: opts.VersionID, + GovernanceBypass: opts.BypassGovernance, + }) + if err != nil { + return nil, sess.humanize(err) + } + res.Deleted = true + rmEmit(res, opts, human, out) + return []objectstorage.DeleteResult{res}, nil +} + +// rmEmit prints one result in human mode, honouring --quiet and +// --only-show-errors (both silence successes only, like aws). Failures +// always go to stderr; structured mode prints nothing here. +func rmEmit(res objectstorage.DeleteResult, opts rmOptions, human bool, out io.Writer) { + if res.Error != "" { + if human { + fmt.Fprintln(os.Stderr, res.HumanLine()) + } + return + } + if !human || opts.Quiet || opts.OnlyShowErrors { + return + } + fmt.Fprintln(out, res.HumanLine()) +} + +// rmPlan enumerates the objects under prefix that pass the filters. +func rmPlan(ctx context.Context, sess *Session, prefix string, opts rmOptions) ([]minio.ObjectInfo, int64, error) { + var ( + objects []minio.ObjectInfo + total int64 + ) + for info := range sess.Client.ListObjects(ctx, sess.Bucket.BucketName, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + WithVersions: opts.Versions, + }) { + if info.Err != nil { + return nil, 0, sess.humanize(info.Err) + } + if info.Key == "" { + continue + } + // Filters see the key relative to the prefix, like aws. + rel := strings.TrimPrefix(info.Key, prefix) + if !opts.Filters.Include(rel) { + continue + } + objects = append(objects, info) + total += info.Size + } + return objects, total, nil +} + +// rmRecursive is `rm --recursive`: lock checks, enumeration, --max-delete, +// confirmation, batched deletion and the partial-failure summary. +func rmRecursive(ctx context.Context, sess *Session, prefix string, opts rmOptions, human bool, out io.Writer, confirm func(string) error) ([]objectstorage.DeleteResult, error) { + if err := rmCheckObjectLock(sess.Bucket, opts.BypassGovernance); err != nil { + return nil, err + } + label := rmBucketLabel(sess.Bucket) + target := "s3://" + label + if prefix != "" { + target += "/" + prefix + } + + objects, total, err := rmPlan(ctx, sess, prefix, opts) + if err != nil { + return nil, err + } + what := "objects" + if opts.Versions { + what = "object versions" + } + if opts.MaxDelete > 0 && len(objects) > opts.MaxDelete { + return nil, exitcode.Errorf(exitcode.Refused, "%d %s match under %s but --max-delete is %d; refusing to delete anything", len(objects), what, target, opts.MaxDelete) + } + + if opts.DryRun { + rows := make([]objectstorage.DeleteResult, 0, len(objects)) + for _, o := range objects { + res := objectstorage.DeleteResult{Bucket: label, Key: o.Key, VersionID: o.VersionID, DryRun: true} + rmEmit(res, opts, human, out) + rows = append(rows, res) + } + objectstorage.Hintf("would delete %d %s (%s)", len(objects), what, objectstorage.HumanSize(total)) + return rows, nil + } + if len(objects) == 0 { + return nil, nil + } + + // target already ends with "/" for a prefix. + question := fmt.Sprintf("Delete %d %s under %s?", len(objects), what, target) + if prefix == "" { + question = fmt.Sprintf("Delete all %d %s in %s (%s)?", len(objects), what, target, rmBucketDetails(sess.Bucket)) + } + if err := confirm(question); err != nil { + return nil, err + } + + var ( + rows []objectstorage.DeleteResult + deleted, failed int + stopped bool + ) + for start := 0; start < len(objects) && !stopped; start += rmBatchSize { + end := start + rmBatchSize + if end > len(objects) { + end = len(objects) + } + results := rmDeleteBatch(ctx, sess, objects[start:end], opts.BypassGovernance, label) + for _, res := range results { + rmEmit(res, opts, human, out) + rows = append(rows, res) + if res.Error == "" { + deleted++ + continue + } + failed++ + if rmIsRetentionError(res.Error) { + stopped = true + } + } + if stopped { + remaining := len(objects) - end + if remaining > 0 { + objectstorage.Warnf("stopped after a retention error; %d %s were not attempted", remaining, what) + } + } + if ctx.Err() != nil { + return rows, objectstorage.Humanize(ctx.Err(), sess.Bucket, &sess.Cred) + } + } + if failed > 0 { + return rows, exitcode.Errorf(exitcode.Partial, "%d deleted, %d failed", deleted, failed) + } + return rows, nil +} + +// rmBucketDetails renders the identity shown when the whole bucket is at +// stake: id, backend name and project. +func rmBucketDetails(b *objectstorage.Bucket) string { + parts := []string{} + if b.ID != "" { + parts = append(parts, b.ID) + } + if b.BucketName != "" && b.BucketName != b.Name { + parts = append(parts, "backend name "+b.BucketName) + } + if p := b.ProjectRef(); p != "" { + parts = append(parts, "project "+p) + } + if len(parts) == 0 { + return b.BucketName + } + return strings.Join(parts, ", ") +} + +// rmDeleteBatch deletes up to rmBatchSize objects with one multi-delete call +// and returns a result per object, in input order. A request-level failure +// (the whole POST rejected) marks every object of the batch as failed. +func rmDeleteBatch(ctx context.Context, sess *Session, batch []minio.ObjectInfo, bypass bool, label string) []objectstorage.DeleteResult { + objectsCh := make(chan minio.ObjectInfo, len(batch)) + for _, o := range batch { + objectsCh <- o + } + close(objectsCh) + + failed := map[string]error{} + var batchErr error + for e := range sess.Client.RemoveObjects(ctx, sess.Bucket.BucketName, objectsCh, minio.RemoveObjectsOptions{GovernanceBypass: bypass}) { + if e.ObjectName == "" { + batchErr = e.Err + continue + } + failed[rmObjectID(e.ObjectName, e.VersionID)] = e.Err + } + + results := make([]objectstorage.DeleteResult, 0, len(batch)) + for _, o := range batch { + res := objectstorage.DeleteResult{Bucket: label, Key: o.Key, VersionID: o.VersionID} + err, ok := failed[rmObjectID(o.Key, o.VersionID)] + if !ok { + err, ok = failed[rmObjectID(o.Key, "")] + } + switch { + case ok: + res.Error = rmErrorText(err) + case batchErr != nil: + res.Error = rmErrorText(batchErr) + default: + res.Deleted = true + } + results = append(results, res) + } + return results +} + +func rmObjectID(key, versionID string) string { return key + "\x00" + versionID } + +// rmErrorText renders a per-object failure compactly (code: message). +func rmErrorText(err error) string { + if err == nil { + return "unknown error" + } + if resp := minio.ToErrorResponse(err); resp.Code != "" { + if resp.Message != "" { + return resp.Code + ": " + resp.Message + } + return resp.Code + } + return objectstorage.Redact(err.Error()) +} + +// rmIsRetentionError reports whether a per-object failure text denotes an +// object lock / retention refusal, after which the command stops. +func rmIsRetentionError(text string) bool { + t := strings.ToLower(text) + return strings.Contains(t, "objectlocked") || + strings.Contains(t, "retention") || + strings.Contains(t, "object lock") || + strings.Contains(t, "governance") +} + +// Compile-time check that DeleteResult renders through the shared renderer. +var _ renderer.ResponseData = objectstorage.DeleteResult{} diff --git a/cmd/s3/rm_test.go b/cmd/s3/rm_test.go new file mode 100644 index 0000000..f57eebc --- /dev/null +++ b/cmd/s3/rm_test.go @@ -0,0 +1,662 @@ +package s3 + +import ( + "bytes" + "context" + "io" + "net/http" + "os" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" +) + +// rmTestSession opens a Session against the fake S3 server. +func rmTestSession(t *testing.T, srv *s3test.Server, bucketName string) *Session { + t.Helper() + b := &objectstorage.Bucket{ID: "bkt_1", Name: "b", BucketName: bucketName, Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: "us-east-1"} + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +func rmYes(string) error { return nil } + +func rmFilters(t *testing.T, rules ...string) *objectstorage.Filters { + t.Helper() + f := &objectstorage.Filters{} + for i := 0; i+1 < len(rules); i += 2 { + if err := f.Add(rules[i] == "include", rules[i+1]); err != nil { + t.Fatal(err) + } + } + return f +} + +func TestRmParseArgs(t *testing.T) { + cases := []struct { + name string + arg string + opts rmOptions + wantKey string + wantCode int + wantMsg string + }{ + {name: "single object", arg: "s3://logs/tmp/old.log", wantKey: "tmp/old.log"}, + {name: "single without scheme", arg: "logs/tmp/old.log", wantKey: "tmp/old.log"}, + {name: "single trailing slash needs recursive", arg: "s3://logs/tmp/", wantCode: exitcode.Usage, wantMsg: "--recursive"}, + {name: "single bucket only", arg: "s3://logs", wantCode: exitcode.Usage, wantMsg: "names a bucket"}, + {name: "recursive normalizes prefix", arg: "s3://logs/tmp", opts: rmOptions{Recursive: true}, wantKey: "tmp/"}, + {name: "recursive keeps trailing slash", arg: "s3://logs/tmp/", opts: rmOptions{Recursive: true}, wantKey: "tmp/"}, + {name: "recursive whole bucket needs all", arg: "s3://logs", opts: rmOptions{Recursive: true}, wantCode: exitcode.Usage, wantMsg: "refusing to delete every object in s3://logs without --all"}, + {name: "recursive slash only needs all", arg: "s3://logs/", opts: rmOptions{Recursive: true}, wantCode: exitcode.Usage, wantMsg: "without --all"}, + {name: "recursive double slash needs all", arg: "s3://logs//", opts: rmOptions{Recursive: true}, wantCode: exitcode.Usage, wantMsg: "without --all"}, + {name: "recursive whole bucket with all", arg: "s3://logs", opts: rmOptions{Recursive: true, All: true}, wantKey: ""}, + {name: "all without recursive", arg: "s3://logs/k", opts: rmOptions{All: true}, wantCode: exitcode.Usage, wantMsg: "--all"}, + {name: "versions without recursive", arg: "s3://logs/k", opts: rmOptions{Versions: true}, wantCode: exitcode.Usage, wantMsg: "--versions"}, + {name: "version-id with recursive", arg: "s3://logs/k/", opts: rmOptions{Recursive: true, VersionID: "v1"}, wantCode: exitcode.Usage, wantMsg: "--version-id"}, + {name: "negative max-delete", arg: "s3://logs/k", opts: rmOptions{MaxDelete: -1}, wantCode: exitcode.Usage, wantMsg: "--max-delete"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ref, err := rmParseArgs(tc.arg, tc.opts) + if tc.wantCode != 0 { + if err == nil { + t.Fatalf("expected error, got ref %+v", ref) + } + if exitcode.Of(err) != tc.wantCode { + t.Errorf("exit code = %d, want %d (%v)", exitcode.Of(err), tc.wantCode, err) + } + if !strings.Contains(err.Error(), tc.wantMsg) { + t.Errorf("error %q does not mention %q", err, tc.wantMsg) + } + return + } + if err != nil { + t.Fatal(err) + } + if ref.Key != tc.wantKey { + t.Errorf("key = %q, want %q", ref.Key, tc.wantKey) + } + }) + } +} + +func TestRmSingleObjectNoHeadAndIdempotent(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/old.log", []byte("x"), "text/plain") + srv.AddObject("logs-7f3a", "tmp/keep.log", []byte("y"), "text/plain") + sess := rmTestSession(t, srv, "logs-7f3a") + + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/old.log", Remote: true}, rmOptions{}, true, &out, rmYes) + if err != nil { + t.Fatal(err) + } + if got := out.String(); got != "delete: s3://b/tmp/old.log\n" { + t.Errorf("stdout = %q", got) + } + if len(rows) != 1 || !rows[0].Deleted || rows[0].Key != "tmp/old.log" { + t.Errorf("rows = %+v", rows) + } + if srv.Object("logs-7f3a", "tmp/old.log") != nil { + t.Error("object still present") + } + if srv.Object("logs-7f3a", "tmp/keep.log") == nil { + t.Error("sibling object was deleted") + } + for _, r := range srv.Requests() { + if r.Method == http.MethodHead || r.Method == http.MethodGet { + t.Errorf("unexpected %s %s: single delete must not HEAD or list", r.Method, r.Path) + } + if r.Method == http.MethodDelete && r.Path != "/logs-7f3a/tmp/old.log" { + t.Errorf("unexpected delete path %s", r.Path) + } + } + + // Missing key: still exit 0 and still reported as deleted. + srv.ResetRequests() + out.Reset() + rows, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/missing.log", Remote: true}, rmOptions{}, true, &out, rmYes) + if err != nil { + t.Fatalf("delete of a missing key must succeed: %v", err) + } + if len(rows) != 1 || out.String() != "delete: s3://b/tmp/missing.log\n" { + t.Errorf("rows=%+v out=%q", rows, out.String()) + } +} + +func TestRmSingleVersionIDAndQuiet(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("vers-7f3a").Versioned = true + // Each put on a versioned bucket gets the next version id (v1, v2…). + srv.AddObject("vers-7f3a", "k", []byte("1"), "") + current := srv.AddObject("vers-7f3a", "k", []byte("22"), "") + if current.VersionID != "v2" { + t.Fatalf("fake server version id = %q, want v2", current.VersionID) + } + sess := rmTestSession(t, srv, "vers-7f3a") + + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "k", Remote: true}, rmOptions{VersionID: "v2", Quiet: true}, true, &out, rmYes) + if err != nil { + t.Fatal(err) + } + if out.Len() != 0 { + t.Errorf("--quiet printed %q", out.String()) + } + if len(rows) != 1 || rows[0].VersionID != "v2" { + t.Errorf("rows = %+v", rows) + } + var sawVersion bool + for _, r := range srv.WriteRequests() { + if r.Method == http.MethodDelete && r.Query.Get("versionId") == "v2" { + sawVersion = true + } + } + if !sawVersion { + t.Error("DELETE did not carry versionId=v2") + } + if o := srv.Object("vers-7f3a", "k"); o == nil || string(o.Data) != "1" || o.VersionID != "v1" { + t.Errorf("previous version (v1) should be current now, got %+v", o) + } +} + +func TestRmRecursiveWithFiltersAndCount(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a.log", []byte("aaa"), "") + srv.AddObject("logs-7f3a", "tmp/b.log", []byte("bb"), "") + srv.AddObject("logs-7f3a", "tmp/notes.txt", []byte("n"), "") + srv.AddObject("logs-7f3a", "tmp/sub/c.log", []byte("c"), "") + srv.AddObject("logs-7f3a", "tmp2/d.log", []byte("d"), "") + srv.AddObject("logs-7f3a", "other.log", []byte("o"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + var out bytes.Buffer + var asked string + confirm := func(q string) error { asked = q; return nil } + opts := rmOptions{Recursive: true, Filters: rmFilters(t, "exclude", "*", "include", "*.log")} + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, opts, true, &out, confirm) + if err != nil { + t.Fatal(err) + } + if asked != "Delete 3 objects under s3://b/tmp/?" { + t.Errorf("confirmation = %q", asked) + } + if len(rows) != 3 { + t.Fatalf("rows = %+v", rows) + } + want := "delete: s3://b/tmp/a.log\ndelete: s3://b/tmp/b.log\ndelete: s3://b/tmp/sub/c.log\n" + if out.String() != want { + t.Errorf("stdout = %q, want %q", out.String(), want) + } + if keys := srv.Keys("logs-7f3a"); strings.Join(keys, ",") != "other.log,tmp/notes.txt,tmp2/d.log" { + t.Errorf("remaining keys = %v", keys) + } + // One multi-delete POST for the batch, no per-object DELETEs. + var posts, deletes int + for _, r := range srv.WriteRequests() { + switch r.Method { + case http.MethodPost: + posts++ + if !r.Query.Has("delete") { + t.Errorf("POST without ?delete: %s", r.Path) + } + case http.MethodDelete: + deletes++ + } + } + if posts != 1 || deletes != 0 { + t.Errorf("posts=%d deletes=%d, want one multi-delete", posts, deletes) + } +} + +func TestRmRecursiveStructuredRowsAndOnlyShowErrors(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a.log", []byte("a"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, OnlyShowErrors: true}, true, &out, rmYes) + if err != nil { + t.Fatal(err) + } + if out.Len() != 0 { + t.Errorf("--only-show-errors printed %q", out.String()) + } + if len(rows) != 1 || !rows[0].Deleted || rows[0].Bucket != "b" || rows[0].Key != "tmp/a.log" { + t.Errorf("rows = %+v", rows) + } + // Structured mode never writes to out either. + srv.AddObject("logs-7f3a", "tmp/b.log", []byte("b"), "") + out.Reset() + rows, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, false, &out, rmYes) + if err != nil || out.Len() != 0 || len(rows) != 1 { + t.Errorf("structured: err=%v out=%q rows=%+v", err, out.String(), rows) + } +} + +func TestRmRecursiveEmptyPrefixIsNoop(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "keep.log", []byte("k"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + called := false + confirm := func(string) error { called = true; return nil } + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "nothing/", Remote: true}, rmOptions{Recursive: true}, true, &out, confirm) + if err != nil || len(rows) != 0 || out.Len() != 0 { + t.Errorf("err=%v rows=%+v out=%q", err, rows, out.String()) + } + if called { + t.Error("no confirmation should be asked when nothing matches") + } + if len(srv.WriteRequests()) != 0 { + t.Error("no writes expected") + } +} + +func TestRmRecursiveMaxDeleteRefusesWithoutWrites(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + for _, k := range []string{"tmp/a", "tmp/b", "tmp/c"} { + srv.AddObject("logs-7f3a", k, []byte("x"), "") + } + sess := rmTestSession(t, srv, "logs-7f3a") + + called := false + confirm := func(string) error { called = true; return nil } + var out bytes.Buffer + _, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, MaxDelete: 2}, true, &out, confirm) + if err == nil || exitcode.Of(err) != exitcode.Refused { + t.Fatalf("expected exit %d, got %v", exitcode.Refused, err) + } + if !strings.Contains(err.Error(), "3 objects match") || !strings.Contains(err.Error(), "--max-delete is 2") { + t.Errorf("message = %q", err) + } + if called { + t.Error("must refuse before asking for confirmation") + } + if n := len(srv.WriteRequests()); n != 0 { + t.Errorf("%d write requests, want 0", n) + } + if len(srv.Keys("logs-7f3a")) != 3 { + t.Error("objects were deleted") + } + + // Exactly at the limit is allowed. + _, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, MaxDelete: 3}, true, &out, rmYes) + if err != nil { + t.Fatalf("max-delete equal to the count must pass: %v", err) + } + if len(srv.Keys("logs-7f3a")) != 0 { + t.Error("objects not deleted") + } +} + +func TestRmRecursiveRefusedConfirmation(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a", []byte("x"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + refuse := func(q string) error { return exitcode.Errorf(exitcode.Refused, "cancelled") } + var out bytes.Buffer + _, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, true, &out, refuse) + if exitcode.Of(err) != exitcode.Refused { + t.Fatalf("expected exit 7, got %v", err) + } + if len(srv.WriteRequests()) != 0 || srv.Object("logs-7f3a", "tmp/a") == nil { + t.Error("declined confirmation must not delete") + } +} + +func TestRmDryRunZeroWrites(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a.log", []byte("aaaa"), "") + srv.AddObject("logs-7f3a", "tmp/b.txt", []byte("bb"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + called := false + confirm := func(string) error { called = true; return nil } + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, DryRun: true}, true, &out, confirm) + if err != nil { + t.Fatal(err) + } + want := "(dryrun) delete: s3://b/tmp/a.log\n(dryrun) delete: s3://b/tmp/b.txt\n" + if out.String() != want { + t.Errorf("stdout = %q, want %q", out.String(), want) + } + if len(rows) != 2 || !rows[0].DryRun || rows[0].Deleted { + t.Errorf("rows = %+v", rows) + } + if called { + t.Error("dry-run must not prompt") + } + if n := len(srv.WriteRequests()); n != 0 { + t.Errorf("%d write requests during dry-run", n) + } + if len(srv.Keys("logs-7f3a")) != 2 { + t.Error("dry-run deleted objects") + } + + // Single-object dry-run: no request at all. + srv.ResetRequests() + out.Reset() + rows, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/a.log", Remote: true}, rmOptions{DryRun: true}, true, &out, confirm) + if err != nil || len(rows) != 1 || out.String() != "(dryrun) delete: s3://b/tmp/a.log\n" { + t.Errorf("single dry-run: err=%v rows=%+v out=%q", err, rows, out.String()) + } + if len(srv.Requests()) != 0 { + t.Error("single-object dry-run must not call the backend") + } +} + +func TestRmRecursiveVersions(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("vers-7f3a").Versioned = true + srv.AddObject("vers-7f3a", "tmp/a", []byte("1"), "") + srv.AddObject("vers-7f3a", "tmp/a", []byte("22"), "") + srv.AddObject("vers-7f3a", "tmp/b", []byte("3"), "") + sess := rmTestSession(t, srv, "vers-7f3a") + + // Without --versions only the current objects are targeted. + var out bytes.Buffer + var asked string + confirm := func(q string) error { asked = q; return nil } + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, DryRun: true}, true, &out, confirm) + if err != nil || len(rows) != 2 { + t.Fatalf("plain plan: err=%v rows=%+v", err, rows) + } + + // With --versions every version is enumerated and deleted. + out.Reset() + rows, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, Versions: true}, true, &out, confirm) + if err != nil { + t.Fatal(err) + } + if asked != "Delete 3 object versions under s3://b/tmp/?" { + t.Errorf("confirmation = %q", asked) + } + if len(rows) != 3 { + t.Fatalf("rows = %+v", rows) + } + perKey := map[string]int{} + for _, r := range rows { + if !r.Deleted || r.VersionID == "" { + t.Errorf("row %+v should be a deleted version", r) + } + perKey[r.Key]++ + } + if perKey["tmp/a"] != 2 || perKey["tmp/b"] != 1 { + t.Errorf("versions per key = %v, want tmp/a:2 tmp/b:1", perKey) + } + if !strings.Contains(out.String(), "delete: s3://b/tmp/a (version v1)") { + t.Errorf("stdout = %q", out.String()) + } + if len(srv.Keys("vers-7f3a")) != 0 { + t.Errorf("remaining keys %v", srv.Keys("vers-7f3a")) + } + var listedVersions bool + for _, r := range srv.Requests() { + if r.Method == http.MethodGet && r.Query.Has("versions") { + listedVersions = true + } + } + if !listedVersions { + t.Error("expected a ListObjectVersions call") + } +} + +func TestRmWholeBucketWithAll(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "a", []byte("1"), "") + srv.AddObject("logs-7f3a", "dir/b", []byte("2"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + sess.Bucket.ProjectSlug = "my-project" + + var asked string + confirm := func(q string) error { asked = q; return nil } + var out bytes.Buffer + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "", Remote: true}, rmOptions{Recursive: true, All: true}, true, &out, confirm) + if err != nil || len(rows) != 2 { + t.Fatalf("err=%v rows=%+v", err, rows) + } + for _, want := range []string{"Delete all 2 objects in s3://b", "bkt_1", "backend name logs-7f3a", "project my-project"} { + if !strings.Contains(asked, want) { + t.Errorf("confirmation %q lacks %q", asked, want) + } + } + if len(srv.Keys("logs-7f3a")) != 0 { + t.Error("bucket not emptied") + } +} + +func TestRmObjectLockRefusals(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("lock-7f3a") + srv.AddObject("lock-7f3a", "tmp/a", []byte("1"), "") + sess := rmTestSession(t, srv, "lock-7f3a") + sess.Bucket.Locking = true + sess.Bucket.RetentionMode = "COMPLIANCE" + + var out bytes.Buffer + _, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, true, &out, rmYes) + if exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "COMPLIANCE") { + t.Errorf("compliance: %v", err) + } + sess.Bucket.RetentionMode = "GOVERNANCE" + _, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, true, &out, rmYes) + if exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "--bypass-governance-retention") { + t.Errorf("governance: %v", err) + } + if len(srv.Requests()) != 0 { + t.Error("lock refusals must happen before any request") + } + // Bypass: proceeds and sends the bypass header. + _, err = runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true, BypassGovernance: true}, true, &out, rmYes) + if err != nil { + t.Fatal(err) + } + var sawBypass bool + for _, r := range srv.WriteRequests() { + if strings.EqualFold(r.Header.Get("X-Amz-Bypass-Governance-Retention"), "true") { + sawBypass = true + } + } + if !sawBypass { + t.Error("multi-delete did not carry x-amz-bypass-governance-retention") + } +} + +func TestRmRecursivePartialFailure(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + srv.AddObject("logs-7f3a", "tmp/a", []byte("1"), "") + sess := rmTestSession(t, srv, "logs-7f3a") + + var out bytes.Buffer + // Listing succeeds; the multi-delete POST is denied. + confirm := func(string) error { + srv.FailNext = &s3test.ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: 403} + return nil + } + rows, err := runRm(context.Background(), sess, objectstorage.Ref{Bucket: "b", Key: "tmp/", Remote: true}, rmOptions{Recursive: true}, true, &out, confirm) + if exitcode.Of(err) != exitcode.Partial { + t.Fatalf("expected exit %d, got %v", exitcode.Partial, err) + } + if !strings.Contains(err.Error(), "0 deleted, 1 failed") { + t.Errorf("summary = %q", err) + } + if len(rows) != 1 || rows[0].Deleted || !strings.Contains(rows[0].Error, "AccessDenied") { + t.Errorf("rows = %+v", rows) + } + if out.Len() != 0 { + t.Errorf("failures must not go to stdout: %q", out.String()) + } +} + +// rmCaptureOutput runs fn with os.Stdout and os.Stderr redirected to pipes and +// returns what was written to each. Command-level tests need it because the +// rm/presign RunE and objectstorage.PrintError write straight to the process +// streams (and cobra echoes errors to os.Stderr unless silenced). +func rmCaptureOutput(t *testing.T, fn func()) (stdout, stderr string) { + t.Helper() + redirect := func(target **os.File) func() string { + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := *target + *target = w + done := make(chan string, 1) + go func() { + b, _ := io.ReadAll(r) + done <- string(b) + }() + return func() string { + _ = w.Close() + *target = old + return <-done + } + } + finishOut := redirect(&os.Stdout) + finishErr := redirect(&os.Stderr) + fn() + // Restore stderr first so a failing assertion below is visible. + stderr = finishErr() + stdout = finishOut() + return stdout, stderr +} + +// TestRmCommandRecursiveWithoutAllIsUsageError drives the cobra command end to +// end: `rm s3://bucket --recursive` without --all must exit 2 through the +// newCmd wrapper and print the refusal exactly once (RunE prints it, cobra is +// silenced). It fails before any bucket lookup, so no API or S3 call is made. +func TestRmCommandRecursiveWithoutAllIsUsageError(t *testing.T) { + for _, target := range []string{"s3://logs", "s3://logs/"} { + t.Run(target, func(t *testing.T) { + cmd := NewRmCmd() + Finalize(cmd) // production installs this in build_s3.go + cmd.SetArgs([]string{target, "--recursive"}) + var err error + stdout, stderr := rmCaptureOutput(t, func() { err = cmd.Execute() }) + if exitcode.Of(err) != exitcode.Usage { + t.Fatalf("exit code = %d, want %d (%v)", exitcode.Of(err), exitcode.Usage, err) + } + if n := strings.Count(stderr, "refusing to delete every object in s3://logs without --all"); n != 1 { + t.Errorf("refusal printed %d times, want exactly once:\n%s", n, stderr) + } + if strings.Contains(stderr, "Usage:") { + t.Errorf("usage must not be echoed on errors:\n%s", stderr) + } + if stdout != "" { + t.Errorf("nothing should reach stdout, got %q", stdout) + } + }) + } +} + +// TestRmCommandSingleMissingKeyExitsZero runs a plain `rm` of a key that does +// not exist through the s3 group in --endpoint-url mode against the fake S3 +// server: S3 deletes are idempotent, so the command exits 0 and reports the +// key as deleted. +func TestRmCommandSingleMissingKeyExitsZero(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("logs-7f3a") + t.Setenv(objectstorage.EnvAccessKeyID, "AK") + t.Setenv(objectstorage.EnvSecretAccessKey, "SK") + + group := NewGroupCmd() + group.AddCommand(NewRmCmd()) + Finalize(group) // production installs this in build_s3.go + group.SetArgs([]string{"rm", "s3://logs-7f3a/tmp/missing.log", "--endpoint-url", srv.URL()}) + var err error + stdout, stderr := rmCaptureOutput(t, func() { err = group.Execute() }) + if err != nil { + t.Fatalf("rm of a missing key must exit 0, got %v\nstderr: %s", err, stderr) + } + if stdout != "delete: s3://logs-7f3a/tmp/missing.log\n" { + t.Errorf("stdout = %q", stdout) + } + if strings.Contains(stderr, "Error") { + t.Errorf("no error expected on stderr:\n%s", stderr) + } + var deletes int + for _, r := range srv.Requests() { + if r.Method == http.MethodDelete { + deletes++ + } + if r.Method == http.MethodHead || r.Method == http.MethodGet { + t.Errorf("unexpected %s %s: single delete must not HEAD or list", r.Method, r.Path) + } + } + if deletes != 1 { + t.Errorf("%d DELETE requests, want 1", deletes) + } +} + +// TestRmCommandRejectsRegionFlag guards the aws-compat shim: --region is an +// unsupported flag on object commands and must fail with exit 2 and the +// pointer to --signing-region, before any network call. +func TestRmCommandRejectsRegionFlag(t *testing.T) { + cmd := NewRmCmd() + Finalize(cmd) + cmd.SetArgs([]string{"s3://logs/tmp/old.log", "--region", "us-east-1"}) + var err error + _, stderr := rmCaptureOutput(t, func() { err = cmd.Execute() }) + if exitcode.Of(err) != exitcode.Usage { + t.Fatalf("exit code = %d, want %d (%v)", exitcode.Of(err), exitcode.Usage, err) + } + if !strings.Contains(stderr, "--region is not supported") || !strings.Contains(stderr, "--signing-region") { + t.Errorf("stderr lacks the directed --region explanation:\n%s", stderr) + } +} + +// TestRmBucketOnlyPointsAtRb covers the migration path of the legacy +// `storage-objects rm `, which deleted a bucket: that alias now lands +// on `s3 rm`, so the usage error names both destinations. +func TestRmBucketOnlyPointsAtRb(t *testing.T) { + _, err := rmParseArgs("bkt_abc123", rmOptions{}) + if err == nil { + t.Fatal("expected a usage error for a bucket-only target") + } + if code := exitcode.Of(err); code != exitcode.Usage { + t.Errorf("exit code = %d, want %d", code, exitcode.Usage) + } + for _, want := range []string{"names a bucket, not an object", "--recursive --all", "lsh s3 delete-bucket s3://bkt_abc123"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q is missing %q", err, want) + } + } + // A prefix keeps the shorter --recursive hint. + if _, err := rmParseArgs("s3://logs/tmp/", rmOptions{}); err == nil || strings.Contains(err.Error(), "lsh s3 delete-bucket") { + t.Errorf("a prefix must keep the --recursive hint, got %v", err) + } +} diff --git a/cmd/s3/s3.go b/cmd/s3/s3.go new file mode 100644 index 0000000..c70f413 --- /dev/null +++ b/cmd/s3/s3.go @@ -0,0 +1,539 @@ +// Package s3 implements `lsh s3`, the object storage command group. It mixes +// control-plane calls to the Latitude API (buckets, access keys, lifecycle, +// metrics, usage) with data-plane calls straight to the bucket's S3 endpoint +// (list, copy, move, delete, get, presign), using the verbs and flags of `aws s3` so +// existing habits and scripts carry over. +package s3 + +import ( + "context" + "fmt" + "os" + "regexp" + "strings" + "sync" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/cli" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// Persistent flag names shared by the data-plane commands. +const ( + flagAccessKey = "access-key" + flagEndpointURL = "endpoint-url" + flagSigningRegion = "signing-region" + flagS3Region = "s3-region" // aws-flavoured alias of --signing-region + flagAddressing = "addressing-style" + flagDryRunAWS = "dryrun" // aws spelling of --dry-run + flagProject = "project" + flagYes = "yes" + flagSite = "site" +) + +// Help groups of the s3 command list. Grouping the fourteen subcommands the +// way the root groups its command families keeps the list from reading flat. +const ( + groupBuckets = "s3-buckets" + groupObjects = "s3-objects" + groupCredentials = "s3-credentials" + groupReports = "s3-reports" +) + +// NewGroupCmd builds the `s3` group. Subcommands are attached by +// cmd/build_s3.go so each lives in its own file. +// +// The canonical verbs follow the rest of the CLI (list, get, copy, move, +// delete, create-bucket, delete-bucket); the short S3 spellings (ls, stat, +// cp, mv, rm, mb, rb) are aliases, so both vocabularies work and the help +// reads in the CLI's own terms. +func NewGroupCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "s3", + // Typing these at the root gets a "did you mean s3" hint. The long + // name is deliberately not an alias: 'lsh help object-storage' is the + // guide, and cobra's help resolves aliases as well as names. + SuggestFor: []string{"s3api", "object-storage", "upload", "download", "configure", "bucket", "buckets", "objects"}, + GroupID: cli.StorageGroupID, + Short: "Object storage: buckets, objects, access keys and lifecycle rules", + Long: `Object storage: buckets, objects, access keys and lifecycle rules. + +Buckets and objects are addressed as s3://[/]. Endpoint, region +and credentials are resolved for you. See 'lsh help object-storage' for how +addressing and access keys work. + +Getting started: + lsh s3 create-bucket s3://backups --region DAL --project my-project + lsh s3 copy ./dump.sql s3://backups/2026/09/ + lsh s3 list s3://backups/2026/09/ + +Short aliases: ls, mb, rb, cp, mv, rm, stat. +Exit codes for scripts: 'lsh help exit-codes'.`, + Example: ` lsh s3 list + lsh s3 list s3://backups/2026/ --recursive --human-readable --summarize + lsh s3 copy s3://backups/2026/09/dump.sql ./restore/ + lsh s3 delete s3://backups/tmp/ --recursive --dry-run + lsh s3 delete-bucket s3://backups + lsh s3 presign s3://backups/report.pdf --expires-in 15m + lsh s3 access-keys create --bucket backups=rw --name ci-deploy + lsh s3 lifecycle create s3://logs --prefix tmp/ --expiration-days 7`, + // --dryrun is the hidden spelling of the global --dry-run: map it onto + // the same switch so every subcommand sees a single value. + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + if v, _ := cmd.Flags().GetBool(flagDryRunAWS); v { + lsh.DryRun = true + } + return nil + }, + } + cmd.AddGroup( + &cobra.Group{ID: groupBuckets, Title: "Buckets:"}, + &cobra.Group{ID: groupObjects, Title: "Objects:"}, + &cobra.Group{ID: groupCredentials, Title: "Credentials:"}, + &cobra.Group{ID: groupReports, Title: "Settings and reports:"}, + ) + pf := cmd.PersistentFlags() + pf.String(flagAccessKey, "", "use the saved access key with this name instead of the automatic selection") + pf.String(flagEndpointURL, "", "address the bucket on this S3 endpoint without the Latitude API (bucket is the backend name; credentials only from the environment)") + pf.String(flagSigningRegion, "", "override the SigV4 signing region derived from the endpoint") + pf.String(flagS3Region, "", "alias of --signing-region") + pf.String(flagAddressing, objectstorage.AddressingPath, "S3 addressing style: path or virtual") + pf.Bool(flagDryRunAWS, false, "alias of --dry-run") + _ = pf.MarkHidden(flagS3Region) + _ = pf.MarkHidden(flagAddressing) + _ = pf.MarkHidden(flagDryRunAWS) + + return cmd +} + +// newCmd applies the conventions every s3 subcommand shares: usage is never +// echoed on errors. Error printing/exit-code handling is installed by +// Finalize once the whole tree is built (commands may assign RunE/PreRunE +// after construction). +func newCmd(c *cobra.Command) *cobra.Command { + c.SilenceUsage = true + return c +} + +// Finalize walks the group's command tree and installs the shared error +// contract on every command: +// - errors raised by cobra itself (bad flag, wrong argument count) are +// printed once and exit 2; +// - errors returned by PreRunE/RunE are printed once through printErr (exit +// code preserved) and cobra is told not to echo them again. +// +// build_s3.go calls it after attaching the subcommands. +func Finalize(root *cobra.Command) { + root.SetFlagErrorFunc(func(cmd *cobra.Command, err error) error { + cmd.SilenceErrors = true + return printErr(exitcode.New(exitcode.Usage, err)) + }) + var walk func(c *cobra.Command) + walk = func(c *cobra.Command) { + c.SilenceUsage = true + // Mark the subtree so shared validation (cli.MakeRootCmd's pre-run) + // can apply these exit codes here without changing the older groups. + if c.Annotations == nil { + c.Annotations = map[string]string{} + } + c.Annotations[exitcode.OptInAnnotation] = "true" + if args := c.Args; args != nil { + c.Args = func(cmd *cobra.Command, a []string) error { + if err := args(cmd, a); err != nil { + cmd.SilenceErrors = true + return printErr(exitcode.New(exitcode.Usage, err)) + } + return nil + } + } + if pre := c.PreRunE; pre != nil { + c.PreRunE = func(cmd *cobra.Command, a []string) error { + return printOnce(cmd, pre(cmd, a)) + } + } + if run := c.RunE; run != nil { + c.RunE = func(cmd *cobra.Command, a []string) error { + return printOnce(cmd, run(cmd, a)) + } + } + for _, sub := range c.Commands() { + walk(sub) + } + } + walk(root) +} + +// printOnce prints err unless the command already did (objectstorage.PrintError +// marks it) and silences cobra so the message is not duplicated; the exit +// code travels with the returned error. +func printOnce(cmd *cobra.Command, err error) error { + if err == nil { + return nil + } + if !objectstorage.IsPrinted(err) { + err = printErr(err) + } + cmd.SilenceErrors = true + return err +} + +// addProjectFlag registers --project. optional=true marks it as a filter / +// disambiguator so the root pre-run never prompts for it. +func addProjectFlag(cmd *cobra.Command, optional bool, usage string) { + cmd.Flags().String(flagProject, "", usage) + if optional { + if cmd.Annotations == nil { + cmd.Annotations = map[string]string{} + } + cmd.Annotations[cli.ProjectOptionalAnnotation] = "true" + } +} + +// addBucketFilterFlags registers the flags that disambiguate a bucket name that +// resolves to several buckets: --storage-class (short -c) and --site. They are +// only useful when a display name is reused across classes or sites; the bkt_ +// ID always resolves without them. +func addBucketFilterFlags(cmd *cobra.Command) { + cmd.Flags().StringP(flagStorageClass, "c", "", "disambiguate a repeated bucket name by storage class (standard or high_performance)") + cmd.Flags().String(flagSite, "", "disambiguate a repeated bucket name by Latitude site (e.g. DAL, TYO4)") +} + +// addYesFlag registers --yes/-y. +func addYesFlag(cmd *cobra.Command) { + cmd.Flags().BoolP(flagYes, "y", false, "do not ask for confirmation") +} + +// projectFlag returns the --project value (or LSH_PROJECT when unset). +func projectFlag(cmd *cobra.Command) string { + if v, _ := cmd.Flags().GetString(flagProject); v != "" { + return v + } + return os.Getenv("LSH_PROJECT") +} + +// profileFlag returns the --profile override. +func profileFlag(cmd *cobra.Command) string { + v, _ := cmd.Flags().GetString("profile") + return v +} + +// endpointOverride returns --endpoint-url or LSH_S3_ENDPOINT_URL. +func endpointOverride(cmd *cobra.Command) string { + if v, _ := cmd.Flags().GetString(flagEndpointURL); v != "" { + return v + } + return os.Getenv(objectstorage.EnvEndpointURL) +} + +// signingRegionOverride returns --signing-region / --s3-region / env. +func signingRegionOverride(cmd *cobra.Command) string { + if v, _ := cmd.Flags().GetString(flagSigningRegion); v != "" { + return v + } + if v, _ := cmd.Flags().GetString(flagS3Region); v != "" { + return v + } + return os.Getenv(objectstorage.EnvSigningRegion) +} + +// dryRun reports whether --dry-run (or --dry-run) is active. +func dryRun() bool { return lsh.DryRun } + +// isHuman reports whether output goes to the human (table) format, in which +// case the aws-style plain lines are printed instead of the table renderer. +func isHuman() bool { return renderer.ResolveFormat() == renderer.FormatTable } + +// render prints structured results through the shared renderer without the +// interactive table (object listings can be huge and must stay pipeable). +func render(items []renderer.ResponseData) { renderer.RenderStatic(items) } + +// printErr prints a humanized error to stderr and returns it for RunE. +func printErr(err error) error { return objectstorage.PrintError(err) } + +// newResolver builds the bucket resolver for a command. +func newResolver(cmd *cobra.Command) *objectstorage.Resolver { + r := &objectstorage.Resolver{ + Project: projectFlag(cmd), + EndpointURL: endpointOverride(cmd), + SigningRegion: signingRegionOverride(cmd), + RetryOptions: []operations.Option{operations.WithRetries(lsh.RetryConfig())}, + HasFilterFlags: cmd.Flags().Lookup(flagSite) != nil, + } + if raw, _ := cmd.Flags().GetString(flagStorageClass); strings.TrimSpace(raw) != "" { + class, err := objectstorage.ParseStorageClass(raw) + if err != nil { + r.FilterErr = fmt.Errorf("--%s: %w", flagStorageClass, err) + } else { + r.ClassFilter = class + } + } + if site, _ := cmd.Flags().GetString(flagSite); strings.TrimSpace(site) != "" { + r.SiteFilter = strings.TrimSpace(site) + } + // Without a token the SDK would send an empty bearer and every API call + // would come back as 401; leave API nil so callers get the "not logged in" + // error (exit 4) with the LSH_S3_ENDPOINT_URL alternative instead. + if r.EndpointURL == "" && viper.GetString("Authorization") != "" { + r.API = apiClient() + } + if r.EndpointURL != "" { + endpointHintOnce.Do(func() { + objectstorage.Hintf("note: --endpoint-url set; bypassing the Latitude API and using credentials from the environment only") + }) + } + return r +} + +// endpointHintOnce makes the endpoint-override note appear once per run even +// when a command builds several resolvers (stat, remote-to-remote copies). +var endpointHintOnce sync.Once + +// apiClient returns the Latitude API client for object storage commands (the +// regular client plus the /storage response normalization the current SDK +// needs; see objectstorage.NewAPIClient). +func apiClient() *sdk.Latitudesh { return objectstorage.NewAPIClient() } + +// resolveBucket resolves a bucket token (name, bkt_ id or backend name). +func resolveBucket(ctx context.Context, cmd *cobra.Command, token string) (*objectstorage.Bucket, error) { + return newResolver(cmd).Resolve(ctx, token) +} + +// Session bundles everything a data-plane command needs for one bucket. +type Session struct { + Bucket *objectstorage.Bucket + Cred objectstorage.Credential + Client *minio.Client +} + +// openBucket resolves the bucket, selects a credential and builds the S3 +// client. write=true requests a credential with write permission. +func openBucket(ctx context.Context, cmd *cobra.Command, token string, write bool) (*Session, error) { + b, err := resolveBucket(ctx, cmd, token) + if err != nil { + return nil, err + } + // A high_performance key is bound to one site, but the SDK model drops the + // bucket's site, so without this the site check in keyMatchesBucket is a + // no-op on the data plane and a key from another site can be selected. + // The site is only a selection constraint, so the lookup is skipped when + // the credential does not come from the profile; when it does, a failed + // lookup has to fail the command — continuing with an empty site turns the + // constraint into a wildcard and lets a key from another site win. + if b.StorageClass == objectstorage.ClassHighPerformance && b.Site == "" && !b.EndpointOverride && usesSavedCredential() { + if fillErr := newResolver(cmd).FillSite(ctx, b); fillErr != nil { + return nil, exitcode.Errorf(exitcode.Of(fillErr), + "could not determine which site bucket %s is in, and a high_performance access key is only valid in its own site: %v\n retry, or name the key explicitly with --access-key ", + b.Display(), fillErr) + } + } + return openResolved(cmd, b, write) +} + +// usesSavedCredential reports whether the credential will come from the active +// profile, which is the only case where the bucket's site changes the outcome. +// LSH_S3_* credentials bypass the profile entirely; an explicit --access-key +// does not, because the named key is still checked against the bucket's site. +func usesSavedCredential() bool { + _, fromEnv, _ := objectstorage.EnvCredential() + return !fromEnv +} + +// openResolved is openBucket for an already-resolved bucket. +func openResolved(cmd *cobra.Command, b *objectstorage.Bucket, write bool) (*Session, error) { + if err := b.Validate(); err != nil { + return nil, err + } + accessKey, _ := cmd.Flags().GetString(flagAccessKey) + cred, err := objectstorage.ResolveCredential(b, objectstorage.CredentialOptions{ + AccessKeyName: accessKey, + ProfileOverride: profileFlag(cmd), + Write: write, + }) + if err != nil { + return nil, err + } + addressing, _ := cmd.Flags().GetString(flagAddressing) + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{ + Debug: lsh.Debug, + Addressing: addressing, + }) + if err != nil { + return nil, err + } + if lsh.Debug { + fmt.Fprintf(os.Stderr, "[s3] bucket=%s backend=%s endpoint=%s signing-region=%s addressing=%s credential=%s\n", + b.Display(), b.BucketName, b.Endpoint, b.SigningRegion, addressing, cred.Describe(b.ID)) + } + return &Session{Bucket: b, Cred: cred, Client: client}, nil +} + +// humanize maps an S3/API error for a session. +func (s *Session) humanize(err error) error { + if s == nil { + return objectstorage.Humanize(err, nil, nil) + } + return objectstorage.Humanize(err, s.Bucket, &s.Cred) +} + +// unsupportedAWSFlags registers flags that exist in `aws s3` but have no +// equivalent on Latitude, so scripts get an explanation instead of +// "unknown flag". Each entry maps the flag to the reason/alternative. +func unsupportedAWSFlags(cmd *cobra.Command, flags map[string]string) { + for name := range flags { + cmd.Flags().String(name, "", "not supported on Latitude object storage") + _ = cmd.Flags().MarkHidden(name) + } + // Bool-style flags must also parse without a value. + pre := cmd.PreRunE + cmd.PreRunE = func(c *cobra.Command, args []string) error { + for name, why := range flags { + if c.Flags().Changed(name) { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s is not supported on Latitude object storage: %s", name, why)) + } + } + if pre != nil { + return pre(c, args) + } + return nil + } +} + +// unsupportedAWSBoolFlags registers boolean aws flags that are rejected with +// an explanation when set. +func unsupportedAWSBoolFlags(cmd *cobra.Command, flags map[string]string) { + for name := range flags { + cmd.Flags().Bool(name, false, "not supported on Latitude object storage") + _ = cmd.Flags().MarkHidden(name) + } + pre := cmd.PreRunE + cmd.PreRunE = func(c *cobra.Command, args []string) error { + for name, why := range flags { + if c.Flags().Changed(name) { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s is not supported on Latitude object storage: %s", name, why)) + } + } + if pre != nil { + return pre(c, args) + } + return nil + } +} + +// joinNonEmpty joins the non-empty strings with sep. +func joinNonEmpty(sep string, parts ...string) string { + var out []string + for _, p := range parts { + if strings.TrimSpace(p) != "" { + out = append(out, p) + } + } + return strings.Join(out, sep) +} + +// rejectRegionFlag registers --region on object commands (ls, cp, mv, sync, +// rm, stat, presign) so aws users get a directed explanation instead of an +// "unknown flag" error: on Latitude --region is a site (mb, access-keys) and +// the SigV4 signing region comes from the bucket's endpoint. +func rejectRegionFlag(cmd *cobra.Command) { + unsupportedAWSFlags(cmd, map[string]string{ + "region": "on object commands the signing region is derived from the bucket's endpoint. To disambiguate a repeated bucket name by location use --site; to override the SigV4 signing region use --signing-region ", + }) +} + +// outputExplicit reports whether the user asked for a structured format on +// the command line (-o/--output or --json), as opposed to LSH_OUTPUT or the +// config file. Secrets are only embedded in structured output on explicit +// request. +func outputExplicit(cmd *cobra.Command) bool { + if viper.GetBool("output_explicit") { + return true + } + if cmd == nil { + return false + } + changed := cmd.Flags().Changed("json") + if f := cmd.Root().PersistentFlags().Lookup("json"); f != nil && f.Changed { + changed = true + } + return changed +} + +// awsRegionPattern matches AWS-style region names (us-east-1, sa-east-1…), +// which users coming from aws type by reflex where a Latitude site is expected. +var awsRegionPattern = regexp.MustCompile(`^[a-z]{2}(?:-[a-z]+)+-[0-9]+$`) + +// looksLikeAWSRegion reports whether s is an AWS region name. +func looksLikeAWSRegion(s string) bool { + return awsRegionPattern.MatchString(strings.ToLower(strings.TrimSpace(s))) +} + +// emptyCell is the placeholder for empty table cells across the group. +const emptyCell = "-" + +// orEmptyCell returns s, or the shared placeholder when empty. +func orEmptyCell(s string) string { + if strings.TrimSpace(s) == "" { + return emptyCell + } + return s +} + +// saveNewKey stores a freshly created access key in the active profile under +// name, appending -2, -3… when that name already holds a different key so a +// saved key is never overwritten silently. It returns the name actually used +// and the profile name. Every creation path (access-keys create, configure, +// mb) goes through it so the clash policy is the same everywhere; rotate +// replaces in place with objectstorage.SaveKey instead. +func saveNewKey(cmd *cobra.Command, name string, k config.StoredAccessKey) (string, string, error) { + _, profileName, profile, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return "", "", err + } + existing := profile.ObjectStorageKeys() + final := name + for i := 2; ; i++ { + cur, taken := existing[final] + if !taken || cur.AccessKeyID == k.AccessKeyID { + break + } + final = fmt.Sprintf("%s-%d", name, i) + } + if _, err := objectstorage.SaveKey(profileFlag(cmd), final, k); err != nil { + return final, profileName, err + } + return final, profileName, nil +} + +// NewLegacyAliasCmd returns the hidden `storage-objects` command kept for +// scripts written against the group this one replaced. Cobra prints the +// deprecation notice on stderr and leaves the command out of the help; the +// arguments are re-dispatched to `lsh s3` untouched (flag parsing is disabled +// here so nothing is interpreted before it reaches the real command). +// +// Two legacy verbs changed meaning and are called out in the notice: `rm` and +// `delete` remove objects now, and a bucket is removed with `delete-bucket`. +func NewLegacyAliasCmd(root *cobra.Command) *cobra.Command { + return &cobra.Command{ + Use: "storage-objects", + Hidden: true, + SilenceUsage: true, + Deprecated: "use 'lsh s3' instead. Note: 'rm' and 'delete' now remove objects; remove a bucket with 'lsh s3 delete-bucket'", + DisableFlagParsing: true, + RunE: func(cmd *cobra.Command, args []string) error { + root.SetArgs(append([]string{"s3"}, args...)) + err := root.Execute() + // The inner run has already reported the error; keep the outer + // Execute from printing it a second time. + cmd.SilenceErrors = true + return err + }, + } +} diff --git a/cmd/s3/s3_test.go b/cmd/s3/s3_test.go new file mode 100644 index 0000000..f7754d6 --- /dev/null +++ b/cmd/s3/s3_test.go @@ -0,0 +1,23 @@ +package s3 + +import ( + "testing" + + "github.com/latitudesh/lsh/internal/objectstorage" +) + +// TestUsesSavedCredential guards the condition that decides whether a bucket's +// site has to be resolved: it only matters when the credential comes from the +// profile, since LSH_S3_* bypasses selection entirely. +func TestUsesSavedCredential(t *testing.T) { + t.Setenv(objectstorage.EnvAccessKeyID, "") + t.Setenv(objectstorage.EnvSecretAccessKey, "") + if !usesSavedCredential() { + t.Error("without environment credentials the profile is used") + } + t.Setenv(objectstorage.EnvAccessKeyID, "AK") + t.Setenv(objectstorage.EnvSecretAccessKey, "SK") + if usesSavedCredential() { + t.Error("environment credentials bypass the profile, so the site is irrelevant") + } +} diff --git a/cmd/s3/stat.go b/cmd/s3/stat.go new file mode 100644 index 0000000..5441b53 --- /dev/null +++ b/cmd/s3/stat.go @@ -0,0 +1,290 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" + cobra "github.com/spf13/cobra" +) + +const flagStatVersionID = "version-id" + +// NewStatCmd builds `lsh s3 get s3://bucket[/key]`. +func NewStatCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "get s3://bucket[/key]", + Aliases: []string{"stat", "describe", "head"}, + GroupID: groupBuckets, + SuggestFor: []string{"show", "info"}, + Short: "Show a bucket or an object (alias: stat)", + Long: `Show a bucket or an object. + +For a bucket (s3://bucket) the details come from the Latitude API: id, names, +endpoint, site, signing region, class, versioning, object lock and which saved +access keys cover it. For an object (s3://bucket/key) the CLI issues a HEAD +request to the S3 endpoint: size, ETag, content type, last modified, version +and user metadata.`, + Example: ` lsh s3 get s3://backups + lsh s3 get s3://backups/2026/09/dump.sql + lsh s3 get s3://backups/2026/09/dump.sql --version-id 3HL4kqtJlcpXroDTDmJ + lsh s3 get s3://backups -o json`, + Args: cobra.ExactArgs(1), + RunE: runStat, + }) + addProjectFlag(cmd, true, "project ID or slug to disambiguate the bucket name") + addBucketFilterFlags(cmd) + cmd.Flags().String(flagStatVersionID, "", "inspect this object version") + rejectRegionFlag(cmd) + return cmd +} + +func runStat(cmd *cobra.Command, args []string) error { + ref, err := objectstorage.ParseRemote(args[0]) + if err != nil { + return printErr(err) + } + versionID, _ := cmd.Flags().GetString(flagStatVersionID) + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + if ref.Key == "" { + if versionID != "" { + return printErr(objectstorage.ErrUsagef("--version-id only applies to objects (s3://bucket/key)")) + } + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + if err := newResolver(cmd).FillSite(ctx, b); err != nil && lsh.Debug { + fmt.Fprintf(os.Stderr, "[s3] could not fill site for %s: %v\n", b.Display(), err) + } + if !isHuman() { + render([]renderer.ResponseData{NewBucketRow(b)}) + return nil + } + writeBucketStat(os.Stdout, b, savedKeysCovering(cmd, b)) + return nil + } + + if strings.HasSuffix(ref.Key, "/") { + return printErr(objectstorage.ErrUsagef("key %q ends with '/'; stat inspects a single object (use 'lsh s3 list %s' to list a prefix)", ref.Key, ref)) + } + sess, err := openBucket(ctx, cmd, ref.Bucket, false) + if err != nil { + return printErr(err) + } + obj, err := statObject(ctx, sess, ref.Key, versionID) + if err != nil { + return printErr(err) + } + if !isHuman() { + render([]renderer.ResponseData{obj}) + return nil + } + writeObjectStat(os.Stdout, obj) + return nil +} + +// statObject issues the HEAD request and converts the result. +func statObject(ctx context.Context, sess *Session, key, versionID string) (objectstorage.Object, error) { + info, err := sess.Client.StatObject(ctx, sess.Bucket.BucketName, key, minio.StatObjectOptions{VersionID: versionID}) + if err != nil { + return objectstorage.Object{}, sess.humanize(err) + } + // A HEAD response never says whether the version is the latest, so the + // listing-only IsLatest flag stays unset; the version ID is still copied + // whenever the backend returns one. + obj := objectstorage.ObjectFromInfo(info, false) + if obj.Key == "" { + obj.Key = key + } + obj.Metadata = statUserMetadata(info) + return obj, nil +} + +// statUserMetadata collects x-amz-meta-* values from a HEAD response. minio fills +// UserMetadata with the prefix stripped; the raw headers are consulted too for +// backends that only surface them there. +func statUserMetadata(info minio.ObjectInfo) map[string]string { + out := map[string]string{} + for k, v := range info.UserMetadata { + lk := strings.ToLower(strings.TrimPrefix(strings.ToLower(k), "x-amz-meta-")) + if lk == "" || v == "" { + continue + } + out[lk] = v + } + for k, v := range info.Metadata { + lk := strings.ToLower(k) + if !strings.HasPrefix(lk, "x-amz-meta-") || len(v) == 0 { + continue + } + name := strings.TrimPrefix(lk, "x-amz-meta-") + if _, dup := out[name]; !dup && name != "" { + out[name] = v[0] + } + } + if len(out) == 0 { + return nil + } + return out +} + +// kv is one "Label: value" line of the human output. +type statKV struct { + label string + value string +} + +// statWriteKV prints aligned "Label: value" lines. +func statWriteKV(w io.Writer, pairs []statKV) { + width := 0 + for _, p := range pairs { + if len(p.label) > width { + width = len(p.label) + } + } + for _, p := range pairs { + fmt.Fprintf(w, "%-*s %s\n", width+1, p.label+":", p.value) + } +} + +// writeBucketStat renders the bucket details for humans. +func writeBucketStat(w io.Writer, b *objectstorage.Bucket, keys []statCoveringKey) { + pairs := []statKV{} + if !b.EndpointOverride { + pairs = append(pairs, + statKV{"ID", b.ID}, + statKV{"Name", b.Name}, + ) + } + pairs = append(pairs, statKV{"Bucket name (backend)", b.BucketName}) + if !b.EndpointOverride { + project := b.ProjectRef() + if b.ProjectName != "" && b.ProjectName != project { + project = fmt.Sprintf("%s (%s)", project, b.ProjectName) + } + pairs = append(pairs, + statKV{"Project", orEmptyCell(project)}, + statKV{"Class", orEmptyCell(b.StorageClass)}, + statKV{"Site", orEmptyCell(firstNonEmptyStr(b.Site, b.City))}, + ) + } + pairs = append(pairs, + statKV{"Endpoint", orEmptyCell(b.Endpoint)}, + statKV{"Signing region", orEmptyCell(b.SigningRegion)}, + ) + if !b.EndpointOverride { + created := "" + if b.CreatedAt != nil { + created = objectstorage.FormatTime(*b.CreatedAt) + } + pairs = append(pairs, + statKV{"Versioning", yesNo(b.Versioning)}, + statKV{"Locking", lockingLabel(b)}, + statKV{"Source", orEmptyCell(b.Source)}, + statKV{"Created", orEmptyCell(created)}, + ) + } + statWriteKV(w, pairs) + if b.EndpointOverride { + return + } + fmt.Fprintln(w, "Access keys covering this bucket:") + if len(keys) == 0 { + fmt.Fprintln(w, " (none saved)") + return + } + for _, k := range keys { + fmt.Fprintf(w, " %s (%s)\n", k.Name, k.Permission) + } +} + +// writeObjectStat renders the object details for humans. +func writeObjectStat(w io.Writer, o objectstorage.Object) { + pairs := []statKV{ + {"Key", o.Key}, + {"Size", fmt.Sprintf("%d (%s)", o.Size, objectstorage.HumanSize(o.Size))}, + {"Last modified", orEmptyCell(objectstorage.FormatTime(o.LastModified))}, + {"ETag", orEmptyCell(o.ETag)}, + {"Content type", orEmptyCell(o.ContentType)}, + } + if o.VersionID != "" { + pairs = append(pairs, statKV{"Version ID", o.VersionID}) + } + if o.StorageClass != "" { + pairs = append(pairs, statKV{"Storage class", o.StorageClass}) + } + if o.IsDeleteMarker { + pairs = append(pairs, statKV{"Delete marker", "yes"}) + } + if len(o.Metadata) > 0 { + names := make([]string, 0, len(o.Metadata)) + for k := range o.Metadata { + names = append(names, k) + } + sort.Strings(names) + parts := make([]string, 0, len(names)) + for _, k := range names { + parts = append(parts, k+"="+o.Metadata[k]) + } + pairs = append(pairs, statKV{"Metadata", strings.Join(parts, " ")}) + } + statWriteKV(w, pairs) +} + +// statCoveringKey is a saved key that grants access to the bucket. +type statCoveringKey struct { + Name string + Permission string +} + +// savedKeysCovering lists the saved keys of the active profile that cover b, +// sorted by name. Profile problems are not errors for stat: the list is +// simply empty. +func savedKeysCovering(cmd *cobra.Command, b *objectstorage.Bucket) []statCoveringKey { + _, _, profile, err := objectstorage.ActiveProfile(profileFlag(cmd)) + if err != nil { + return nil + } + return statCoveringKeys(profile.ObjectStorageKeys(), b) +} + +// statCoveringKeys applies the class/site/project compatibility rules and +// StoredAccessKey.Covers to the saved keys. +func statCoveringKeys(keys map[string]config.StoredAccessKey, b *objectstorage.Bucket) []statCoveringKey { + var out []statCoveringKey + for name, k := range keys { + if k.Scope == config.ScopeUnknown { + continue + } + if k.StorageClass != "" && b.StorageClass != "" && k.StorageClass != b.StorageClass { + continue + } + if k.Site != "" && b.Site != "" && !strings.EqualFold(k.Site, b.Site) { + continue + } + if k.ProjectID != "" && b.ProjectID != "" && k.ProjectID != b.ProjectID { + continue + } + if !k.Covers(b.ID, false) { + continue + } + perm := k.Permission(b.ID) + if k.Scope == config.ScopeFullAccess { + perm = "fullaccess" + } + out = append(out, statCoveringKey{Name: name, Permission: perm}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} diff --git a/cmd/s3/stat_test.go b/cmd/s3/stat_test.go new file mode 100644 index 0000000..238e7e5 --- /dev/null +++ b/cmd/s3/stat_test.go @@ -0,0 +1,284 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" + "github.com/minio/minio-go/v7" +) + +func TestStatObject_ReturnsMetadata(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + obj := srv.AddObject("backups-7f3a", "2026/09/dump.sql", []byte("select 1;"), "application/sql") + obj.Metadata = map[string]string{"owner": "ops", "env": "prod"} + sess := newRbStatSession(t, srv, "backups-7f3a") + + got, err := statObject(context.Background(), sess, "2026/09/dump.sql", "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Key != "2026/09/dump.sql" { + t.Errorf("key = %q", got.Key) + } + if got.Size != int64(len("select 1;")) { + t.Errorf("size = %d", got.Size) + } + if got.ContentType != "application/sql" { + t.Errorf("content type = %q", got.ContentType) + } + if got.ETag == "" || strings.Contains(got.ETag, `"`) { + t.Errorf("etag should be set and unquoted, got %q", got.ETag) + } + if got.LastModified.IsZero() { + t.Errorf("last modified should be set") + } + if got.Metadata["owner"] != "ops" || got.Metadata["env"] != "prod" { + t.Errorf("metadata = %v", got.Metadata) + } + if got.VersionID != "" { + t.Errorf("unversioned object should have no version id, got %q", got.VersionID) + } + if got.IsLatest != nil { + t.Errorf("a HEAD response carries no is_latest flag, got %v", *got.IsLatest) + } + + // Only a HEAD reaches the backend. + reqs := srv.Requests() + if len(reqs) != 1 || reqs[0].Method != http.MethodHead { + t.Errorf("expected a single HEAD, got %+v", reqs) + } + + var out bytes.Buffer + writeObjectStat(&out, got) + text := out.String() + for _, want := range []string{"Key:", "2026/09/dump.sql", "Size:", "9 (9 Bytes)", "Content type:", "application/sql", "Metadata:", "env=prod owner=ops"} { + if !strings.Contains(text, want) { + t.Errorf("output lacks %q:\n%s", want, text) + } + } +} + +func TestStatObject_VersionID(t *testing.T) { + srv := s3test.New() + defer srv.Close() + bkt := srv.CreateBucket("backups-7f3a") + bkt.Versioned = true + // Pin distinct version IDs (the fake server would reuse v1 for both). + srv.AddObject("backups-7f3a", "a.txt", []byte("one"), "text/plain").VersionID = "v1" + srv.AddObject("backups-7f3a", "a.txt", []byte("three"), "text/plain").VersionID = "v2" + sess := newRbStatSession(t, srv, "backups-7f3a") + + got, err := statObject(context.Background(), sess, "a.txt", "v1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Size != 3 || got.VersionID != "v1" { + t.Errorf("got size=%d version=%q, want the old version", got.Size, got.VersionID) + } + // HEAD never reports whether a version is the latest (only listings do), so + // stat must not fabricate is_latest even when --version-id was given. + if got.IsLatest != nil { + t.Errorf("is_latest must stay unset for stat, got %v", *got.IsLatest) + } + reqs := srv.Requests() + if len(reqs) != 1 || reqs[0].Query.Get("versionId") != "v1" { + t.Errorf("HEAD should carry versionId=v1, got %+v", reqs) + } +} + +func TestStatObject_NotFoundIsExit3(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + sess := newRbStatSession(t, srv, "backups-7f3a") + + _, err := statObject(context.Background(), sess, "missing.txt", "") + if err == nil { + t.Fatalf("expected an error") + } + if exitcode.Of(err) != exitcode.NotFound { + t.Errorf("exit code = %d, want %d; err=%v", exitcode.Of(err), exitcode.NotFound, err) + } + if !strings.Contains(err.Error(), "not found") { + t.Errorf("message %q should say not found", err.Error()) + } +} + +func TestStatObject_AccessDeniedIsExit5(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + srv.DenyAll = true + sess := newRbStatSession(t, srv, "backups-7f3a") + + _, err := statObject(context.Background(), sess, "a.txt", "") + if err == nil || exitcode.Of(err) != exitcode.Permission { + t.Fatalf("expected exit 5, got %v (code %d)", err, exitcode.Of(err)) + } + if strings.Contains(err.Error(), "SK") { + t.Errorf("error must not include the secret: %q", err.Error()) + } +} + +func TestStatUserMetadata(t *testing.T) { + info := minio.ObjectInfo{ + UserMetadata: map[string]string{"Owner": "ops", "X-Amz-Meta-Env": "prod"}, + Metadata: http.Header{"X-Amz-Meta-Team": {"storage"}, "Content-Type": {"text/plain"}, "X-Amz-Meta-Owner": {"ignored"}}, + } + got := statUserMetadata(info) + want := map[string]string{"owner": "ops", "env": "prod", "team": "storage"} + if len(got) != len(want) { + t.Fatalf("metadata = %v, want %v", got, want) + } + for k, v := range want { + if got[k] != v { + t.Errorf("metadata[%q] = %q, want %q", k, got[k], v) + } + } + if statUserMetadata(minio.ObjectInfo{}) != nil { + t.Errorf("no metadata should yield nil") + } +} + +func TestWriteBucketStat(t *testing.T) { + created := time.Date(2026, 9, 7, 15, 12, 1, 0, time.UTC) + b := &objectstorage.Bucket{ + ID: "bkt_1Gjbang9n0L2w", + Name: "backups", + BucketName: "backups-7f3a", + Endpoint: "https://s3.us-central-1.storage.sh", + SigningRegion: "us-central-1", + StorageClass: "standard", + Site: "DAL", + ProjectID: "proj_1", + ProjectSlug: "my-project", + Versioning: true, + Locking: true, + RetentionMode: "GOVERNANCE", + RetentionDays: 30, + Source: "default", + CreatedAt: &created, + } + var out bytes.Buffer + writeBucketStat(&out, b, []statCoveringKey{{Name: "lsh-me-standard", Permission: "fullaccess"}, {Name: "ci", Permission: "readonly"}}) + text := out.String() + for _, want := range []string{ + "ID:", "bkt_1Gjbang9n0L2w", + "Name:", "backups", + "Bucket name (backend):", "backups-7f3a", + "Project:", "my-project", + "Class:", "standard", + "Site:", "DAL", + "Endpoint:", "https://s3.us-central-1.storage.sh", + "Signing region:", "us-central-1", + "Versioning:", "yes", + "Locking:", "GOVERNANCE (30d)", + "Source:", "default", + "Created:", + "Access keys covering this bucket:", + " lsh-me-standard (fullaccess)", + " ci (readonly)", + } { + if !strings.Contains(text, want) { + t.Errorf("output lacks %q:\n%s", want, text) + } + } + // Labels are aligned: every value starts in the same column. + col := -1 + for _, line := range strings.Split(strings.TrimSpace(text), "\n") { + if !strings.Contains(line, ":") || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "Access keys") { + continue + } + i := strings.Index(line, ": ") + rest := line[i+1:] + valueCol := i + 1 + (len(rest) - len(strings.TrimLeft(rest, " "))) + if col == -1 { + col = valueCol + } else if valueCol != col { + t.Errorf("value column %d differs from %d in %q", valueCol, col, line) + } + } + + out.Reset() + writeBucketStat(&out, b, nil) + if !strings.Contains(out.String(), "(none saved)") { + t.Errorf("empty key list should print (none saved):\n%s", out.String()) + } +} + +func TestWriteBucketStat_EndpointOverride(t *testing.T) { + b := &objectstorage.Bucket{BucketName: "raw-bucket", Endpoint: "https://objects.tyo4.storage.sh", SigningRegion: "tyo4", EndpointOverride: true} + var out bytes.Buffer + writeBucketStat(&out, b, nil) + text := out.String() + if strings.Contains(text, "ID:") || strings.Contains(text, "Access keys") { + t.Errorf("API-only fields must be skipped in endpoint-override mode:\n%s", text) + } + if !strings.Contains(text, "raw-bucket") || !strings.Contains(text, "tyo4") { + t.Errorf("backend name and signing region should be shown:\n%s", text) + } +} + +func TestStatCoveringKeys(t *testing.T) { + b := &objectstorage.Bucket{ID: "bkt_1", StorageClass: "standard", Site: "DAL", ProjectID: "proj_1"} + keys := map[string]config.StoredAccessKey{ + "full": {Scope: config.ScopeFullAccess, StorageClass: "standard", ProjectID: "proj_1"}, + "limited-rw": {Scope: config.ScopeLimitedAccess, StorageClass: "standard", Buckets: map[string]string{"bkt_1": "rw"}}, + "limited-ro": {Scope: config.ScopeLimitedAccess, StorageClass: "standard", Buckets: map[string]string{"bkt_1": "readonly"}}, + "other-bkt": {Scope: config.ScopeLimitedAccess, StorageClass: "standard", Buckets: map[string]string{"bkt_2": "rw"}}, + "other-class": {Scope: config.ScopeFullAccess, StorageClass: "high_performance", Site: "TYO4"}, + "other-proj": {Scope: config.ScopeFullAccess, StorageClass: "standard", ProjectID: "proj_2"}, + "unknown": {Scope: config.ScopeUnknown}, + } + got := statCoveringKeys(keys, b) + if len(got) != 3 { + t.Fatalf("covering keys = %+v, want full, limited-ro, limited-rw", got) + } + if got[0].Name != "full" || got[0].Permission != "fullaccess" { + t.Errorf("got[0] = %+v", got[0]) + } + if got[1].Name != "limited-ro" || got[1].Permission != "readonly" { + t.Errorf("got[1] = %+v", got[1]) + } + if got[2].Name != "limited-rw" || got[2].Permission != "rw" { + t.Errorf("got[2] = %+v", got[2]) + } +} + +// TestBucketRowStructuredWithoutAPIPayload covers endpoint-override mode: there +// is no API document to marshal, so the row used to render as "{}" while the +// human output showed the bucket name, endpoint and signing region. +func TestBucketRowStructuredWithoutAPIPayload(t *testing.T) { + b := &objectstorage.Bucket{ + Name: "backups-7f3a", BucketName: "backups-7f3a", + Endpoint: "https://s3.us-central-1.storage.sh", SigningRegion: "us-central-1", + EndpointOverride: true, + } + raw, err := json.Marshal(NewBucketRow(b)) + if err != nil { + t.Fatal(err) + } + var doc map[string]string + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatalf("unmarshal %s: %v", raw, err) + } + for k, want := range map[string]string{ + "name": "backups-7f3a", "bucket_name": "backups-7f3a", + "endpoint": "https://s3.us-central-1.storage.sh", "signing_region": "us-central-1", + } { + if doc[k] != want { + t.Errorf("%s = %q, want %q (in %s)", k, doc[k], want, raw) + } + } +} diff --git a/cmd/s3/sync.go b/cmd/s3/sync.go new file mode 100644 index 0000000..30d1654 --- /dev/null +++ b/cmd/s3/sync.go @@ -0,0 +1,238 @@ +package s3 + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "time" + + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/spf13/cobra" +) + +// syncOptions are the flags specific to sync. +type syncOptions struct { + // Delete removes destination entries that are absent from the source. + Delete bool + // SizeOnly compares only sizes. + SizeOnly bool + // ExactTimestamps makes same-sized downloads skip only when the + // timestamps match exactly. + ExactTimestamps bool +} + +// NewSyncCmd builds `lsh s3 sync `. +func NewSyncCmd() *cobra.Command { + var f transferFlags + var so syncOptions + cmd := newCmd(&cobra.Command{ + Use: "sync ", + GroupID: groupObjects, + Short: "Synchronize directories and prefixes", + Long: `Recursively copy new and changed files between a local directory and a +prefix, or between two prefixes on the same endpoint. + +An entry is transferred when it is missing at the destination, its size +differs, or the source is newer than the destination. --size-only ignores +timestamps; --exact-timestamps (S3 -> local) skips same-sized files only when +the timestamps match exactly. ETags are never compared. --delete removes +destination entries that no longer exist at the source; --exclude/--include +apply to paths relative to the source (and to the destination for --delete). +Deletions run after the copies and are skipped entirely if any copy failed, so +a partial run never leaves the destination without both versions; re-run the +sync to apply them. They also never leave the destination tree: a candidate +reached through a directory symlink is reported and kept. + +Output lines use upload:/download:/copy:/delete:. --dry-run (or --dry-run) +prints the plan without writing or deleting anything.`, + Example: ` lsh s3 sync ./site s3://www + lsh s3 sync s3://backups/2026/ ./backups/2026/ + lsh s3 sync ./logs s3://logs/host-1/ --exclude "*" --include "*.log" --delete --dry-run + lsh s3 sync s3://backups/ s3://archive/backups/ --size-only`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return runSyncCommand(cmd, args, &f, so) + }, + }) + addTransferFlags(cmd, &f, false) + cmd.Flags().BoolVar(&so.Delete, "delete", false, "delete destination entries that are absent from the source (skipped when a copy fails)") + cmd.Flags().BoolVar(&so.SizeOnly, "size-only", false, "compare by size only, ignoring timestamps") + cmd.Flags().BoolVar(&so.ExactTimestamps, "exact-timestamps", false, "S3 -> local: skip same-sized files only when the timestamps match exactly") + return cmd +} + +// runSyncCommand is the RunE body of sync. +func runSyncCommand(cmd *cobra.Command, args []string, f *transferFlags, so syncOptions) error { + srcRef, dstRef, err := classifyOperands(args[0], args[1], false, true) + if err != nil { + return printErr(err) + } + opts, err := buildTransferOptions(cmd, f) + if err != nil { + return printErr(err) + } + opts.Recursive = true + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + src, dst, err := openEndpoints(ctx, cmd, srcRef, dstRef, false) + if err != nil { + return printErr(err) + } + plan, err := buildSyncPlan(ctx, src, dst, opts, so) + if err != nil { + return printErr(err) + } + results, err := runTransfers(ctx, plan, opts, os.Stdout, os.Stderr) + return finishTransfer(plan, opts, results, err) +} + +// buildSyncPlan enumerates both sides and decides what to transfer and, with +// --delete, what to remove. Only read-only calls are made. +func buildSyncPlan(ctx context.Context, src, dst endpoint, opts *transferOptions, so syncOptions) (*transferPlan, error) { + plan := &transferPlan{Src: src, Dst: dst} + if err := checkCrossEndpoint(src, dst); err != nil { + return nil, err + } + if src.local() && dst.local() { + return nil, objectstorage.ErrUsagef("both operands are local paths; at least one must be an s3:// location") + } + + srcEntries, err := enumerate(ctx, src, opts.FollowSymlinks) + if err != nil { + return nil, err + } + dstEntries, err := enumerateDestination(ctx, dst, opts.FollowSymlinks) + if err != nil { + return nil, err + } + dstByRel := make(map[string]entry, len(dstEntries)) + for _, e := range dstEntries { + dstByRel[e.Rel] = e + } + + op := opFor(src, dst, false) + download := src.remote() && dst.local() + dstPrefix := "" + if dst.remote() { + dstPrefix = objectstorage.NormalizePrefix(dst.Ref.Key) + } + inSource := make(map[string]bool, len(srcEntries)) + for _, e := range srcEntries { + if !opts.Filters.Include(e.Rel) { + continue + } + inSource[e.Rel] = true + d, exists := dstByRel[e.Rel] + if !syncNeedsTransfer(e, d, exists, so, download) { + continue + } + item := transferItem{Op: op, Size: e.Size, ModTime: e.ModTime} + if src.remote() { + item.Src, item.SrcKey = src.remoteURI(e.Key), e.Key + } else { + item.Src, item.SrcPath = displayLocal(e.Path), e.Path + } + if dst.remote() { + key := dstPrefix + e.Rel + if sameObject(src, dst, e.Key, key) { + continue + } + item.Dst, item.DstKey = dst.remoteURI(key), key + } else { + path, display, err := localRelDestination(dst.Ref.Raw, e.Rel) + if err != nil { + return nil, err + } + item.Dst, item.DstPath = display, path + } + plan.Items = append(plan.Items, item) + } + + if so.Delete { + // A local destination may contain directory symlinks, which the + // enumeration follows by default. Deleting through one of them would + // remove files outside the tree the user pointed at, so a candidate + // whose parent directory resolves outside the destination root is + // skipped with a warning. Removing a link that itself lives inside the + // root stays allowed: only the link goes away, not its target. + root := "" + if dst.local() { + if resolved, err := filepath.EvalSymlinks(dst.Ref.Raw); err == nil { + root = resolved + } + } + for _, d := range dstEntries { + if inSource[d.Rel] || !opts.Filters.Include(d.Rel) { + continue + } + if root != "" && !deleteInsideRoot(root, d.Path) { + objectstorage.Warnf("not deleting %s: it resolves outside %s through a symbolic link; pass --no-follow-symlinks to leave links out of the sync", displayLocal(d.Path), displayLocal(dst.Ref.Raw)) + continue + } + del := deleteItem{Path: d.Path, Key: d.Key} + if dst.remote() { + del.Display = dst.remoteURI(d.Key) + } else { + del.Display = displayLocal(d.Path) + } + plan.Deletes = append(plan.Deletes, del) + } + } + return plan, nil +} + +// deleteInsideRoot reports whether removing path only affects the destination +// tree: its parent directory, with every symbolic link resolved, must be root +// or below it. root is expected to be resolved already. +func deleteInsideRoot(root, path string) bool { + parent, err := filepath.EvalSymlinks(filepath.Dir(path)) + if err != nil { + return false + } + rel, err := filepath.Rel(root, parent) + if err != nil { + return false + } + return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))) +} + +// enumerateDestination lists the destination; a local directory that does +// not exist yet is simply empty. +func enumerateDestination(ctx context.Context, dst endpoint, followSymlinks bool) ([]entry, error) { + if dst.remote() { + return enumerate(ctx, dst, followSymlinks) + } + st, err := os.Stat(dst.Ref.Raw) + switch { + case errors.Is(err, os.ErrNotExist): + return nil, nil + case err != nil: + return nil, localErr(err) + case !st.IsDir(): + return nil, objectstorage.ErrUsagef("%s is not a directory", dst.Ref.Raw) + } + return enumerate(ctx, dst, followSymlinks) +} + +// syncNeedsTransfer applies the comparison rules. Timestamps are compared at +// second precision because S3 does not keep sub-second modification times. +func syncNeedsTransfer(src, dst entry, exists bool, so syncOptions, download bool) bool { + if !exists { + return true + } + if src.Size != dst.Size { + return true + } + if so.SizeOnly { + return false + } + s, d := src.ModTime.Truncate(time.Second), dst.ModTime.Truncate(time.Second) + if download && so.ExactTimestamps { + return !s.Equal(d) + } + return s.After(d) +} diff --git a/cmd/s3/transfer.go b/cmd/s3/transfer.go new file mode 100644 index 0000000..84a50b9 --- /dev/null +++ b/cmd/s3/transfer.go @@ -0,0 +1,1405 @@ +package s3 + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" + "golang.org/x/term" +) + +// Shared machinery for cp, mv and sync: operand classification, the aws +// destination rules, local/remote enumeration, the transfer loop and its +// output. Nothing in this file depends on cobra except the flag registration +// helpers at the bottom, so the core can be tested against s3test. + +// Transfer operation names (also the aws output prefixes). +const ( + opUpload = "upload" + opDownload = "download" + opCopy = "copy" + opMove = "move" + opDelete = "delete" +) + +// stdinPartSize is the smallest multipart part size used for stdin uploads +// (minio's default). S3 allows 10000 parts, so it covers streams up to +// stdinMaxPlainSize; --expected-size grows the parts beyond that. +const stdinPartSize = 16 * 1024 * 1024 + +// stdinMaxParts is the S3 limit on parts per multipart upload. +const stdinMaxParts = 10000 + +// stdinMaxPlainSize is the largest stream that fits in 10000 parts of +// stdinPartSize (160 GiB) without --expected-size. +const stdinMaxPlainSize = int64(stdinPartSize) * stdinMaxParts + +// partialSuffix marks a download in progress; the file is renamed on success. +const partialSuffix = ".lsh-partial" + +// transferOptions carries every knob the transfer core understands. The +// cobra layer fills it from flags; tests build it directly. +type transferOptions struct { + Recursive bool + Filters *objectstorage.Filters + + ContentType string + CacheControl string + ContentEncoding string + ContentDisposition string + ContentLanguage string + Expires time.Time + Metadata map[string]string + NoGuessMime bool + + NoOverwrite bool + ExpectedSize int64 + FollowSymlinks bool + + // Move deletes the source after a successful copy (mv). + Move bool + // DryRun plans and prints without writing anything. + DryRun bool + + // Human prints aws-style lines to Out; otherwise results are only + // collected for the structured renderer. + Human bool + Quiet bool + OnlyShowErrors bool + // ShowProgress enables the single-line progress meter on Hints. + ShowProgress bool + + // Stdin/Stdout back the "-" operands (defaults: os.Stdin/os.Stdout). + Stdin io.Reader + Stdout io.Writer +} + +func (o *transferOptions) showLines() bool { + return o.Human && !o.Quiet && !o.OnlyShowErrors +} + +func (o *transferOptions) stdin() io.Reader { + if o.Stdin != nil { + return o.Stdin + } + return os.Stdin +} + +func (o *transferOptions) stdout() io.Writer { + if o.Stdout != nil { + return o.Stdout + } + return os.Stdout +} + +// endpoint is one side of a transfer: a parsed operand plus, for remote +// references, the open bucket session. +type endpoint struct { + Ref objectstorage.Ref + Sess *Session +} + +func (e endpoint) remote() bool { return e.Ref.Remote } +func (e endpoint) stdio() bool { return e.Ref.Stdio } +func (e endpoint) local() bool { return !e.Ref.Remote && !e.Ref.Stdio } + +// remoteURI renders s3:/// using the bucket token as typed. +func (e endpoint) remoteURI(key string) string { return e.Ref.WithKey(key).String() } + +// transferItem is one planned operation. +type transferItem struct { + Op string + // Display strings for output lines. + Src, Dst string + // SrcPath/DstPath are local paths; SrcKey/DstKey are object keys. The + // unused pair is empty. Stdio sides have both empty. + SrcPath, SrcKey string + DstPath, DstKey string + // Size is known for local sources and listed/stat'ed remote sources. + Size int64 + ModTime time.Time +} + +// transferPlan is the outcome of planning: the two endpoints and the items. +type transferPlan struct { + Src, Dst endpoint + Items []transferItem + // Deletes are destination entries removed by `sync --delete`. + Deletes []deleteItem +} + +// deleteItem is one `sync --delete` removal. +type deleteItem struct { + Display string + Path string // local + Key string // remote +} + +// entry is a file or object discovered while enumerating a side, keyed by +// its path relative to the source directory / prefix (forward slashes). +type entry struct { + Rel string + Path string // local absolute-ish path (as joined from the operand) + Key string // remote key + Size int64 + ModTime time.Time +} + +// --------------------------------------------------------------------------- +// Operand validation + +// classifyOperands validates the source/destination combination shared by +// cp, mv and sync and returns the parsed refs. +func classifyOperands(srcArg, dstArg string, move, sync bool) (objectstorage.Ref, objectstorage.Ref, error) { + src, err := objectstorage.ParseTransferArg(srcArg) + if err != nil { + return src, objectstorage.Ref{}, err + } + dst, err := objectstorage.ParseTransferArg(dstArg) + if err != nil { + return src, dst, err + } + switch { + case !src.Remote && !dst.Remote: + if src.Stdio || dst.Stdio { + return src, dst, objectstorage.ErrUsagef("one of and must be an s3:// location") + } + return src, dst, objectstorage.ErrUsagef("both operands are local paths; at least one must be an s3:// location (use cp(1) for local copies)") + case src.Stdio && sync, dst.Stdio && sync: + return src, dst, objectstorage.ErrUsagef("sync does not support '-' (stdin/stdout)") + case src.Stdio && move, dst.Stdio && move: + return src, dst, objectstorage.ErrUsagef("mv does not support '-' (stdin/stdout); use cp") + } + return src, dst, nil +} + +// sameEndpoint reports whether two buckets live on the same S3 endpoint. +func sameEndpoint(a, b *objectstorage.Bucket) bool { + norm := func(s string) string { return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(s), "/")) } + return norm(a.Endpoint) == norm(b.Endpoint) +} + +// checkCrossEndpoint rejects server-side copies between different endpoints. +func checkCrossEndpoint(src, dst endpoint) error { + if !src.remote() || !dst.remote() { + return nil + } + if sameEndpoint(src.Sess.Bucket, dst.Sess.Bucket) { + return nil + } + return objectstorage.ErrUsagef("buckets %s (%s) and %s (%s) are on different endpoints; server-side copy is only possible within one endpoint — download and re-upload, or use rclone with 'lsh s3 configure export --format rclone'", + src.Sess.Bucket.Display(), src.Sess.Bucket.Endpoint, dst.Sess.Bucket.Display(), dst.Sess.Bucket.Endpoint) +} + +// needsStreamedCopy reports whether a failed server-side copy should be retried +// by streaming the object. Only an authorization failure between two buckets +// that resolved different credentials qualifies: any other error (missing +// object, quota, backend outage) would fail the same way twice. +func needsStreamedCopy(src, dst endpoint, err error) bool { + if !src.remote() || !dst.remote() { + return false + } + if minio.ToErrorResponse(err).Code != "AccessDenied" { + return false + } + return src.Sess.Cred.AccessKeyID != dst.Sess.Cred.AccessKeyID +} + +// streamCopy copies one object through the client: GET with the source session, +// PUT with the destination one. It is the fallback for buckets a single +// credential cannot span, so cp/mv/sync keep working instead of reporting an +// access denial the user cannot act on. +func streamCopy(ctx context.Context, src, dst endpoint, item transferItem, opts *transferOptions, res *objectstorage.TransferResult, hints io.Writer, label string) error { + obj, err := src.Sess.Client.GetObject(ctx, src.Sess.Bucket.BucketName, item.SrcKey, minio.GetObjectOptions{}) + if err != nil { + return err + } + defer obj.Close() + stat, err := obj.Stat() + if err != nil { + // The read is the source's; blaming the destination (as the shared + // humanizer would) points at the wrong bucket and the wrong key. + return src.Sess.humanize(err) + } + meter := meterFor(opts, hints, label, stat.Size) + defer meter.finish() + put := streamedPutOptions(stat, opts, progressReader(meter)) + contentType := put.ContentType + info, err := dst.Sess.Client.PutObject(ctx, dst.Sess.Bucket.BucketName, item.DstKey, obj, stat.Size, put) + if err != nil { + return dst.Sess.humanize(err) + } + res.ETag, res.ContentType = strings.Trim(info.ETag, `"`), contentType + if info.Size > 0 { + res.Size = info.Size + } + return nil +} + +// streamedPutOptions makes a streamed copy keep what a server-side copy would +// have kept: the source's user metadata and its caching/content headers. The +// explicit flags still win — putOptions applies them first and this only fills +// what they left empty — so a cross-credential copy does not silently rewrite +// an object's metadata. +func streamedPutOptions(stat minio.ObjectInfo, opts *transferOptions, progress io.Reader) minio.PutObjectOptions { + put := putOptions(firstNonEmptyStr(opts.ContentType, stat.ContentType), opts, progress) + if len(put.UserMetadata) == 0 { + put.UserMetadata = userMetadataOf(stat.Metadata) + } + if put.CacheControl == "" { + put.CacheControl = stat.Metadata.Get("Cache-Control") + } + if put.ContentEncoding == "" { + put.ContentEncoding = firstNonEmptyStr(stat.ContentEncoding, stat.Metadata.Get("Content-Encoding")) + } + if put.ContentDisposition == "" { + put.ContentDisposition = stat.Metadata.Get("Content-Disposition") + } + if put.ContentLanguage == "" { + put.ContentLanguage = stat.Metadata.Get("Content-Language") + } + if put.Expires.IsZero() { + put.Expires = stat.Expires + } + return put +} + +// userMetadataOf extracts the x-amz-meta-* response headers, stripped of the +// prefix. ObjectInfo.UserMetadata only carries them against MinIO servers, so +// the raw headers are the portable source. +func userMetadataOf(h http.Header) map[string]string { + out := map[string]string{} + for key, values := range h { + lower := strings.ToLower(key) + if name := strings.TrimPrefix(lower, "x-amz-meta-"); name != lower && len(values) > 0 { + out[name] = values[0] + } + } + if len(out) == 0 { + return nil + } + return out +} + +// sameObject reports whether both endpoints name the same object. +func sameObject(src, dst endpoint, srcKey, dstKey string) bool { + if !src.remote() || !dst.remote() { + return false + } + return sameEndpoint(src.Sess.Bucket, dst.Sess.Bucket) && + src.Sess.Bucket.BucketName == dst.Sess.Bucket.BucketName && srcKey == dstKey +} + +// --------------------------------------------------------------------------- +// Planning (cp / mv) + +// buildPlan enumerates the source and applies the aws destination rules. It +// only performs read-only calls (stat, list), so it is safe under --dry-run. +func buildPlan(ctx context.Context, src, dst endpoint, opts *transferOptions) (*transferPlan, error) { + plan := &transferPlan{Src: src, Dst: dst} + op := opFor(src, dst, opts.Move) + + if err := checkCrossEndpoint(src, dst); err != nil { + return nil, err + } + + switch { + case src.stdio(): + if opts.Recursive { + return nil, objectstorage.ErrUsagef("--recursive cannot be combined with stdin ('-')") + } + if dst.Ref.IsDir() { + return nil, objectstorage.ErrUsagef("%s names a prefix; uploading from stdin needs a full object key (s3:///)", dst.Ref) + } + plan.Items = append(plan.Items, transferItem{Op: op, Src: "-", Dst: dst.remoteURI(dst.Ref.Key), DstKey: dst.Ref.Key, Size: opts.ExpectedSize}) + return plan, nil + + case dst.stdio(): + if opts.Recursive { + return nil, objectstorage.ErrUsagef("--recursive cannot be combined with stdout ('-')") + } + if src.Ref.Key == "" || strings.HasSuffix(src.Ref.Key, "/") { + return nil, objectstorage.ErrUsagef("%s names a prefix, not an object; streaming to stdout needs a single object key", src.Ref) + } + info, err := src.Sess.Client.StatObject(ctx, src.Sess.Bucket.BucketName, src.Ref.Key, minio.StatObjectOptions{}) + if err != nil { + return nil, src.Sess.humanize(err) + } + plan.Items = append(plan.Items, transferItem{Op: op, Src: src.remoteURI(src.Ref.Key), Dst: "-", SrcKey: src.Ref.Key, Size: info.Size, ModTime: info.LastModified}) + return plan, nil + + case opts.Recursive: + return buildRecursivePlan(ctx, plan, op, opts) + } + + // Single object / file. + var item transferItem + item.Op = op + switch { + case src.local(): + st, err := os.Stat(src.Ref.Raw) + if err != nil { + return nil, localErr(err) + } + if st.IsDir() { + return nil, objectstorage.ErrUsagef("%s is a directory; use --recursive to copy its contents", src.Ref.Raw) + } + name := filepath.Base(src.Ref.Raw) + key := dst.Ref.Key + if dst.Ref.IsDir() { + key = objectstorage.JoinKey(dst.Ref.Key, name) + } + item.Src, item.SrcPath = displayLocal(src.Ref.Raw), src.Ref.Raw + item.Dst, item.DstKey = dst.remoteURI(key), key + item.Size, item.ModTime = st.Size(), st.ModTime() + if !opts.Filters.Include(name) { + return plan, nil + } + default: // remote source + if src.Ref.Key == "" || strings.HasSuffix(src.Ref.Key, "/") { + return nil, objectstorage.ErrUsagef("%s names a prefix, not an object; use --recursive to copy everything under it", src.Ref) + } + info, err := src.Sess.Client.StatObject(ctx, src.Sess.Bucket.BucketName, src.Ref.Key, minio.StatObjectOptions{}) + if err != nil { + return nil, src.Sess.humanize(err) + } + name := objectstorage.BaseName(src.Ref.Key) + item.Src, item.SrcKey = src.remoteURI(src.Ref.Key), src.Ref.Key + item.Size, item.ModTime = info.Size, info.LastModified + if dst.remote() { + key := dst.Ref.Key + if dst.Ref.IsDir() { + key = objectstorage.JoinKey(dst.Ref.Key, name) + } + if sameObject(src, dst, src.Ref.Key, key) { + return nil, objectstorage.ErrUsagef("source and destination are the same object (%s)", item.Src) + } + item.Dst, item.DstKey = dst.remoteURI(key), key + } else { + path, display, err := localDestination(dst.Ref.Raw, name) + if err != nil { + return nil, err + } + item.Dst, item.DstPath = display, path + } + if !opts.Filters.Include(name) { + return plan, nil + } + } + plan.Items = append(plan.Items, item) + return plan, nil +} + +// buildRecursivePlan handles --recursive: both sides are directories / +// prefixes and every relative path is mirrored. +func buildRecursivePlan(ctx context.Context, plan *transferPlan, op string, opts *transferOptions) (*transferPlan, error) { + src, dst := plan.Src, plan.Dst + entries, err := enumerate(ctx, src, opts.FollowSymlinks) + if err != nil { + return nil, err + } + dstPrefix := "" + if dst.remote() { + dstPrefix = objectstorage.NormalizePrefix(dst.Ref.Key) + } + for _, e := range entries { + if !opts.Filters.Include(e.Rel) { + continue + } + item := transferItem{Op: op, Size: e.Size, ModTime: e.ModTime} + if src.remote() { + item.Src, item.SrcKey = src.remoteURI(e.Key), e.Key + } else { + item.Src, item.SrcPath = displayLocal(e.Path), e.Path + } + if dst.remote() { + key := dstPrefix + e.Rel + if sameObject(src, dst, e.Key, key) { + continue + } + item.Dst, item.DstKey = dst.remoteURI(key), key + } else { + path, display, err := localRelDestination(dst.Ref.Raw, e.Rel) + if err != nil { + return nil, err + } + item.Dst, item.DstPath = display, path + } + plan.Items = append(plan.Items, item) + } + return plan, nil +} + +// opFor picks the output verb for a direction. +func opFor(src, dst endpoint, move bool) string { + if move { + return opMove + } + switch { + case src.remote() && dst.remote(): + return opCopy + case src.remote(): + return opDownload + default: + return opUpload + } +} + +// --------------------------------------------------------------------------- +// Enumeration + +// enumerate lists a directory (local) or a prefix (remote) recursively and +// returns entries keyed by relative path, sorted. +func enumerate(ctx context.Context, e endpoint, followSymlinks bool) ([]entry, error) { + var out []entry + var err error + if e.remote() { + out, err = listRemote(ctx, e.Sess, objectstorage.NormalizePrefix(e.Ref.Key)) + } else { + out, err = walkLocal(e.Ref.Raw, followSymlinks) + } + if err != nil { + return nil, err + } + sort.Slice(out, func(i, j int) bool { return out[i].Rel < out[j].Rel }) + return out, nil +} + +// listRemote lists every object under prefix. Directory markers (keys ending +// in "/") are skipped like aws does. +func listRemote(ctx context.Context, sess *Session, prefix string) ([]entry, error) { + var out []entry + for info := range sess.Client.ListObjects(ctx, sess.Bucket.BucketName, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if info.Err != nil { + return nil, sess.humanize(info.Err) + } + if strings.HasSuffix(info.Key, "/") || len(info.Key) <= len(prefix) { + continue + } + out = append(out, entry{Rel: info.Key[len(prefix):], Key: info.Key, Size: info.Size, ModTime: info.LastModified}) + } + return out, nil +} + +// walkLocal lists the regular files under root. Symlinks are followed (with +// loop protection) when follow is true and ignored otherwise. +func walkLocal(root string, follow bool) ([]entry, error) { + st, err := os.Stat(root) + if err != nil { + return nil, localErr(err) + } + if !st.IsDir() { + return nil, objectstorage.ErrUsagef("%s is not a directory", root) + } + visited := map[string]bool{} + if real, err := filepath.EvalSymlinks(root); err == nil { + visited[real] = true + } + var out []entry + var walk func(dir, relPrefix string) error + walk = func(dir, relPrefix string) error { + des, err := os.ReadDir(dir) + if err != nil { + return localErr(err) + } + for _, d := range des { + path := filepath.Join(dir, d.Name()) + rel := relPrefix + d.Name() + info, err := d.Info() + if err != nil { + return localErr(err) + } + if info.Mode()&os.ModeSymlink != 0 { + if !follow { + continue + } + info, err = os.Stat(path) + if err != nil { + objectstorage.Warnf("skipping %s: %v", path, err) + continue + } + if info.IsDir() { + real, err := filepath.EvalSymlinks(path) + if err != nil || visited[real] { + continue + } + visited[real] = true + } + } + switch { + case info.IsDir(): + if err := walk(path, rel+"/"); err != nil { + return err + } + case info.Mode().IsRegular(): + out = append(out, entry{Rel: rel, Path: path, Size: info.Size(), ModTime: info.ModTime()}) + } + } + return nil + } + if err := walk(root, ""); err != nil { + return nil, err + } + return out, nil +} + +// --------------------------------------------------------------------------- +// Local path helpers + +// displayLocal renders a local path the way aws prints it (os.path.relpath): +// relative to the working directory when possible, with a leading "./" only +// when the result has no directory component ("./dump.sql", "sub/b.txt", +// "../other/x"). Every local side of an output line (sources, single and +// recursive destinations, sync deletes) goes through it. +func displayLocal(path string) string { + if cwd, err := os.Getwd(); err == nil { + abs := path + if !filepath.IsAbs(abs) { + abs = filepath.Join(cwd, path) + } + // Rel fails across Windows volumes; the absolute path is shown then. + if rel, err := filepath.Rel(cwd, abs); err == nil { + path = rel + } + } + if path == "." || path == ".." || strings.ContainsRune(path, filepath.Separator) || filepath.VolumeName(path) != "" { + return path + } + return "." + string(filepath.Separator) + path +} + +// endsWithSeparator reports whether a path ends with a path separator. +func endsWithSeparator(p string) bool { + return strings.HasSuffix(p, "/") || strings.HasSuffix(p, string(filepath.Separator)) +} + +// escapeErr is the exit-7 error for a key that would be written outside the +// destination directory. +func escapeErr(key string) error { + return exitcode.Errorf(exitcode.Refused, "refusing to write %q: the key escapes the destination directory", key) +} + +// safeLocalName validates the file name appended to a destination directory +// for a single download: after FromSlash+Clean it must be a plain name (no +// separator, not "." or "..", no volume name), otherwise the key would land +// outside the directory. +func safeLocalName(name string) (string, error) { + clean := filepath.Clean(filepath.FromSlash(name)) + if clean == "." || clean == ".." || filepath.IsAbs(clean) || strings.ContainsRune(clean, filepath.Separator) || filepath.VolumeName(clean) != "" { + return "", escapeErr(name) + } + return clean, nil +} + +// localDestination applies the aws rule for a single download: an existing +// directory or a trailing separator means "put inside", anything else +// is the literal file path. It returns the path to write and the display +// form; a name that would escape the directory is refused (exit 7). +func localDestination(dst, name string) (path, display string, err error) { + inside := endsWithSeparator(dst) + if !inside { + if st, statErr := os.Stat(dst); statErr == nil && st.IsDir() { + inside = true + } + } + if !inside { + return dst, displayLocal(dst), nil + } + clean, err := safeLocalName(name) + if err != nil { + return "", "", err + } + path = filepath.Join(dst, clean) + return path, displayLocal(path), nil +} + +// localRelDestination joins a relative key path under the destination +// directory for recursive downloads, refusing paths that escape it. +func localRelDestination(dstDir, rel string) (path, display string, err error) { + clean := filepath.Clean(filepath.FromSlash(rel)) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) || filepath.VolumeName(clean) != "" { + return "", "", escapeErr(rel) + } + path = filepath.Join(dstDir, clean) + return path, displayLocal(path), nil +} + +// localErr wraps a filesystem error with an exit code. +func localErr(err error) error { + switch { + case errors.Is(err, os.ErrNotExist): + return exitcode.New(exitcode.NotFound, err) + case errors.Is(err, os.ErrPermission): + return exitcode.New(exitcode.Permission, err) + } + return exitcode.New(exitcode.Generic, err) +} + +// --------------------------------------------------------------------------- +// Content type and headers + +// guessContentType implements the cp rules: explicit flag, else extension, +// else sniffing the first bytes, else application/octet-stream. +// --no-guess-mime-type yields binary/octet-stream (like aws). +func guessContentType(path string, opts *transferOptions) string { + if opts.ContentType != "" { + return opts.ContentType + } + if opts.NoGuessMime { + return "binary/octet-stream" + } + if ext := filepath.Ext(path); ext != "" { + if ct := mime.TypeByExtension(ext); ct != "" { + return ct + } + } + f, err := os.Open(path) + if err != nil { + return "application/octet-stream" + } + defer f.Close() + buf := make([]byte, 512) + n, _ := io.ReadFull(f, buf) + if n == 0 { + return "application/octet-stream" + } + return http.DetectContentType(buf[:n]) +} + +// putOptions builds the minio options shared by uploads. +func putOptions(contentType string, opts *transferOptions, progress io.Reader) minio.PutObjectOptions { + return minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: opts.Metadata, + CacheControl: opts.CacheControl, + ContentEncoding: opts.ContentEncoding, + ContentDisposition: opts.ContentDisposition, + ContentLanguage: opts.ContentLanguage, + Expires: opts.Expires, + Progress: progress, + } +} + +// parseMetadata parses the aws shorthand `k=v,k2=v2` (repeatable, merged). +func parseMetadata(values []string) (map[string]string, error) { + if len(values) == 0 { + return nil, nil + } + out := map[string]string{} + for _, v := range values { + for _, pair := range strings.Split(v, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + k, val, ok := strings.Cut(pair, "=") + k = strings.TrimSpace(k) + if !ok || k == "" { + return nil, objectstorage.ErrUsagef("invalid --metadata %q: expected key=value[,key2=value2]", pair) + } + out[k] = strings.TrimSpace(val) + } + } + return out, nil +} + +// parseExpires accepts RFC3339 or YYYY-MM-DD. +func parseExpires(s string) (time.Time, error) { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{}, nil + } + for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, s); err == nil { + return t, nil + } + } + return time.Time{}, objectstorage.ErrUsagef("invalid --expires %q: use RFC3339 (2026-09-07T15:04:05Z) or YYYY-MM-DD", s) +} + +// --------------------------------------------------------------------------- +// Progress + +// progressMeter is a minio Progress reader that rewrites one stderr line +// every ~200ms. It also works as a counting writer wrapper for downloads. +type progressMeter struct { + w io.Writer + label string + total int64 + done atomic.Int64 + + mu sync.Mutex // guards last and rendered (minio uploads parts concurrently) + last time.Time + rendered bool +} + +func newProgressMeter(w io.Writer, label string, total int64) *progressMeter { + return &progressMeter{w: w, label: label, total: total} +} + +// Read implements minio's progress hook contract: every call reports len(b) +// bytes transferred. +func (p *progressMeter) Read(b []byte) (int, error) { + p.add(int64(len(b))) + return len(b), nil +} + +func (p *progressMeter) add(n int64) { + if p == nil { + return + } + done := p.done.Add(n) + now := time.Now() + p.mu.Lock() + defer p.mu.Unlock() + // total is 0 for a stdin upload: there is no final frame to force, so the + // throttle applies to every update. + if now.Sub(p.last) < 200*time.Millisecond && (p.total <= 0 || done < p.total) { + return + } + p.last = now + p.render(done) +} + +// render draws the line; the caller holds p.mu. +func (p *progressMeter) render(done int64) { + p.rendered = true + if p.total > 0 { + pct := float64(done) / float64(p.total) * 100 + if pct > 100 { + pct = 100 + } + fmt.Fprintf(p.w, "\r\033[K%s: %s / %s (%.0f%%)", p.label, objectstorage.HumanSize(done), objectstorage.HumanSize(p.total), pct) + return + } + fmt.Fprintf(p.w, "\r\033[K%s: %s", p.label, objectstorage.HumanSize(done)) +} + +// finish clears the progress line. +func (p *progressMeter) finish() { + if p == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + if p.rendered { + fmt.Fprint(p.w, "\r\033[K") + } +} + +// countingWriter feeds a progress meter while copying downloads. +type countingWriter struct { + w io.Writer + p *progressMeter +} + +func (c *countingWriter) Write(b []byte) (int, error) { + n, err := c.w.Write(b) + c.p.add(int64(n)) + return n, err +} + +// meterFor returns a progress meter when enabled, nil otherwise. A nil +// *progressMeter must never be passed as minio Progress (it is an interface). +func meterFor(opts *transferOptions, hints io.Writer, label string, total int64) *progressMeter { + if !opts.ShowProgress { + return nil + } + return newProgressMeter(hints, label, total) +} + +func progressReader(p *progressMeter) io.Reader { + if p == nil { + return nil + } + return p +} + +// --------------------------------------------------------------------------- +// Execution + +// runTransfers executes a plan. Human lines go to out, hints/progress/errors +// to hints. It returns the collected results and, when something failed, an +// exit-coded error (the item's own error for a single-item plan, exit 1 +// otherwise). Deletes from sync plans are executed after the copies, and only +// when every copy succeeded (see below). +func runTransfers(ctx context.Context, plan *transferPlan, opts *transferOptions, out, hints io.Writer) ([]objectstorage.TransferResult, error) { + // total is the planned size and never changes: it decides where a failure + // is printed (emit) and whether the caller prints the item's own error. + // attempted excludes deletions that were skipped, so the failure summary + // counts only what ran. + total := len(plan.Items) + len(plan.Deletes) + attempted := total + results := make([]objectstorage.TransferResult, 0, total) + failures := 0 + var firstErr error + + emit := func(res objectstorage.TransferResult) { + results = append(results, res) + if res.Error != "" { + // Failures always go to stderr, even with --quiet. A single-item + // plan returns its error instead, so it is printed once. + if total > 1 { + fmt.Fprintln(hints, humanLine(res)) + } + return + } + if !opts.showLines() { + return + } + if plan.Dst.stdio() { + // stdout carries the object itself, so lines are suppressed — + // except under --dry-run, where nothing is streamed and the plan + // would otherwise be invisible. + if opts.DryRun { + fmt.Fprintln(hints, humanLine(res)) + } + return + } + fmt.Fprintln(out, humanLine(res)) + } + + for _, item := range plan.Items { + if err := ctx.Err(); err != nil { + return results, exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + res := objectstorage.TransferResult{Op: item.Op, Source: item.Src, Destination: item.Dst, Size: item.Size, DryRun: opts.DryRun} + skip, err := shouldSkipExisting(ctx, plan, item, opts, hints) + if err == nil && skip { + continue + } + if err == nil && !opts.DryRun { + err = transferOne(ctx, plan, item, opts, &res, hints) + } + if err != nil { + if ctx.Err() != nil { + return results, exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + err = humanizeFor(plan, err) + res.Error = err.Error() + failures++ + if firstErr == nil { + firstErr = err + } + } + emit(res) + } + + // `sync --delete` only plans deletions for keys the source does not have, + // so applying them after a failed copy would leave the destination with + // neither the object that failed to transfer nor the one that was about to + // be pruned. Skip them instead: re-running the sync applies them once the + // transfers succeed, and nothing has been lost in the meantime. + deletes := plan.Deletes + if failures > 0 && len(deletes) > 0 { + fmt.Fprintf(hints, "warning: %d planned deletion(s) not applied because %d transfer(s) failed; the destination is not in sync — re-run once the transfers succeed\n", len(deletes), failures) + attempted -= len(deletes) + deletes = nil + } + + for _, del := range deletes { + if err := ctx.Err(); err != nil { + return results, exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + res := objectstorage.TransferResult{Op: opDelete, Destination: del.Display, DryRun: opts.DryRun} + if !opts.DryRun { + if err := deleteOne(ctx, plan.Dst, del); err != nil { + if ctx.Err() != nil { + return results, exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + err = humanizeFor(plan, err) + res.Error = err.Error() + failures++ + if firstErr == nil { + firstErr = err + } + } + } + emit(res) + } + + if failures == 0 { + return results, nil + } + if total == 1 { + return results, firstErr + } + return results, exitcode.Errorf(exitcode.Generic, "%d of %d transfers failed", failures, attempted) +} + +// humanLine renders a result as an aws-style line; deletes have no source. +func humanLine(res objectstorage.TransferResult) string { + if res.Op != opDelete { + return res.HumanLine() + } + prefix := "" + if res.DryRun { + prefix = "(dryrun) " + } + if res.Error != "" { + return fmt.Sprintf("%sdelete failed: %s: %s", prefix, res.Destination, res.Error) + } + return fmt.Sprintf("%sdelete: %s", prefix, res.Destination) +} + +// humanizeFor maps an error through the most relevant session. +func humanizeFor(plan *transferPlan, err error) error { + var already *exitcode.Error + if errors.As(err, &already) { + return err + } + if plan.Dst.remote() { + return plan.Dst.Sess.humanize(err) + } + if plan.Src.remote() { + return plan.Src.Sess.humanize(err) + } + return objectstorage.Humanize(err, nil, nil) +} + +// shouldSkipExisting implements --no-overwrite: an existing destination is +// left alone and reported on hints. +func shouldSkipExisting(ctx context.Context, plan *transferPlan, item transferItem, opts *transferOptions, hints io.Writer) (bool, error) { + if !opts.NoOverwrite || plan.Dst.stdio() { + return false, nil + } + exists := false + if plan.Dst.remote() { + _, err := plan.Dst.Sess.Client.StatObject(ctx, plan.Dst.Sess.Bucket.BucketName, item.DstKey, minio.StatObjectOptions{}) + switch { + case err == nil: + exists = true + case minio.ToErrorResponse(err).StatusCode == http.StatusNotFound: + default: + return false, err + } + } else { + _, err := os.Stat(item.DstPath) + switch { + case err == nil: + exists = true + case errors.Is(err, os.ErrNotExist): + default: + return false, err + } + } + if exists { + fmt.Fprintf(hints, "skip: %s already exists (--no-overwrite)\n", item.Dst) + } + return exists, nil +} + +// transferOne performs a single planned item and fills the result. +func transferOne(ctx context.Context, plan *transferPlan, item transferItem, opts *transferOptions, res *objectstorage.TransferResult, hints io.Writer) error { + src, dst := plan.Src, plan.Dst + label := fmt.Sprintf("%s -> %s", item.Src, item.Dst) + switch { + case src.stdio(): + return uploadStdin(ctx, dst, item, opts, res, hints) + case dst.stdio(): + return download(ctx, src, item, opts, res, opts.stdout(), nil) + case src.local() && dst.remote(): + meter := meterFor(opts, hints, label, item.Size) + defer meter.finish() + ct := guessContentType(item.SrcPath, opts) + info, err := dst.Sess.Client.FPutObject(ctx, dst.Sess.Bucket.BucketName, item.DstKey, item.SrcPath, putOptions(ct, opts, progressReader(meter))) + if err != nil { + return err + } + res.ETag, res.ContentType = strings.Trim(info.ETag, `"`), ct + if info.Size > 0 { + res.Size = info.Size + } + if opts.Move { + return os.Remove(item.SrcPath) + } + return nil + case src.remote() && dst.local(): + meter := meterFor(opts, hints, label, item.Size) + defer meter.finish() + if err := download(ctx, src, item, opts, res, nil, meter); err != nil { + return err + } + if opts.Move { + return src.Sess.Client.RemoveObject(ctx, src.Sess.Bucket.BucketName, item.SrcKey, minio.RemoveObjectOptions{}) + } + return nil + default: // remote -> remote + dstOpts := minio.CopyDestOptions{Bucket: dst.Sess.Bucket.BucketName, Object: item.DstKey} + if len(opts.Metadata) > 0 || opts.ContentType != "" || opts.CacheControl != "" || opts.ContentEncoding != "" || opts.ContentDisposition != "" || opts.ContentLanguage != "" || !opts.Expires.IsZero() { + dstOpts.ReplaceMetadata = true + dstOpts.UserMetadata = opts.Metadata + dstOpts.ContentType = opts.ContentType + dstOpts.CacheControl = opts.CacheControl + dstOpts.ContentEncoding = opts.ContentEncoding + dstOpts.ContentDisposition = opts.ContentDisposition + dstOpts.ContentLanguage = opts.ContentLanguage + dstOpts.Expires = opts.Expires + } + info, err := dst.Sess.Client.CopyObject(ctx, dstOpts, minio.CopySrcOptions{Bucket: src.Sess.Bucket.BucketName, Object: item.SrcKey}) + switch { + case err == nil: + res.ETag = strings.Trim(info.ETag, `"`) + case needsStreamedCopy(src, dst, err): + // A CopyObject carries a single identity: it is signed with the + // destination credential and the backend reads the source as that + // same identity. When the two buckets resolved different keys the + // destination key cannot read the source, so the object is streamed + // through this machine instead — read with the source credential, + // written with the destination one. + fmt.Fprintf(hints, "%s and %s resolved different access keys, so the copy streams through this machine instead of the backend\n", src.Sess.Bucket.Display(), dst.Sess.Bucket.Display()) + if streamErr := streamCopy(ctx, src, dst, item, opts, res, hints, label); streamErr != nil { + return streamErr + } + default: + return err + } + if opts.Move { + return src.Sess.Client.RemoveObject(ctx, src.Sess.Bucket.BucketName, item.SrcKey, minio.RemoveObjectOptions{}) + } + return nil + } +} + +// stdinPartSizeFor picks the multipart part size for a stdin upload. Without +// --expected-size it is stdinPartSize; with it, the size minio would use for +// an object of that length (so streams above 160 GiB still fit in 10000 +// parts), never below stdinPartSize. The value only shapes the parts: the +// upload itself is always length-unknown and succeeds whatever the real size. +func stdinPartSizeFor(expected int64) uint64 { + if expected <= 0 { + return stdinPartSize + } + _, partSize, _, err := minio.OptimalPartInfo(expected, 0) + if err != nil || partSize < stdinPartSize { + return stdinPartSize + } + return uint64(partSize) +} + +// uploadStdin streams stdin as a multipart upload of unknown length. minio +// reads one part at a time into a buffer of the part size, so memory use is +// one part; --expected-size only sizes the parts and the progress total. +func uploadStdin(ctx context.Context, dst endpoint, item transferItem, opts *transferOptions, res *objectstorage.TransferResult, hints io.Writer) error { + rd := bufio.NewReaderSize(opts.stdin(), 512) + ct := opts.ContentType + switch { + case ct != "": + case opts.NoGuessMime: + ct = "binary/octet-stream" + default: + ct = "application/octet-stream" + if ext := filepath.Ext(item.DstKey); ext != "" { + if byExt := mime.TypeByExtension(ext); byExt != "" { + ct = byExt + } + } + if ct == "application/octet-stream" { + if head, _ := rd.Peek(512); len(head) > 0 { + ct = http.DetectContentType(head) + } + } + } + partSize := stdinPartSizeFor(opts.ExpectedSize) + if opts.ExpectedSize <= 0 { + fmt.Fprintf(hints, "note: streaming from stdin in %s parts (streams up to %s; pass --expected-size for larger ones)\n", objectstorage.HumanSize(stdinPartSize), objectstorage.HumanSize(stdinMaxPlainSize)) + } + meter := meterFor(opts, hints, "- -> "+item.Dst, opts.ExpectedSize) + defer meter.finish() + po := putOptions(ct, opts, progressReader(meter)) + po.PartSize = partSize + // Size -1: the real length is whatever stdin delivers. Passing the + // expected size here would make minio fail on a shorter stream and + // truncate a longer one. + info, err := dst.Sess.Client.PutObject(ctx, dst.Sess.Bucket.BucketName, item.DstKey, rd, -1, po) + if err != nil { + return err + } + res.ETag, res.ContentType, res.Size = strings.Trim(info.ETag, `"`), ct, info.Size + return nil +} + +// download fetches one object either into w (stdout) or into item.DstPath +// through a temporary "..lsh-partial" file renamed on success. The +// local file's mtime is set to the object's LastModified so sync can compare. +func download(ctx context.Context, src endpoint, item transferItem, opts *transferOptions, res *objectstorage.TransferResult, w io.Writer, meter *progressMeter) error { + obj, err := src.Sess.Client.GetObject(ctx, src.Sess.Bucket.BucketName, item.SrcKey, minio.GetObjectOptions{}) + if err != nil { + return err + } + defer obj.Close() + info, err := obj.Stat() + if err != nil { + return err + } + res.Size, res.ETag, res.ContentType = info.Size, strings.Trim(info.ETag, `"`), info.ContentType + + var body io.Reader = obj + if w != nil { + if meter != nil { + _, err = io.Copy(&countingWriter{w: w, p: meter}, body) + } else { + _, err = io.Copy(w, body) + } + return err + } + + dir := filepath.Dir(item.DstPath) + if err := os.MkdirAll(dir, 0o755); err != nil { + return localErr(err) + } + tmp := filepath.Join(dir, "."+filepath.Base(item.DstPath)+partialSuffix) + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return localErr(err) + } + var dst io.Writer = f + if meter != nil { + dst = &countingWriter{w: f, p: meter} + } + if _, err := io.Copy(dst, body); err != nil { + f.Close() + os.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return localErr(err) + } + if err := os.Rename(tmp, item.DstPath); err != nil { + os.Remove(tmp) + return localErr(err) + } + if !info.LastModified.IsZero() { + _ = os.Chtimes(item.DstPath, info.LastModified, info.LastModified) + } + return nil +} + +// deleteOne removes a destination entry (sync --delete). +func deleteOne(ctx context.Context, dst endpoint, del deleteItem) error { + if dst.remote() { + return dst.Sess.Client.RemoveObject(ctx, dst.Sess.Bucket.BucketName, del.Key, minio.RemoveObjectOptions{}) + } + if err := os.Remove(del.Path); err != nil && !errors.Is(err, os.ErrNotExist) { + return localErr(err) + } + return nil +} + +// --------------------------------------------------------------------------- +// Cobra glue shared by cp, mv and sync + +// transferFlags holds the raw flag values before conversion. +type transferFlags struct { + recursive bool + filters objectstorage.Filters + contentType string + cacheControl string + contentEncoding string + contentDisposition string + contentLanguage string + expires string + metadata []string + noGuessMime bool + noOverwrite bool + expectedSize int64 + quiet bool + onlyShowErrors bool + noProgress bool + followSymlinks bool + noFollowSymlinks bool +} + +// addTransferFlags registers the flags shared by cp, mv and sync. recursive +// controls whether --recursive/-r is offered (sync is always recursive). +func addTransferFlags(cmd *cobra.Command, f *transferFlags, recursive bool) { + fs := cmd.Flags() + if recursive { + fs.BoolVarP(&f.recursive, "recursive", "r", false, "copy every file/object under the source directory or prefix, mirroring relative paths") + } + objectstorage.FilterFlags(fs, &f.filters) + fs.StringVar(&f.contentType, "content-type", "", "Content-Type to store (default: guessed from the extension, then the content)") + fs.StringVar(&f.cacheControl, "cache-control", "", "Cache-Control header to store with the object") + fs.StringVar(&f.contentEncoding, "content-encoding", "", "Content-Encoding header to store with the object") + fs.StringVar(&f.contentDisposition, "content-disposition", "", "Content-Disposition header to store with the object") + fs.StringVar(&f.contentLanguage, "content-language", "", "Content-Language header to store with the object") + fs.StringVar(&f.expires, "expires", "", "Expires header (RFC3339 or YYYY-MM-DD)") + fs.StringArrayVar(&f.metadata, "metadata", nil, "user metadata as key=value[,key2=value2] (repeatable, merged)") + fs.BoolVar(&f.noGuessMime, "no-guess-mime-type", false, "store binary/octet-stream instead of guessing the Content-Type") + fs.BoolVar(&f.noOverwrite, "no-overwrite", false, "skip destinations that already exist") + fs.Int64Var(&f.expectedSize, "expected-size", 0, "approximate size in bytes of a stdin upload; only sizes the multipart parts (needed above 160 GiB) and the progress total, the upload succeeds whatever the real length") + fs.BoolVarP(&f.quiet, "quiet", "q", false, "do not print the per-object lines") + fs.BoolVar(&f.onlyShowErrors, "only-show-errors", false, "print only errors and warnings") + fs.BoolVar(&f.noProgress, "no-progress", false, "do not show the progress meter") + fs.BoolVar(&f.followSymlinks, "follow-symlinks", true, "follow symbolic links when enumerating local directories") + fs.BoolVar(&f.noFollowSymlinks, "no-follow-symlinks", false, "ignore symbolic links when enumerating local directories") + addProjectFlag(cmd, true, "project that owns the bucket(s) (disambiguates names shared across projects)") + + // `aws s3 cp ... --region x` gets the directed explanation (exit 2). + rejectRegionFlag(cmd) + unsupportedAWSFlags(cmd, map[string]string{ + "acl": "buckets are private; grant access with scoped access keys (lsh s3 access-keys create)", + "grants": "buckets are private; grant access with scoped access keys (lsh s3 access-keys create)", + "sse": "server-side encryption is managed by the platform", + "sse-c": "customer-provided encryption keys are not supported", + "sse-c-key": "customer-provided encryption keys are not supported", + "sse-kms-key-id": "KMS is not available on Latitude object storage", + "sse-c-copy-source": "customer-provided encryption keys are not supported", + "sse-c-copy-source-key": "customer-provided encryption keys are not supported", + "metadata-directive": "server-side copies keep the source metadata; pass --metadata/--content-type to replace it", + "copy-props": "server-side copies always keep the source properties", + "checksum-algorithm": "the CLI uses Content-MD5; additional checksums are rejected by S3-compatible backends", + "checksum-mode": "checksum validation on download is not available", + "request-payer": "there is no requester-pays billing on Latitude", + "storage-class": "storage class is a bucket attribute on Latitude (standard|high_performance); set it with lsh s3 create-bucket --storage-class", + }) + if sse := fs.Lookup("sse"); sse != nil { + sse.NoOptDefVal = "AES256" + } +} + +// buildTransferOptions converts flags into transferOptions. +func buildTransferOptions(cmd *cobra.Command, f *transferFlags) (*transferOptions, error) { + meta, err := parseMetadata(f.metadata) + if err != nil { + return nil, err + } + expires, err := parseExpires(f.expires) + if err != nil { + return nil, err + } + if f.expectedSize < 0 { + return nil, objectstorage.ErrUsagef("--expected-size must be a positive number of bytes") + } + follow := f.followSymlinks + if f.noFollowSymlinks { + follow = false + } + human := isHuman() + return &transferOptions{ + Recursive: f.recursive, + Filters: &f.filters, + ContentType: f.contentType, + CacheControl: f.cacheControl, + ContentEncoding: f.contentEncoding, + ContentDisposition: f.contentDisposition, + ContentLanguage: f.contentLanguage, + Expires: expires, + Metadata: meta, + NoGuessMime: f.noGuessMime, + NoOverwrite: f.noOverwrite, + ExpectedSize: f.expectedSize, + FollowSymlinks: follow, + DryRun: dryRun(), + Human: human, + Quiet: f.quiet, + OnlyShowErrors: f.onlyShowErrors, + ShowProgress: human && !f.quiet && !f.onlyShowErrors && !f.noProgress && stderrIsTerminal(), + }, nil +} + +// stderrIsTerminal reports whether progress can be drawn. +func stderrIsTerminal() bool { return term.IsTerminal(int(os.Stderr.Fd())) } + +// openEndpoints resolves the remote side(s) of a transfer. The destination +// always needs write permission; the source needs it only for mv. +func openEndpoints(ctx context.Context, cmd *cobra.Command, src, dst objectstorage.Ref, move bool) (endpoint, endpoint, error) { + s, d := endpoint{Ref: src}, endpoint{Ref: dst} + var err error + if src.Remote { + if s.Sess, err = openBucket(ctx, cmd, src.Bucket, move); err != nil { + return s, d, err + } + } + if !dst.Remote { + return s, d, nil + } + switch { + case s.Sess != nil && src.Bucket == dst.Bucket && move: + // Same bucket and the source session already has write permission. + d.Sess = s.Sess + case s.Sess != nil && src.Bucket == dst.Bucket: + // Same bucket: skip the second API lookup but select a write credential. + d.Sess, err = openResolved(cmd, s.Sess.Bucket, true) + default: + d.Sess, err = openBucket(ctx, cmd, dst.Bucket, true) + } + return s, d, err +} + +// finishTransfer renders structured output and returns the command error. +func finishTransfer(plan *transferPlan, opts *transferOptions, results []objectstorage.TransferResult, err error) error { + if !opts.Human && !plan.Dst.stdio() { + render(objectstorage.AsResponseData(results)) + } + if err != nil { + return printErr(err) + } + return nil +} + +// runCopyCommand is the RunE body shared by cp and mv. +func runCopyCommand(cmd *cobra.Command, args []string, f *transferFlags, move bool) error { + srcRef, dstRef, err := classifyOperands(args[0], args[1], move, false) + if err != nil { + return printErr(err) + } + opts, err := buildTransferOptions(cmd, f) + if err != nil { + return printErr(err) + } + opts.Move = move + + ctx, stop := objectstorage.SignalContext(context.Background()) + defer stop() + + src, dst, err := openEndpoints(ctx, cmd, srcRef, dstRef, move) + if err != nil { + return printErr(err) + } + plan, err := buildPlan(ctx, src, dst, opts) + if err != nil { + return printErr(err) + } + if len(plan.Items) == 0 && opts.showLines() && !dst.stdio() { + objectstorage.Hintf("nothing to %s", verbFor(move)) + } + results, err := runTransfers(ctx, plan, opts, os.Stdout, os.Stderr) + return finishTransfer(plan, opts, results, err) +} + +func verbFor(move bool) string { + if move { + return "move" + } + return "copy" +} + +// destinationRulesHelp documents the aws fileformat rules in --help. +const destinationRulesHelp = `Destination rules: + s3:// destination ending in "/" or naming only the bucket -> the source file name is appended + s3:// destination without a trailing "/" -> the key is used literally + (cp dump.sql s3://b/2026/09 creates the key "2026/09"; the output shows it) + local destination that is an existing directory or ends in the path separator + -> the file name is appended (directories are created) + any other local destination -> literal file path + --recursive forces directory semantics on both sides and mirrors paths relative to the source. + +"-" reads the source from stdin or writes the destination to stdout (nothing else is printed on stdout). +Both operands remote -> server-side copy, only within the same endpoint.` diff --git a/cmd/s3/transfer_test.go b/cmd/s3/transfer_test.go new file mode 100644 index 0000000..86b379c --- /dev/null +++ b/cmd/s3/transfer_test.go @@ -0,0 +1,1194 @@ +package s3 + +import ( + "bytes" + "context" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/objectstorage/s3test" + "github.com/minio/minio-go/v7" + "github.com/spf13/cobra" +) + +// newTestSession builds a Session against the fake server for bucket name. +func newTestSession(t *testing.T, srv *s3test.Server, name string) *Session { + t.Helper() + b := &objectstorage.Bucket{ID: "bkt_1", Name: "b", BucketName: name, Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: "us-east-1"} + cred := objectstorage.NewCredential("AK", "SK", "test") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +func remoteEP(sess *Session, key string) endpoint { + raw := "s3://" + sess.Bucket.Name + if key != "" { + raw += "/" + key + } + return endpoint{Ref: objectstorage.Ref{Raw: raw, Bucket: sess.Bucket.Name, Key: key, Remote: true, HadScheme: true}, Sess: sess} +} + +func localEP(path string) endpoint { return endpoint{Ref: objectstorage.Ref{Raw: path}} } + +func stdioEP() endpoint { return endpoint{Ref: objectstorage.Ref{Raw: "-", Stdio: true}} } + +func baseOpts() *transferOptions { + return &transferOptions{Human: true, FollowSymlinks: true, Filters: &objectstorage.Filters{}} +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// run plans and executes a cp/mv between two endpoints. +func run(t *testing.T, src, dst endpoint, opts *transferOptions) ([]objectstorage.TransferResult, string, string, error) { + t.Helper() + ctx := context.Background() + plan, err := buildPlan(ctx, src, dst, opts) + if err != nil { + return nil, "", "", err + } + var out, hints bytes.Buffer + results, err := runTransfers(ctx, plan, opts, &out, &hints) + return results, out.String(), hints.String(), err +} + +func TestUploadDownloadRoundTrip(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + + // Local paths are displayed relative to the working directory (aws + // behaviour), so the test runs inside the temp dir with relative operands. + dir := t.TempDir() + t.Chdir(dir) + file := "report.json" + writeFile(t, file, `{"ok":true}`) + + opts := baseOpts() + opts.Metadata = map[string]string{"owner": "ops", "team": "infra"} + opts.CacheControl = "max-age=60" + opts.ContentLanguage = "en" + results, out, _, err := run(t, localEP(file), remoteEP(sess, "2026/09/"), opts) + if err != nil { + t.Fatalf("upload: %v", err) + } + if want := "upload: ./report.json to s3://b/2026/09/report.json\n"; out != want { + t.Errorf("stdout = %q, want %q", out, want) + } + if len(results) != 1 || results[0].Op != "upload" || results[0].ContentType != "application/json" { + t.Errorf("results = %+v", results) + } + obj := srv.Object("bucket-a", "2026/09/report.json") + if obj == nil { + t.Fatalf("object not stored; keys=%v", srv.Keys("bucket-a")) + } + if string(obj.Data) != `{"ok":true}` { + t.Errorf("data = %q", obj.Data) + } + if obj.ContentType != "application/json" { + t.Errorf("content type = %q, want application/json (guessed from extension)", obj.ContentType) + } + if obj.Metadata["owner"] != "ops" || obj.Metadata["team"] != "infra" { + t.Errorf("metadata = %v", obj.Metadata) + } + if obj.Headers["Cache-Control"] != "max-age=60" || obj.Headers["Content-Language"] != "en" { + t.Errorf("headers = %v", obj.Headers) + } + if srv.HasChecksumHeaders() { + t.Error("upload must not send checksum trailers") + } + + // Download into a directory (trailing separator appends the base name); + // the destination is shown relative to cwd without "./" because it has a + // directory component. + dest := "restore" + string(filepath.Separator) + results, out, _, err = run(t, remoteEP(sess, "2026/09/report.json"), localEP(dest), baseOpts()) + if err != nil { + t.Fatalf("download: %v", err) + } + if want := "download: s3://b/2026/09/report.json to " + filepath.Join("restore", "report.json") + "\n"; out != want { + t.Errorf("stdout = %q, want %q", out, want) + } + got, err := os.ReadFile(filepath.Join(dir, "restore", "report.json")) + if err != nil || string(got) != `{"ok":true}` { + t.Fatalf("downloaded content = %q, err = %v", got, err) + } + if results[0].Size != int64(len(`{"ok":true}`)) || results[0].ContentType != "application/json" { + t.Errorf("download result = %+v", results[0]) + } + if _, err := os.Stat(filepath.Join(dir, "restore", ".report.json"+partialSuffix)); !os.IsNotExist(err) { + t.Error("partial file must be renamed away") + } +} + +func TestContentTypeSniffAndNoGuess(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + + // No extension: sniffed from content. + html := filepath.Join(dir, "page") + writeFile(t, html, "hi") + if _, _, _, err := run(t, localEP(html), remoteEP(sess, ""), baseOpts()); err != nil { + t.Fatal(err) + } + if ct := srv.Object("bucket-a", "page").ContentType; !strings.HasPrefix(ct, "text/html") { + t.Errorf("sniffed content type = %q", ct) + } + + // --no-guess-mime-type stores binary/octet-stream. + opts := baseOpts() + opts.NoGuessMime = true + if _, _, _, err := run(t, localEP(html), remoteEP(sess, "raw"), opts); err != nil { + t.Fatal(err) + } + if ct := srv.Object("bucket-a", "raw").ContentType; ct != "binary/octet-stream" { + t.Errorf("no-guess content type = %q", ct) + } + + // Explicit flag wins. + opts = baseOpts() + opts.ContentType = "application/x-custom" + if _, _, _, err := run(t, localEP(html), remoteEP(sess, "custom"), opts); err != nil { + t.Fatal(err) + } + if ct := srv.Object("bucket-a", "custom").ContentType; ct != "application/x-custom" { + t.Errorf("explicit content type = %q", ct) + } +} + +// The destination rules from the aws user guide (and the spec table). +func TestDestinationRules(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "dir/file.txt", []byte("remote"), "text/plain") + sess := newTestSession(t, srv, "bucket-a") + + // Run inside the temp dir: local display strings are relative to cwd. + t.Chdir(t.TempDir()) + file := "file.txt" + writeFile(t, file, "local") + existing := "existing" + if err := os.Mkdir(existing, 0o755); err != nil { + t.Fatal(err) + } + tree := "tree" + writeFile(t, filepath.Join(tree, "a.txt"), "a") + writeFile(t, filepath.Join(tree, "sub", "b.txt"), "b") + + cases := []struct { + name string + src, dst endpoint + recursive bool + wantKeys []string // remote destination keys + wantPaths []string // local destination paths + wantDst []string // display strings + }{ + {"file to bucket root", localEP(file), remoteEP(sess, ""), false, []string{"file.txt"}, nil, []string{"s3://b/file.txt"}}, + {"file to prefix with slash", localEP(file), remoteEP(sess, "2026/09/"), false, []string{"2026/09/file.txt"}, nil, []string{"s3://b/2026/09/file.txt"}}, + {"file to literal key", localEP(file), remoteEP(sess, "2026/09"), false, []string{"2026/09"}, nil, []string{"s3://b/2026/09"}}, + {"file to renamed key", localEP(file), remoteEP(sess, "renamed.bin"), false, []string{"renamed.bin"}, nil, []string{"s3://b/renamed.bin"}}, + // Destinations with a directory component are shown without "./", + // a bare file name with it (aws relpath rules). + {"object to existing dir", remoteEP(sess, "dir/file.txt"), localEP(existing), false, nil, []string{filepath.Join(existing, "file.txt")}, []string{filepath.Join(existing, "file.txt")}}, + {"object to new dir with slash", remoteEP(sess, "dir/file.txt"), localEP("new/"), false, nil, []string{filepath.Join("new", "file.txt")}, []string{filepath.Join("new", "file.txt")}}, + {"object to literal file", remoteEP(sess, "dir/file.txt"), localEP("copy.txt"), false, nil, []string{"copy.txt"}, []string{"." + string(filepath.Separator) + "copy.txt"}}, + {"recursive dir to prefix without slash", localEP(tree), remoteEP(sess, "pre"), true, []string{"pre/a.txt", "pre/sub/b.txt"}, nil, []string{"s3://b/pre/a.txt", "s3://b/pre/sub/b.txt"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + opts := baseOpts() + opts.Recursive = tc.recursive + plan, err := buildPlan(context.Background(), tc.src, tc.dst, opts) + if err != nil { + t.Fatalf("buildPlan: %v", err) + } + var keys, paths, dsts []string + for _, it := range plan.Items { + if it.DstKey != "" { + keys = append(keys, it.DstKey) + } + if it.DstPath != "" { + paths = append(paths, it.DstPath) + } + dsts = append(dsts, it.Dst) + } + if strings.Join(keys, ",") != strings.Join(tc.wantKeys, ",") { + t.Errorf("keys = %v, want %v", keys, tc.wantKeys) + } + if strings.Join(paths, ",") != strings.Join(tc.wantPaths, ",") { + t.Errorf("paths = %v, want %v", paths, tc.wantPaths) + } + if strings.Join(dsts, ",") != strings.Join(tc.wantDst, ",") { + t.Errorf("display = %v, want %v", dsts, tc.wantDst) + } + }) + } +} + +func TestSingleObjectUsageErrors(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + + // Directory without --recursive. + if _, err := buildPlan(context.Background(), localEP(dir), remoteEP(sess, ""), baseOpts()); exitcode.Of(err) != exitcode.Usage { + t.Errorf("dir without --recursive: err = %v", err) + } + // Prefix source without --recursive. + if _, err := buildPlan(context.Background(), remoteEP(sess, "pre/"), localEP(dir), baseOpts()); exitcode.Of(err) != exitcode.Usage { + t.Errorf("prefix without --recursive: err = %v", err) + } + // Missing object -> not found. + if _, err := buildPlan(context.Background(), remoteEP(sess, "missing.txt"), localEP(dir), baseOpts()); exitcode.Of(err) != exitcode.NotFound { + t.Errorf("missing object: err = %v", err) + } + // Missing local file -> not found. + if _, err := buildPlan(context.Background(), localEP(filepath.Join(dir, "nope")), remoteEP(sess, ""), baseOpts()); exitcode.Of(err) != exitcode.NotFound { + t.Errorf("missing file: err = %v", err) + } + // Recursive download refuses keys escaping the destination. + srv.AddObject("bucket-a", "pre/../../etc/passwd", []byte("x"), "") + opts := baseOpts() + opts.Recursive = true + if _, err := buildPlan(context.Background(), remoteEP(sess, "pre/"), localEP(dir), opts); exitcode.Of(err) != exitcode.Refused { + t.Errorf("escaping key: err = %v", err) + } + // Single-object download into a directory applies the same guard to the + // appended name (exit 7); a literal file destination never uses the name. + for _, name := range []string{"..", ".", "sub/x", "../x"} { + if _, _, err := localDestination(dir, name); exitcode.Of(err) != exitcode.Refused || !strings.Contains(err.Error(), "escapes the destination directory") { + t.Errorf("localDestination(dir, %q): err = %v", name, err) + } + } + if _, _, err := localDestination(filepath.Join(dir, "literal.txt"), ".."); err != nil { + t.Errorf("literal destination must not check the name: %v", err) + } + srv.AddObject("bucket-a", "pre/..", []byte("x"), "") + if _, err := buildPlan(context.Background(), remoteEP(sess, "pre/.."), localEP(dir), baseOpts()); exitcode.Of(err) != exitcode.Refused { + t.Errorf("single download of an escaping key: err = %v", err) + } +} + +func TestClassifyOperands(t *testing.T) { + if _, _, err := classifyOperands("./a", "./b", false, false); exitcode.Of(err) != exitcode.Usage { + t.Errorf("local->local: %v", err) + } + if _, _, err := classifyOperands("-", "./b", false, false); exitcode.Of(err) != exitcode.Usage { + t.Errorf("stdin->local: %v", err) + } + if _, _, err := classifyOperands("-", "s3://b/k", true, false); exitcode.Of(err) != exitcode.Usage { + t.Errorf("mv from stdin: %v", err) + } + if _, _, err := classifyOperands("./a", "s3://b/", false, true); err != nil { + t.Errorf("sync local->remote: %v", err) + } + src, dst, err := classifyOperands("s3://b/k", "-", false, false) + if err != nil || !src.Remote || !dst.Stdio { + t.Errorf("remote->stdout: %v %+v %+v", err, src, dst) + } +} + +func TestNoOverwrite(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + file := filepath.Join(dir, "f.txt") + writeFile(t, file, "v1") + + opts := baseOpts() + opts.NoOverwrite = true + if _, _, _, err := run(t, localEP(file), remoteEP(sess, ""), opts); err != nil { + t.Fatal(err) + } + writeFile(t, file, "v2") + srv.ResetRequests() + results, out, hints, err := run(t, localEP(file), remoteEP(sess, ""), opts) + if err != nil { + t.Fatal(err) + } + if len(results) != 0 || out != "" { + t.Errorf("skipped transfer must produce no result/line: %+v %q", results, out) + } + if !strings.Contains(hints, "skip: s3://b/f.txt already exists") { + t.Errorf("hints = %q", hints) + } + if n := len(srv.WriteRequests()); n != 0 { + t.Errorf("no-overwrite issued %d write requests", n) + } + if string(srv.Object("bucket-a", "f.txt").Data) != "v1" { + t.Error("object was overwritten") + } + + // Local destination is honoured too. + dest := filepath.Join(dir, "out.txt") + writeFile(t, dest, "keep") + if _, _, hints, err = run(t, remoteEP(sess, "f.txt"), localEP(dest), opts); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(dest); string(got) != "keep" || !strings.Contains(hints, "skip: ") { + t.Errorf("local file overwritten: %q hints=%q", got, hints) + } +} + +func TestDryRunMakesNoWrites(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "pre/x.txt", []byte("x"), "") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + t.Chdir(dir) // display strings are relative to cwd + writeFile(t, filepath.Join(dir, "a.txt"), "a") + writeFile(t, filepath.Join(dir, "sub", "b.txt"), "b") + + opts := baseOpts() + opts.Recursive, opts.DryRun = true, true + results, out, _, err := run(t, localEP("."), remoteEP(sess, "pre/"), opts) + if err != nil { + t.Fatal(err) + } + want := "(dryrun) upload: ./a.txt to s3://b/pre/a.txt\n" + + "(dryrun) upload: " + filepath.Join("sub", "b.txt") + " to s3://b/pre/sub/b.txt\n" + if out != want { + t.Errorf("stdout = %q, want %q", out, want) + } + if len(results) != 2 || !results[0].DryRun { + t.Errorf("results = %+v", results) + } + // Dry-run download and mv too. + opts.Move = true + if _, out, _, err = run(t, remoteEP(sess, "pre/"), localEP(filepath.Join(dir, "down")), opts); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(out, "(dryrun) move: s3://b/pre/x.txt to ") { + t.Errorf("stdout = %q", out) + } + if srv.Object("bucket-a", "pre/x.txt") == nil { + t.Error("dry-run mv deleted the source") + } + if _, err := os.Stat(filepath.Join(dir, "down")); !os.IsNotExist(err) { + t.Error("dry-run created the destination directory") + } + if w := srv.WriteRequests(); len(w) != 0 { + t.Errorf("dry-run issued %d write requests: %+v", len(w), w[0].Method+" "+w[0].Path) + } +} + +func TestRecursiveUploadWithFilters(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "a.log"), "a") + writeFile(t, filepath.Join(dir, "b.txt"), "b") + writeFile(t, filepath.Join(dir, "sub", "c.log"), "c") + writeFile(t, filepath.Join(dir, "sub", "d.tmp"), "d") + + opts := baseOpts() + opts.Recursive = true + _ = opts.Filters.Add(false, "*") + _ = opts.Filters.Add(true, "*.log") + results, out, _, err := run(t, localEP(dir), remoteEP(sess, "logs/"), opts) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(srv.Keys("bucket-a"), ","); got != "logs/a.log,logs/sub/c.log" { + t.Errorf("keys = %s", got) + } + if len(results) != 2 || strings.Count(out, "upload: ") != 2 { + t.Errorf("results = %d, out = %q", len(results), out) + } +} + +func TestQuietSuppressesLines(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + file := filepath.Join(t.TempDir(), "q.txt") + writeFile(t, file, "q") + opts := baseOpts() + opts.Quiet = true + results, out, _, err := run(t, localEP(file), remoteEP(sess, ""), opts) + if err != nil || out != "" || len(results) != 1 { + t.Errorf("quiet: err=%v out=%q results=%d", err, out, len(results)) + } +} + +func TestStdoutDestinationWritesOnlyPayload(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + payload := strings.Repeat("payload-", 1000) + srv.AddObject("bucket-a", "big.bin", []byte(payload), "application/octet-stream") + sess := newTestSession(t, srv, "bucket-a") + + var stdout bytes.Buffer + opts := baseOpts() + opts.Stdout = &stdout + opts.ShowProgress = true // must go to hints, never stdout + results, out, _, err := run(t, remoteEP(sess, "big.bin"), stdioEP(), opts) + if err != nil { + t.Fatal(err) + } + if stdout.String() != payload { + t.Errorf("stdout payload mismatch (len %d)", stdout.Len()) + } + if out != "" { + t.Errorf("no lines may be printed on stdout when the destination is '-': %q", out) + } + if len(results) != 1 || results[0].Destination != "-" { + t.Errorf("results = %+v", results) + } +} + +func TestStdinUpload(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + + opts := baseOpts() + opts.Stdin = strings.NewReader("hello from stdin\n") + results, out, hints, err := run(t, stdioEP(), remoteEP(sess, "in/notes.txt"), opts) + if err != nil { + t.Fatal(err) + } + obj := srv.Object("bucket-a", "in/notes.txt") + if obj == nil || string(obj.Data) != "hello from stdin\n" { + t.Fatalf("object = %+v", obj) + } + if !strings.HasPrefix(obj.ContentType, "text/plain") { + t.Errorf("content type = %q", obj.ContentType) + } + if out != "upload: - to s3://b/in/notes.txt\n" || len(results) != 1 { + t.Errorf("out = %q results=%+v", out, results) + } + if !strings.Contains(hints, "streaming from stdin") { + t.Errorf("expected memory note on stderr, got %q", hints) + } + // A prefix destination is a usage error for stdin. + if _, err := buildPlan(context.Background(), stdioEP(), remoteEP(sess, "in/"), baseOpts()); exitcode.Of(err) != exitcode.Usage { + t.Errorf("stdin to prefix: %v", err) + } +} + +// --expected-size is a hint for part sizing and the progress total only: the +// upload must store the whole stream whether it is shorter or longer than the +// flag (minio would otherwise fail on a short stream and truncate a long one). +func TestStdinUploadExpectedSizeIsOnlyAHint(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + payload := strings.Repeat("0123456789", 4) // 40 bytes + + for _, expected := range []int64{5, 1 << 30} { + opts := baseOpts() + opts.Stdin = strings.NewReader(payload) + opts.ExpectedSize = expected + key := "in/hint.bin" + results, _, hints, err := run(t, stdioEP(), remoteEP(sess, key), opts) + if err != nil { + t.Fatalf("expected-size %d: %v", expected, err) + } + obj := srv.Object("bucket-a", key) + if obj == nil || string(obj.Data) != payload { + t.Fatalf("expected-size %d: stored %q, want the full %d-byte payload", expected, obj.Data, len(payload)) + } + if len(results) != 1 || results[0].Size != int64(len(payload)) { + t.Errorf("expected-size %d: result size = %+v, want %d", expected, results, len(payload)) + } + if strings.Contains(hints, "streaming from stdin") { + t.Errorf("expected-size %d: the part-size note must only appear without --expected-size: %q", expected, hints) + } + } + + // Part sizing: minio's default up to 160 GiB, larger multiples beyond. + if got := stdinPartSizeFor(0); got != stdinPartSize { + t.Errorf("stdinPartSizeFor(0) = %d", got) + } + if got := stdinPartSizeFor(5); got != stdinPartSize { + t.Errorf("stdinPartSizeFor(5) = %d", got) + } + if got := stdinPartSizeFor(stdinMaxPlainSize); got != stdinPartSize { + t.Errorf("stdinPartSizeFor(160 GiB) = %d", got) + } + if got := stdinPartSizeFor(1 << 40); got <= stdinPartSize || got%stdinPartSize != 0 || got*stdinMaxParts < 1<<40 { + t.Errorf("stdinPartSizeFor(1 TiB) = %d, want a multiple of 16 MiB that fits 10000 parts", got) + } +} + +func TestMvRemovesSource(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + t.Chdir(t.TempDir()) // display strings are relative to cwd + file := "m.txt" + writeFile(t, file, "move me") + + opts := baseOpts() + opts.Move = true + _, out, _, err := run(t, localEP(file), remoteEP(sess, "moved/"), opts) + if err != nil { + t.Fatal(err) + } + if out != "move: ./m.txt to s3://b/moved/m.txt\n" { + t.Errorf("out = %q", out) + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Error("local source must be removed after a successful upload") + } + if srv.Object("bucket-a", "moved/m.txt") == nil { + t.Fatal("object missing") + } + + // Remote -> remote move within the bucket (server-side copy + delete). + _, out, _, err = run(t, remoteEP(sess, "moved/m.txt"), remoteEP(sess, "archive/"), opts) + if err != nil { + t.Fatal(err) + } + if out != "move: s3://b/moved/m.txt to s3://b/archive/m.txt\n" { + t.Errorf("out = %q", out) + } + if srv.Object("bucket-a", "moved/m.txt") != nil || srv.Object("bucket-a", "archive/m.txt") == nil { + t.Errorf("keys after move = %v", srv.Keys("bucket-a")) + } + if string(srv.Object("bucket-a", "archive/m.txt").Data) != "move me" { + t.Error("copied data mismatch") + } + + // Same object is refused. + if _, err := buildPlan(context.Background(), remoteEP(sess, "archive/m.txt"), remoteEP(sess, "archive/"), opts); exitcode.Of(err) != exitcode.Usage { + t.Errorf("same object: %v", err) + } +} + +func TestMvKeepsSourceWhenCopyFails(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + file := filepath.Join(t.TempDir(), "keep.txt") + writeFile(t, file, "keep") + + opts := baseOpts() + opts.Move = true + srv.FailNext = &s3test.ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: http.StatusForbidden} + results, _, hints, err := run(t, localEP(file), remoteEP(sess, "x/"), opts) + if exitcode.Of(err) != exitcode.Permission { + t.Errorf("err = %v", err) + } + if _, statErr := os.Stat(file); statErr != nil { + t.Error("source must survive a failed copy") + } + // Single-item plans return the error (printed once by the command) and + // do not duplicate it as a failure line. + if strings.Contains(hints, "move failed: ") || len(results) != 1 || results[0].Error == "" { + t.Errorf("hints = %q results = %+v", hints, results) + } +} + +func TestCrossEndpointCopyRefused(t *testing.T) { + srvA, srvB := s3test.New(), s3test.New() + defer srvA.Close() + defer srvB.Close() + srvA.CreateBucket("bucket-a") + srvB.CreateBucket("bucket-b") + a, b := newTestSession(t, srvA, "bucket-a"), newTestSession(t, srvB, "bucket-b") + _, err := buildPlan(context.Background(), remoteEP(a, "k"), remoteEP(b, "k"), baseOpts()) + if exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "configure export --format rclone") { + t.Errorf("err = %v", err) + } +} + +func TestRecursiveFailuresContinue(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "1.txt"), "1") + writeFile(t, filepath.Join(dir, "2.txt"), "2") + + opts := baseOpts() + opts.Recursive = true + srv.FailNext = &s3test.ErrorResponse{Code: "InternalError", Message: "boom", Status: 500} + results, out, hints, err := run(t, localEP(dir), remoteEP(sess, ""), opts) + if exitcode.Of(err) != exitcode.Generic || !strings.Contains(err.Error(), "1 of 2 transfers failed") { + t.Errorf("err = %v", err) + } + if len(results) != 2 || results[0].Error == "" || results[1].Error != "" { + t.Errorf("results = %+v", results) + } + if !strings.Contains(hints, "upload failed: ") || strings.Count(out, "upload: ") != 1 { + t.Errorf("out=%q hints=%q", out, hints) + } +} + +func TestSyncPlan(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "new.txt"), "new") + writeFile(t, filepath.Join(dir, "same.txt"), "same") + writeFile(t, filepath.Join(dir, "bigger.txt"), "bigger now") + writeFile(t, filepath.Join(dir, "skip.tmp"), "tmp") + old := time.Now().Add(-2 * time.Hour) + for _, n := range []string{"same.txt", "bigger.txt"} { + if err := os.Chtimes(filepath.Join(dir, n), old, old); err != nil { + t.Fatal(err) + } + } + srv.AddObject("bucket-a", "site/same.txt", []byte("same"), "") + srv.AddObject("bucket-a", "site/bigger.txt", []byte("small"), "") + srv.AddObject("bucket-a", "site/stale.txt", []byte("stale"), "") + srv.AddObject("bucket-a", "site/keep.tmp", []byte("keep"), "") + + opts := baseOpts() + opts.Recursive = true + _ = opts.Filters.Add(false, "*.tmp") + plan, err := buildSyncPlan(context.Background(), localEP(dir), remoteEP(sess, "site"), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + var keys []string + for _, it := range plan.Items { + keys = append(keys, it.DstKey) + } + if got := strings.Join(keys, ","); got != "site/bigger.txt,site/new.txt" { + t.Errorf("planned uploads = %s", got) + } + if len(plan.Deletes) != 1 || plan.Deletes[0].Key != "site/stale.txt" { + t.Errorf("planned deletes = %+v (filters must protect keep.tmp)", plan.Deletes) + } + + var out, hints bytes.Buffer + results, err := runTransfers(context.Background(), plan, opts, &out, &hints) + if err != nil { + t.Fatalf("run: %v (%s)", err, hints.String()) + } + if got := strings.Join(srv.Keys("bucket-a"), ","); got != "site/bigger.txt,site/keep.tmp,site/new.txt,site/same.txt" { + t.Errorf("keys after sync = %s", got) + } + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(lines) != 3 || !strings.HasPrefix(lines[0], "upload: ") || lines[2] != "delete: s3://b/site/stale.txt" { + t.Errorf("lines = %q", lines) + } + if len(results) != 3 || results[2].Op != "delete" { + t.Errorf("results = %+v", results) + } + + // Second run: nothing to do. + plan, err = buildSyncPlan(context.Background(), localEP(dir), remoteEP(sess, "site"), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + if len(plan.Items) != 0 || len(plan.Deletes) != 0 { + t.Errorf("second sync must be a no-op: items=%+v deletes=%+v", plan.Items, plan.Deletes) + } + + // Size-only ignores a newer source of the same size. + writeFile(t, filepath.Join(dir, "same.txt"), "SAME") + future := time.Now().Add(time.Hour) + _ = os.Chtimes(filepath.Join(dir, "same.txt"), future, future) + plan, _ = buildSyncPlan(context.Background(), localEP(dir), remoteEP(sess, "site"), opts, syncOptions{SizeOnly: true}) + if len(plan.Items) != 0 { + t.Errorf("--size-only planned %+v", plan.Items) + } + plan, _ = buildSyncPlan(context.Background(), localEP(dir), remoteEP(sess, "site"), opts, syncOptions{}) + if len(plan.Items) != 1 || plan.Items[0].DstKey != "site/same.txt" { + t.Errorf("newer source must be uploaded: %+v", plan.Items) + } +} + +func TestSyncDownloadDryRunAndDelete(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "data/a.txt", []byte("a"), "") + srv.AddObject("bucket-a", "data/sub/b.txt", []byte("b"), "") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + t.Chdir(dir) // display strings are relative to cwd + writeFile(t, filepath.Join(dir, "extra.txt"), "extra") + + opts := baseOpts() + opts.Recursive, opts.DryRun = true, true + plan, err := buildSyncPlan(context.Background(), remoteEP(sess, "data/"), localEP("."), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + var out, hints bytes.Buffer + if _, err := runTransfers(context.Background(), plan, opts, &out, &hints); err != nil { + t.Fatal(err) + } + want := "(dryrun) download: s3://b/data/a.txt to ./a.txt\n" + + "(dryrun) download: s3://b/data/sub/b.txt to " + filepath.Join("sub", "b.txt") + "\n" + + "(dryrun) delete: ./extra.txt\n" + if out.String() != want { + t.Errorf("out = %q, want %q", out.String(), want) + } + if _, err := os.Stat(filepath.Join(dir, "extra.txt")); err != nil { + t.Error("dry-run deleted a local file") + } + + opts.DryRun = false + out.Reset() + if _, err := runTransfers(context.Background(), plan, opts, &out, &hints); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "sub", "b.txt")); string(got) != "b" { + t.Errorf("downloaded b.txt = %q", got) + } + if _, err := os.Stat(filepath.Join(dir, "extra.txt")); !os.IsNotExist(err) { + t.Error("--delete must remove local extras") + } + // Downloaded files carry the object's mtime so the next sync is a no-op. + plan, err = buildSyncPlan(context.Background(), remoteEP(sess, "data/"), localEP("."), opts, syncOptions{Delete: true, ExactTimestamps: true}) + if err != nil { + t.Fatal(err) + } + if len(plan.Items) != 0 || len(plan.Deletes) != 0 { + t.Errorf("re-sync must be a no-op: %+v %+v", plan.Items, plan.Deletes) + } +} + +func TestParseMetadataAndExpires(t *testing.T) { + m, err := parseMetadata([]string{"a=1,b=2", "c=x=y"}) + if err != nil || m["a"] != "1" || m["b"] != "2" || m["c"] != "x=y" { + t.Errorf("metadata = %v err=%v", m, err) + } + if _, err := parseMetadata([]string{"novalue"}); exitcode.Of(err) != exitcode.Usage { + t.Errorf("invalid metadata: %v", err) + } + if ts, err := parseExpires("2026-09-07"); err != nil || ts.Year() != 2026 || ts.Day() != 7 { + t.Errorf("date expires = %v %v", ts, err) + } + if _, err := parseExpires("2026-09-07T10:00:00Z"); err != nil { + t.Errorf("rfc3339 expires: %v", err) + } + if _, err := parseExpires("tomorrow"); exitcode.Of(err) != exitcode.Usage { + t.Errorf("bad expires: %v", err) + } +} + +// displayLocal follows aws (os.path.relpath from cwd): "./" only for a bare +// file name, directory components as-is, absolute paths made relative. +func TestDisplayLocal(t *testing.T) { + t.Chdir(t.TempDir()) + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + sep := string(filepath.Separator) + cases := map[string]string{ + "dump.sql": "." + sep + "dump.sql", + "." + sep + "dump.sql": "." + sep + "dump.sql", + filepath.Join("sub", "b.txt"): filepath.Join("sub", "b.txt"), + filepath.Join("..", "x"): filepath.Join("..", "x"), + filepath.Join(cwd, "dump.sql"): "." + sep + "dump.sql", + filepath.Join(cwd, "sub", "x"): filepath.Join("sub", "x"), + filepath.Join(filepath.Dir(cwd), "o"): filepath.Join("..", "o"), + ".": ".", + } + for in, want := range cases { + if got := displayLocal(in); got != want { + t.Errorf("displayLocal(%q) = %q, want %q", in, got, want) + } + } +} + +func TestCommandsRegisterFlags(t *testing.T) { + cp, mv, sync := NewCpCmd(), NewMvCmd(), NewSyncCmd() + for _, name := range []string{"recursive", "exclude", "include", "content-type", "metadata", "no-overwrite", "expected-size", "quiet", "only-show-errors", "no-progress", "follow-symlinks", "no-follow-symlinks", "project", "storage-class", "acl", "sse", "region"} { + if cp.Flags().Lookup(name) == nil { + t.Errorf("cp lacks --%s", name) + } + if mv.Flags().Lookup(name) == nil { + t.Errorf("mv lacks --%s", name) + } + } + for _, name := range []string{"delete", "size-only", "exact-timestamps", "exclude", "include", "project", "region"} { + if sync.Flags().Lookup(name) == nil { + t.Errorf("sync lacks --%s", name) + } + } + // `aws s3 cp ... --region x` gets the directed explanation (exit 2) on + // every transfer command. + for _, c := range []*cobra.Command{mv, sync} { + if err := c.Flags().Set("region", "us-east-1"); err != nil { + t.Fatal(err) + } + err := c.PreRunE(c, nil) + if exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "--signing-region") { + t.Errorf("%s --region: %v", c.Name(), err) + } + } + if sync.Flags().Lookup("recursive") != nil { + t.Error("sync is always recursive and must not expose --recursive") + } + if !strings.Contains(cp.Long, "2026/09") || !strings.Contains(mv.Long, "2026/09") { + t.Error("destination rules must be documented in the help text") + } + // --storage-class is rejected with the Latitude explanation. + if err := cp.Flags().Set("storage-class", "STANDARD_IA"); err != nil { + t.Fatal(err) + } + err := cp.PreRunE(cp, nil) + if exitcode.Of(err) != exitcode.Usage || !strings.Contains(err.Error(), "lsh s3 create-bucket --storage-class") { + t.Errorf("--storage-class: %v", err) + } +} + +func TestMultipartUploadWithProgress(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + sess := newTestSession(t, srv, "bucket-a") + t.Chdir(t.TempDir()) // display strings are relative to cwd + file := "big.bin" + data := bytes.Repeat([]byte{0xAB}, 17<<20) // above minio's 16 MiB part size + if err := os.WriteFile(file, data, 0o644); err != nil { + t.Fatal(err) + } + + opts := baseOpts() + opts.ShowProgress = true + results, out, hints, err := run(t, localEP(file), remoteEP(sess, "big.bin"), opts) + if err != nil { + t.Fatal(err) + } + obj := srv.Object("bucket-a", "big.bin") + if obj == nil || len(obj.Data) != len(data) || !bytes.Equal(obj.Data, data) { + t.Fatalf("multipart object mismatch (len %d)", len(obj.Data)) + } + if results[0].Size != int64(len(data)) || out != "upload: ./big.bin to s3://b/big.bin\n" { + t.Errorf("results = %+v out = %q", results, out) + } + if !strings.Contains(hints, "%)") || !strings.HasSuffix(hints, "\r\033[K") { + t.Errorf("progress must be drawn and cleared on hints: %q", hints) + } + if srv.HasChecksumHeaders() { + t.Error("multipart upload must not send checksum trailers") + } +} + +// TestSyncDeleteSkippedAfterFailure locks the sync safety rule: when a +// transfer fails, the planned deletions are not applied, so the destination is +// never left without both the object that failed and the one being pruned. +func TestSyncDeleteSkippedAfterFailure(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "data/a.txt", []byte("a"), "") + sess := newTestSession(t, srv, "bucket-a") + dir := t.TempDir() + t.Chdir(dir) // display strings are relative to cwd + writeFile(t, filepath.Join(dir, "extra.txt"), "extra") + // A directory where the download must write its file fails the transfer + // without involving the fake backend. + if err := os.MkdirAll(filepath.Join(dir, "a.txt"), 0o755); err != nil { + t.Fatal(err) + } + + opts := baseOpts() + opts.Recursive = true + plan, err := buildSyncPlan(context.Background(), remoteEP(sess, "data/"), localEP("."), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + if len(plan.Items) != 1 { + t.Fatalf("planned items = %+v, want the one download", plan.Items) + } + var planned []string + for _, d := range plan.Deletes { + planned = append(planned, d.Path) + } + if len(plan.Deletes) == 0 { + t.Fatalf("expected extra.txt to be planned for deletion, got %v", planned) + } + + var out, hints bytes.Buffer + results, err := runTransfers(context.Background(), plan, opts, &out, &hints) + if err == nil { + t.Fatal("expected the failed download to be reported") + } + if _, statErr := os.Stat(filepath.Join(dir, "extra.txt")); statErr != nil { + t.Errorf("a failed sync must not apply its deletions: %v", statErr) + } + if !strings.Contains(hints.String(), "not applied because") { + t.Errorf("hints = %q, want the skipped-deletions warning", hints.String()) + } + // Skipping the deletions must not turn the run into a single-item plan: + // the failure was already reported on hints, so returning the item's own + // error would make the command wrapper print it a second time. + if !strings.Contains(err.Error(), "transfers failed") { + t.Errorf("err = %v, want the summary error (the failure is already on stderr)", err) + } + if n := strings.Count(hints.String(), "download failed:"); n != 1 { + t.Errorf("the failure must be reported once, found %d times:\n%s", n, hints.String()) + } + for _, r := range results { + if r.Op == opDelete { + t.Errorf("delete was executed after a failure: %+v", r) + } + } +} + +// TestSyncDeleteStaysInsideDestination locks the containment rule for +// `sync --delete` on a local destination: the enumeration follows directory +// symlinks, so a deletion candidate reached through one resolves outside the +// tree the user pointed at and must not be removed. +func TestSyncDeleteStaysInsideDestination(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "data/keep.txt", []byte("keep"), "") + sess := newTestSession(t, srv, "bucket-a") + + outside := t.TempDir() + writeFile(t, filepath.Join(outside, "secret.txt"), "do not delete me") + dst := t.TempDir() + if err := os.Symlink(outside, filepath.Join(dst, "link")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + // A genuine destination-only file is the positive control. + writeFile(t, filepath.Join(dst, "stale.txt"), "stale") + + opts := baseOpts() + opts.Recursive = true + if !opts.FollowSymlinks { + t.Fatal("this test needs symlink following enabled") + } + plan, err := buildSyncPlan(context.Background(), remoteEP(sess, "data/"), localEP(dst), opts, syncOptions{Delete: true}) + if err != nil { + t.Fatal(err) + } + var planned []string + for _, d := range plan.Deletes { + planned = append(planned, d.Path) + if strings.Contains(d.Path, "link") || strings.HasPrefix(d.Path, outside) { + t.Errorf("planned a deletion outside the destination: %s", d.Path) + } + } + if len(plan.Deletes) != 1 || !strings.HasSuffix(plan.Deletes[0].Path, "stale.txt") { + t.Fatalf("planned deletes = %v, want only stale.txt", planned) + } + + var out, hints bytes.Buffer + if _, err := runTransfers(context.Background(), plan, opts, &out, &hints); err != nil { + t.Fatalf("run: %v (%s)", err, hints.String()) + } + if _, err := os.Stat(filepath.Join(outside, "secret.txt")); err != nil { + t.Errorf("a file outside the destination was deleted: %v", err) + } + if _, err := os.Stat(filepath.Join(dst, "stale.txt")); !os.IsNotExist(err) { + t.Errorf("the destination-only file should have been deleted: %v", err) + } +} + +func TestDeleteInsideRoot(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sub"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "link")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + resolved, err := filepath.EvalSymlinks(root) + if err != nil { + t.Fatal(err) + } + cases := []struct { + path string + want bool + }{ + {filepath.Join(root, "a.txt"), true}, + {filepath.Join(root, "sub", "a.txt"), true}, + // The link itself lives in the root: removing it only drops the link. + {filepath.Join(root, "link"), true}, + {filepath.Join(root, "link", "a.txt"), false}, + {filepath.Join(outside, "a.txt"), false}, + // A parent that cannot be resolved is never deleted through. + {filepath.Join(root, "missing", "a.txt"), false}, + } + for _, c := range cases { + if got := deleteInsideRoot(resolved, c.path); got != c.want { + t.Errorf("deleteInsideRoot(%s) = %v, want %v", c.path, got, c.want) + } + } +} + +// TestProgressThrottleWithUnknownTotal covers the throttle fix: with total 0 +// (a stdin upload) the "final frame" condition was always false, so every +// transport chunk redrew the stderr line. +func TestProgressThrottleWithUnknownTotal(t *testing.T) { + var buf bytes.Buffer + p := newProgressMeter(&buf, "upload", 0) + for i := 0; i < 50; i++ { + p.add(1024) + } + if n := strings.Count(buf.String(), "\r"); n > 1 { + t.Errorf("redrew the line %d times for 50 chunks; the throttle must apply when the total is unknown:\n%q", n, buf.String()) + } + // A known total still forces the last frame. + buf.Reset() + q := newProgressMeter(&buf, "upload", 2048) + q.add(1024) + q.add(1024) + if !strings.Contains(buf.String(), "upload") { + t.Errorf("a completed transfer must render its final frame, got %q", buf.String()) + } +} + +// TestDryRunToStdoutTargetReportsOnStderr covers `cp s3://b/key - --dryrun`: +// stdout carries the object, so the plan line has to go to stderr instead of +// being swallowed entirely. +func TestDryRunToStdoutTargetReportsOnStderr(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bucket-a") + srv.AddObject("bucket-a", "readme.txt", []byte("hi"), "text/plain") + sess := newTestSession(t, srv, "bucket-a") + + opts := baseOpts() + opts.DryRun = true + _, out, hints, err := run(t, remoteEP(sess, "readme.txt"), stdioEP(), opts) + if err != nil { + t.Fatalf("dry run: %v", err) + } + if out != "" { + t.Errorf("stdout must stay empty for a '-' destination, got %q", out) + } + if !strings.Contains(hints, "(dryrun)") || !strings.Contains(hints, "readme.txt") { + t.Errorf("the plan must be reported on stderr, got %q", hints) + } +} + +// newTestSessionAs builds a session for one bucket with a named credential, so +// a test can hold two sessions with different access keys on one endpoint. +func newTestSessionAs(t *testing.T, srv *s3test.Server, display, backend, keyID string) *Session { + t.Helper() + b := &objectstorage.Bucket{ID: "bkt_" + backend, Name: display, BucketName: backend, Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: "us-east-1"} + cred := objectstorage.NewCredential(keyID, "SK", "profile") + client, err := objectstorage.NewS3Client(b, cred, objectstorage.ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + return &Session{Bucket: b, Cred: cred, Client: client} +} + +// TestCopyStreamsWhenCredentialsDiffer covers the cross-key fallback: a +// server-side CopyObject carries one identity, so when the two buckets resolved +// different keys the destination key cannot read the source. The object is +// streamed through the client instead of failing with an access denial. +func TestCopyStreamsWhenCredentialsDiffer(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("source-bucket") + srv.CreateBucket("target-bucket") + source := srv.AddObject("source-bucket", "data.txt", []byte("payload"), "text/plain") + source.Metadata = map[string]string{"owner": "ops"} + source.Headers = map[string]string{"Cache-Control": "max-age=60"} + + srcSess := newTestSessionAs(t, srv, "logs", "source-bucket", "AK1") + dstSess := newTestSessionAs(t, srv, "backups", "target-bucket", "AK2") + opts := baseOpts() + src, dst := remoteEP(srcSess, "data.txt"), remoteEP(dstSess, "data.txt") + plan, err := buildPlan(context.Background(), src, dst, opts) + if err != nil { + t.Fatal(err) + } + // Refuse the server-side copy only, as the backend would when the + // destination key has no read permission on the source bucket. Planning + // already happened, so the failure lands on CopyObject. + srv.FailNext = &s3test.ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: 403} + + var outBuf, hintsBuf bytes.Buffer + results, err := runTransfers(context.Background(), plan, opts, &outBuf, &hintsBuf) + hints := hintsBuf.String() + if err != nil { + t.Fatalf("copy across credentials: %v (%s)", err, hints) + } + obj := srv.Object("target-bucket", "data.txt") + if obj == nil { + t.Fatalf("the destination object was not written; keys=%v", srv.Keys("target-bucket")) + } + if string(obj.Data) != "payload" { + t.Errorf("destination object = %q, want the streamed payload", obj.Data) + } + // A streamed copy must keep what a server-side copy would have kept. + if obj.ContentType != "text/plain" { + t.Errorf("content type = %q, want the source's", obj.ContentType) + } + if obj.Metadata["owner"] != "ops" { + t.Errorf("user metadata = %v, want the source's owner=ops", obj.Metadata) + } + if obj.Headers["Cache-Control"] != "max-age=60" { + t.Errorf("headers = %v, want the source's Cache-Control", obj.Headers) + } + if !strings.Contains(hints, "different access keys") { + t.Errorf("the fallback must say why it streams:\n%s", hints) + } + if len(results) != 1 || results[0].Error != "" { + t.Errorf("results = %+v, want one successful copy", results) + } +} + +// TestNeedsStreamedCopy keeps the fallback narrow: only an authorization +// failure between two different credentials qualifies. +func TestNeedsStreamedCopy(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("alpha-bucket") + srv.CreateBucket("beta-bucket") + src := remoteEP(newTestSessionAs(t, srv, "src", "alpha-bucket", "AK1"), "k") + dstOther := remoteEP(newTestSessionAs(t, srv, "dst", "beta-bucket", "AK2"), "k") + dstSame := remoteEP(newTestSessionAs(t, srv, "dst", "beta-bucket", "AK1"), "k") + denied := minio.ErrorResponse{Code: "AccessDenied", StatusCode: 403} + missing := minio.ErrorResponse{Code: "NoSuchKey", StatusCode: 404} + + if !needsStreamedCopy(src, dstOther, denied) { + t.Error("access denied across two credentials must stream") + } + if needsStreamedCopy(src, dstSame, denied) { + t.Error("the same credential cannot be fixed by streaming") + } + if needsStreamedCopy(src, dstOther, missing) { + t.Error("a missing object would fail the same way twice") + } + if needsStreamedCopy(localEP("./f"), dstOther, denied) { + t.Error("only remote -> remote copies use CopyObject") + } +} diff --git a/cmd/s3/usage.go b/cmd/s3/usage.go new file mode 100644 index 0000000..f2482c4 --- /dev/null +++ b/cmd/s3/usage.go @@ -0,0 +1,490 @@ +package s3 + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/latitudesh-go-sdk/types" + "github.com/latitudesh/lsh/cmd/lsh" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/utils" + "github.com/spf13/cobra" +) + +const ( + flagSince = "since" + flagUntil = "until" + flagUsageBucket = "bucket" + flagGroupBy = "group-by" + optHumanReadable = "human-readable" + + defaultUsageSince = "30d" + + // usageNameLookupMax caps the per-bucket calls used to map the API's + // numeric storage_id to bucket names; above it the raw ids are shown. + usageNameLookupMax = 50 + // usageLookupConcurrency bounds those parallel calls. + usageLookupConcurrency = metricsConcurrency +) + +// Usage groupings. +const ( + groupByDay = "day" + groupByBucket = "bucket" + groupByTier = "tier" + groupByRegion = "region" +) + +var usageGroupings = []string{groupByDay, groupByBucket, groupByTier, groupByRegion} + +// usageSample is one daily usage record after name resolution. +type usageSample struct { + Date string // YYYY-MM-DD + Project string + StorageID string // the API's numeric storage id ("2028"), not the bkt_ id + Bucket string // display name when known, else StorageID + Named bool // Bucket is a resolved display name + Tier string + Region string + Bytes int64 +} + +// UsageRow is an aggregated usage record (one per group key). +type UsageRow struct { + Date string `json:"date,omitempty"` + Project string `json:"project,omitempty"` + Bucket string `json:"bucket,omitempty"` + StorageID string `json:"storage_id,omitempty"` + Tier string `json:"tier,omitempty"` + Region string `json:"region,omitempty"` + // Bytes is the sum of the daily samples in the group (byte-days when the + // group spans several days). + Bytes int64 `json:"bytes"` + // Days is the number of distinct days in the group; AvgBytes = Bytes/Days. + Days int `json:"days,omitempty"` + AvgBytes int64 `json:"avg_bytes,omitempty"` + Size string `json:"size,omitempty"` // human size of Bytes (-H only) + AvgSize string `json:"avg_size,omitempty"` // human size of AvgBytes (-H only) + + groupBy string + human bool + // rawIDs is set when no bucket name could be resolved (too many buckets + // for the lookup, or every bucket gone): the table then shows the API's + // storage_id under "Storage ID" instead of a "Bucket" column. + rawIDs bool +} + +func (u UsageRow) TableRow() table.Row { + bytesCell := fmt.Sprintf("%d", u.Bytes) + avgCell := fmt.Sprintf("%d", u.AvgBytes) + if u.human { + bytesCell = objectstorage.HumanSize(u.Bytes) + avgCell = objectstorage.HumanSize(u.AvgBytes) + } + row := table.Row{ + "project": {Label: "Project", Value: u.Project}, + "tier": {Label: "Tier", Value: u.Tier}, + "region": {Label: "Region", Value: u.Region}, + "bytes": {Label: "Bytes", Value: bytesCell}, + } + if u.rawIDs { + row["storage_id"] = table.Cell{Label: "Storage ID", Value: u.StorageID} + } else { + row["bucket"] = table.Cell{Label: "Bucket", Value: u.Bucket} + } + if u.groupBy == groupByDay || u.groupBy == "" { + row["date"] = table.Cell{Label: "Date", Value: u.Date} + } else { + row["days"] = table.Cell{Label: "Days", Value: fmt.Sprintf("%d", u.Days)} + row["avg_bytes"] = table.Cell{Label: "Avg/Day", Value: avgCell} + } + return row +} + +// parseUsageWindow resolves --since/--until (utils.ParseTimeRef syntax) with +// the defaults since=30d and until=now, and rejects an inverted window. +func parseUsageWindow(since, until string, now time.Time) (time.Time, time.Time, error) { + since = strings.TrimSpace(since) + if since == "" { + since = defaultUsageSince + } + start, err := utils.ParseTimeRef(since, now) + if err != nil { + return time.Time{}, time.Time{}, exitcode.Errorf(exitcode.Usage, "--%s: %v", flagSince, err) + } + end := now + if u := strings.TrimSpace(until); u != "" && !strings.EqualFold(u, "now") { + end, err = utils.ParseTimeRef(u, now) + if err != nil { + return time.Time{}, time.Time{}, exitcode.Errorf(exitcode.Usage, "--%s: %v", flagUntil, err) + } + } + if start.After(end) { + return time.Time{}, time.Time{}, exitcode.Errorf(exitcode.Usage, "--%s (%s) is after --%s (%s)", flagSince, start.Format("2006-01-02"), flagUntil, end.Format("2006-01-02")) + } + return start, end, nil +} + +// validateGroupBy checks the --group-by value. +func validateGroupBy(v string) (string, error) { + v = strings.ToLower(strings.TrimSpace(v)) + if v == "" { + return groupByDay, nil + } + for _, g := range usageGroupings { + if v == g { + return g, nil + } + } + return "", exitcode.Errorf(exitcode.Usage, "invalid --%s %q: expected one of %s", flagGroupBy, v, strings.Join(usageGroupings, ", ")) +} + +// usageSamples converts API rows for one project. names maps the API's +// numeric storage ids to display names (see usageStorageIDNames); +// fallbackBucket is used when the request was filtered to one bucket and the +// id is unknown. +func usageSamples(data []components.StorageUsageData, project string, names map[string]string, fallbackBucket string) []usageSample { + out := make([]usageSample, 0, len(data)) + for _, d := range data { + a := d.Attributes + if a == nil { + continue + } + // The endpoint has no storage_type filter and reports object, file and + // block rows; summing them would inflate object storage usage. + if t := strings.ToLower(utils.Str(a.StorageType)); t != "" && t != "object" { + continue + } + s := usageSample{Project: project, StorageID: utils.Str(a.StorageID), Tier: utils.Str(a.Tier), Region: utils.Str(a.Region)} + if a.Date != nil { + s.Date = a.Date.String() + } + if a.Bytes != nil { + s.Bytes = *a.Bytes + } + s.Bucket = s.StorageID + if n, ok := names[s.StorageID]; ok && n != "" { + s.Bucket, s.Named = n, true + } else if fallbackBucket != "" { + s.Bucket, s.Named = fallbackBucket, true + } + out = append(out, s) + } + return out +} + +// aggregateUsage sums bytes per group. Every grouping keeps the project in +// the key so --all-projects stays readable; dimensions that are not part of +// the key are shown when uniform inside the group and left empty otherwise. +func aggregateUsage(samples []usageSample, groupBy string, human bool) []UsageRow { + type acc struct { + row UsageRow + days map[string]struct{} + bucket map[string]struct{} + sid map[string]struct{} + tier map[string]struct{} + region map[string]struct{} + } + groups := map[string]*acc{} + var order []string + var named bool + for _, s := range samples { + named = named || s.Named + var key string + switch groupBy { + case groupByBucket: + key = s.Project + "\x00" + s.StorageID + case groupByTier: + key = s.Project + "\x00" + s.Tier + case groupByRegion: + key = s.Project + "\x00" + s.Region + default: + key = s.Project + "\x00" + s.Date + } + g, ok := groups[key] + if !ok { + g = &acc{ + row: UsageRow{Project: s.Project, groupBy: groupBy, human: human}, + days: map[string]struct{}{}, + bucket: map[string]struct{}{}, + sid: map[string]struct{}{}, + tier: map[string]struct{}{}, + region: map[string]struct{}{}, + } + groups[key] = g + order = append(order, key) + } + g.row.Bytes += s.Bytes + g.days[s.Date] = struct{}{} + g.bucket[s.Bucket] = struct{}{} + g.sid[s.StorageID] = struct{}{} + g.tier[s.Tier] = struct{}{} + g.region[s.Region] = struct{}{} + } + + rows := make([]UsageRow, 0, len(groups)) + for _, key := range order { + g := groups[key] + r := g.row + r.Days = len(g.days) + if groupBy == groupByDay { + r.Date = uniform(g.days) + } + r.Bucket = uniform(g.bucket) + r.StorageID = uniform(g.sid) + r.Tier = uniform(g.tier) + r.Region = uniform(g.region) + if r.Days > 0 { + r.AvgBytes = r.Bytes / int64(r.Days) + } + if human { + r.Size = objectstorage.HumanSize(r.Bytes) + r.AvgSize = objectstorage.HumanSize(r.AvgBytes) + } + if groupBy == groupByDay { + // Per-day rows: Days is always 1 and the average equals Bytes. + r.Days, r.AvgBytes, r.AvgSize = 0, 0, "" + } + r.rawIDs = !named + rows = append(rows, r) + } + + sort.SliceStable(rows, func(i, j int) bool { + if groupBy == groupByDay { + if rows[i].Date != rows[j].Date { + return rows[i].Date < rows[j].Date + } + return rows[i].Project < rows[j].Project + } + if rows[i].Bytes != rows[j].Bytes { + return rows[i].Bytes > rows[j].Bytes + } + if rows[i].Project != rows[j].Project { + return rows[i].Project < rows[j].Project + } + return rows[i].Bucket+rows[i].Tier+rows[i].Region < rows[j].Bucket+rows[j].Tier+rows[j].Region + }) + return rows +} + +// uniform returns the single value of set, or "" when it has several. +func uniform(set map[string]struct{}) string { + if len(set) != 1 { + return "" + } + for v := range set { + return v + } + return "" +} + +// usageStorageIDNames maps the numeric storage_id carried by usage rows to +// bucket display names. The API exposes that id only through the usage rows +// themselves, but filter[storage_id] accepts the bkt_ id, so the window is +// queried once per bucket and the storage_id seen in the answer is recorded +// for that bucket. Calls run with bounded concurrency; a bucket whose call +// fails or returns no rows simply stays unmapped (its rows, if any, show the +// raw id). project is used for buckets that do not carry their own. +func usageStorageIDNames(ctx context.Context, api *sdk.Latitudesh, buckets []*objectstorage.Bucket, project string, start, end *types.Date, opts []operations.Option) map[string]string { + names := map[string]string{} + var mu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, usageLookupConcurrency) + for _, b := range buckets { + if b.ID == "" || b.Name == "" { + continue + } + wg.Add(1) + go func(b *objectstorage.Bucket) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + p := b.ProjectRef() + if p == "" { + p = project + } + id := b.ID + resp, err := api.ObjectStorage.GetStorageUsage(ctx, p, &id, start, end, opts...) + if err != nil { + lsh.LogDebugf("usage: could not map storage id of %s: %v", b.Display(), err) + return + } + if resp.StorageUsage == nil { + return + } + mu.Lock() + defer mu.Unlock() + for _, d := range resp.StorageUsage.Data { + if d.Attributes == nil { + continue + } + if sid := utils.Str(d.Attributes.StorageID); sid != "" { + names[sid] = b.Name + } + } + }(b) + } + wg.Wait() + return names +} + +// NewUsageCmd builds `lsh s3 usage`. +func NewUsageCmd() *cobra.Command { + cmd := newCmd(&cobra.Command{ + Use: "usage", + GroupID: groupReports, + Short: "Day-by-day storage history for a project", + Long: `Show the daily storage usage of a project (bytes stored per day), optionally +restricted to one bucket and aggregated per bucket, tier or region. + +--project is required unless --all-projects or --bucket is given. Dates accept +durations (30d, 2w, 24h) or ISO dates (2026-06-01). Aggregated rows report the +sum of the daily samples (bytes), the number of days and the average per day.`, + Example: ` lsh s3 usage --project my-project + lsh s3 usage --project my-project --since 7d --group-by bucket --human-readable + lsh s3 usage --bucket s3://backups --since 2026-08-01 --until 2026-08-31 -o csv + lsh s3 usage --all-projects --group-by region -o json`, + Args: cobra.NoArgs, + RunE: runUsage, + }) + // Registered as optional so the root pre-run never prompts; the command + // enforces the --project / --all-projects / --bucket requirement itself. + addProjectFlag(cmd, true, "project to report (ID or slug); required unless --all-projects or --bucket") + cmd.Flags().Bool(optAllProjects, false, "report every project that has buckets") + cmd.Flags().String(flagSince, defaultUsageSince, "start of the window: duration (30d, 2w) or date (2026-06-01)") + cmd.Flags().String(flagUntil, "", "end of the window: duration or date (default: now)") + cmd.Flags().String(flagUsageBucket, "", "only this bucket (s3://, name or bkt_ id)") + cmd.Flags().String(flagGroupBy, groupByDay, "aggregate rows by day, bucket, tier or region") + cmd.Flags().BoolP(optHumanReadable, "H", false, "show sizes in human-readable units") + return cmd +} + +func runUsage(cmd *cobra.Command, args []string) error { + ctx, cancel := objectstorage.SignalContext(context.Background()) + defer cancel() + + if endpointOverride(cmd) != "" { + return printErr(exitcode.Errorf(exitcode.Usage, "usage comes from the Latitude API; unset --endpoint-url / %s to use this command", objectstorage.EnvEndpointURL)) + } + sinceFlag, _ := cmd.Flags().GetString(flagSince) + untilFlag, _ := cmd.Flags().GetString(flagUntil) + start, end, err := parseUsageWindow(sinceFlag, untilFlag, time.Now()) + if err != nil { + return printErr(err) + } + groupFlag, _ := cmd.Flags().GetString(flagGroupBy) + groupBy, err := validateGroupBy(groupFlag) + if err != nil { + return printErr(err) + } + human, _ := cmd.Flags().GetBool(optHumanReadable) + allProjects, _ := cmd.Flags().GetBool(optAllProjects) + if allProjects && cmd.Flags().Changed(flagProject) { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s and --%s are mutually exclusive", flagProject, optAllProjects)) + } + bucketFlag, _ := cmd.Flags().GetString(flagUsageBucket) + project := projectFlag(cmd) + if allProjects { + project = "" + } + + // Optional bucket filter; it can also supply the project. + var storageID *string + var bucketName string + if bucketFlag != "" { + ref, err := objectstorage.ParseBucketOnly(bucketFlag) + if err != nil { + return printErr(err) + } + b, err := resolveBucket(ctx, cmd, ref.Bucket) + if err != nil { + return printErr(err) + } + if b.ID == "" { + return printErr(exitcode.Errorf(exitcode.Usage, "bucket %s has no API id; usage needs the Latitude API", b.Display())) + } + id := b.ID + storageID = &id + bucketName = b.Name + if project == "" && !allProjects { + project = b.ProjectRef() + } + } + if project == "" && !allProjects { + return printErr(exitcode.Errorf(exitcode.Usage, "--%s is required (pass --%s=, --%s, --%s or set LSH_PROJECT)", flagProject, flagProject, optAllProjects, flagUsageBucket)) + } + + // Bucket list: the buckets whose storage ids get a name and, with + // --all-projects, the set of projects to query. + r := newResolver(cmd) + r.Project = project + list, err := r.ListBuckets(ctx) + if err != nil { + return printErr(err) + } + buckets := make([]*objectstorage.Bucket, 0, len(list)) + projectSet := map[string]struct{}{} + for _, d := range list { + b := objectstorage.BucketFromData(d) + buckets = append(buckets, b) + if p := b.ProjectRef(); p != "" { + projectSet[p] = struct{}{} + } + } + projects := []string{project} + if allProjects { + projects = projects[:0] + for p := range projectSet { + projects = append(projects, p) + } + sort.Strings(projects) + if len(projects) == 0 { + objectstorage.Hintf("no buckets found in any project; nothing to report") + render(nil) + return nil + } + } + + api := apiClient() + opts := []operations.Option{operations.WithRetries(lsh.RetryConfig())} + startDate, endDate := types.NewDate(start), types.NewDate(end) + + // The rows carry a numeric storage_id, not the bkt_ id, so names come from + // one filtered call per bucket. Skipped with --bucket (the filtered rows + // all belong to that bucket) and for very large bucket sets, where the + // table falls back to a Storage ID column. + var names map[string]string + switch { + case storageID != nil: + case len(buckets) > usageNameLookupMax: + objectstorage.Hintf("more than %d buckets; showing storage ids instead of bucket names", usageNameLookupMax) + default: + names = usageStorageIDNames(ctx, api, buckets, project, startDate, endDate, opts) + } + + var samples []usageSample + for _, p := range projects { + resp, err := api.ObjectStorage.GetStorageUsage(ctx, p, storageID, startDate, endDate, opts...) + if err != nil { + return printErr(objectstorage.HumanizeAPI(err, fmt.Sprintf("usage for project %q", p))) + } + if resp.StorageUsage == nil { + continue + } + samples = append(samples, usageSamples(resp.StorageUsage.Data, p, names, bucketName)...) + } + if len(samples) == 0 && isHuman() { + objectstorage.Hintf("no usage recorded between %s and %s", start.Format("2006-01-02"), end.Format("2006-01-02")) + } + render(objectstorage.AsResponseData(aggregateUsage(samples, groupBy, human))) + return nil +} diff --git a/cmd/s3/usage_test.go b/cmd/s3/usage_test.go new file mode 100644 index 0000000..fab716a --- /dev/null +++ b/cmd/s3/usage_test.go @@ -0,0 +1,411 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/types" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/objectstorage" +) + +func TestParseUsageWindow(t *testing.T) { + now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) + + start, end, err := parseUsageWindow("", "", now) + if err != nil { + t.Fatalf("defaults: %v", err) + } + if !start.Equal(now.AddDate(0, 0, -30)) || !end.Equal(now) { + t.Errorf("defaults = %s..%s, want 30d..now", start, end) + } + + start, end, err = parseUsageWindow("7d", "now", now) + if err != nil || !start.Equal(now.Add(-7*24*time.Hour)) || !end.Equal(now) { + t.Errorf("7d..now = %s..%s (%v)", start, end, err) + } + + start, end, err = parseUsageWindow("2026-08-01", "2026-08-31", now) + if err != nil || start.Format("2006-01-02") != "2026-08-01" || end.Format("2006-01-02") != "2026-08-31" { + t.Errorf("absolute dates = %s..%s (%v)", start, end, err) + } + + start, end, err = parseUsageWindow("2w", "1d", now) + if err != nil || !start.Equal(now.Add(-14*24*time.Hour)) || !end.Equal(now.Add(-24*time.Hour)) { + t.Errorf("2w..1d = %s..%s (%v)", start, end, err) + } + + for _, c := range [][2]string{{"yesterday", ""}, {"", "soon"}, {"1d", "7d"}, {"2026-09-10", "2026-09-01"}} { + _, _, err := parseUsageWindow(c[0], c[1], now) + if err == nil { + t.Errorf("parseUsageWindow(%q, %q): expected error", c[0], c[1]) + continue + } + if exitcode.Of(err) != exitcode.Usage { + t.Errorf("parseUsageWindow(%q, %q): exit %d, want %d", c[0], c[1], exitcode.Of(err), exitcode.Usage) + } + } +} + +func TestValidateGroupBy(t *testing.T) { + for in, want := range map[string]string{"": "day", "day": "day", "Bucket": "bucket", " tier ": "tier", "REGION": "region"} { + got, err := validateGroupBy(in) + if err != nil || got != want { + t.Errorf("validateGroupBy(%q) = %q, %v; want %q", in, got, err, want) + } + } + if _, err := validateGroupBy("week"); err == nil || exitcode.Of(err) != exitcode.Usage { + t.Errorf("validateGroupBy(week) = %v", err) + } +} + +func usageDatum(date, storageID, tier, region string, bytes int64) components.StorageUsageData { + d := types.MustDateFromString(date) + return components.StorageUsageData{Attributes: &components.StorageUsageAttributes{ + Date: &d, StorageID: ptrS(storageID), Tier: ptrS(tier), Region: ptrS(region), Bytes: ptrI(bytes), StorageType: ptrS("object"), + }} +} + +// usageFixtureData is the raw API answer for one project. The storage_id the +// API returns is a numeric string ("2028"), not the bkt_ id, which is why the +// names map is keyed by those numbers (see usageStorageIDNames). +func usageFixtureData() []components.StorageUsageData { + return []components.StorageUsageData{ + usageDatum("2026-09-01", "2028", "standard", "DAL", 1000), + usageDatum("2026-09-01", "42", "high", "TYO4", 100), + usageDatum("2026-09-02", "2028", "standard", "DAL", 2000), + usageDatum("2026-09-02", "42", "high", "TYO4", 200), + usageDatum("2026-09-03", "2028", "standard", "DAL", 3000), + usageDatum("2026-09-03", "7", "standard", "DAL", 50), // unknown storage id + {}, // row without attributes is skipped + } +} + +func usageFixture() []usageSample { + names := map[string]string{"2028": "backups", "42": "logs"} + return usageSamples(usageFixtureData(), "my-project", names, "") +} + +func TestUsageSamples(t *testing.T) { + samples := usageFixture() + if len(samples) != 6 { + t.Fatalf("got %d samples, want 6", len(samples)) + } + if samples[0].Bucket != "backups" || samples[1].Bucket != "logs" || samples[5].Bucket != "7" { + t.Errorf("bucket names = %q %q %q", samples[0].Bucket, samples[1].Bucket, samples[5].Bucket) + } + if !samples[0].Named || !samples[1].Named || samples[5].Named { + t.Errorf("named flags = %v %v %v, want true true false", samples[0].Named, samples[1].Named, samples[5].Named) + } + if samples[0].StorageID != "2028" { + t.Errorf("storage id = %q, want the API's numeric id", samples[0].StorageID) + } + if samples[0].Date != "2026-09-01" || samples[0].Project != "my-project" || samples[0].Bytes != 1000 { + t.Errorf("sample = %+v", samples[0]) + } + // The bkt_ id never appears in usage rows, so a map keyed by it resolves nothing. + if byBkt := usageSamples(usageFixtureData(), "p", map[string]string{"bkt_1": "backups"}, ""); byBkt[0].Named { + t.Errorf("bkt_ keyed names must not match numeric storage ids: %+v", byBkt[0]) + } + // With a bucket filter, unknown ids fall back to the filtered bucket's name. + filtered := usageSamples([]components.StorageUsageData{usageDatum("2026-09-01", "99", "standard", "DAL", 1)}, "p", nil, "backups") + if filtered[0].Bucket != "backups" || !filtered[0].Named { + t.Errorf("fallback bucket = %+v", filtered[0]) + } +} + +// usageStorageIDs is the fake API's bkt_ id -> numeric storage_id table. +var usageStorageIDs = map[string]string{"bkt_1": "2028", "bkt_2": "42", "bkt_3": "300"} + +// fakeUsageAPI serves GET /storage/usage. With filter[storage_id]= it +// answers rows carrying that bucket's numeric storage_id (the real API +// behaviour); bkt_err fails with 500 and bkt_empty has no rows. It records +// the peak concurrency and the projects it was asked for. +func fakeUsageAPI(t *testing.T, peak *int32, projects *sync.Map) *httptest.Server { + var inflight int32 + var mu sync.Mutex + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cur := atomic.AddInt32(&inflight, 1) + defer atomic.AddInt32(&inflight, -1) + mu.Lock() + if cur > *peak { + *peak = cur + } + mu.Unlock() + time.Sleep(20 * time.Millisecond) + + if r.URL.Path != "/storage/usage" { + http.Error(w, "unexpected path "+r.URL.Path, http.StatusNotFound) + return + } + q := r.URL.Query() + if q.Get("filter[start_date]") == "" || q.Get("filter[end_date]") == "" { + t.Errorf("lookup must carry the usage window, got %s", r.URL.RawQuery) + } + id := q.Get("filter[storage_id]") + projects.Store(id, q.Get("filter[project]")) + w.Header().Set("Content-Type", "application/vnd.api+json") + switch id { + case "bkt_err": + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"errors":[{"status":"500","title":"Internal Server Error","detail":"boom"}]}`) + return + case "bkt_empty": + fmt.Fprint(w, `{"data":[]}`) + return + } + sid := usageStorageIDs[id] + fmt.Fprintf(w, `{"data":[ + {"id":"u1","type":"storage_usage","attributes":{"date":"2026-09-01","storage_id":%q,"project_id":"1","storage_type":"object","tier":"standard","region":"DAL","bytes":1000}}, + {"id":"u2","type":"storage_usage","attributes":{"date":"2026-09-02","storage_id":%q,"project_id":"1","storage_type":"object","tier":"standard","region":"DAL","bytes":2000}} + ]}`, sid, sid) + })) +} + +func TestUsageStorageIDNames(t *testing.T) { + var peak int32 + var projects sync.Map + srv := fakeUsageAPI(t, &peak, &projects) + defer srv.Close() + api := sdk.New(sdk.WithServerURL(srv.URL), sdk.WithSecurity("test-token")) + + buckets := []*objectstorage.Bucket{ + {ID: "bkt_1", Name: "backups", ProjectSlug: "proj"}, + {ID: "bkt_2", Name: "logs", ProjectSlug: "proj"}, + {ID: "bkt_3", Name: "no-project"}, // falls back to the command's project + {ID: "bkt_err", Name: "broken", ProjectSlug: "proj"}, + {ID: "bkt_empty", Name: "idle", ProjectSlug: "proj"}, + {ID: "", Name: "no-id"}, // skipped: nothing to filter on + } + // Pad with more buckets than the concurrency bound to observe the fan-out. + for i := 10; i < 30; i++ { + buckets = append(buckets, &objectstorage.Bucket{ID: fmt.Sprintf("bkt_%d", i), Name: fmt.Sprintf("b%d", i), ProjectSlug: "proj"}) + } + start, end := types.NewDate(time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)), types.NewDate(time.Date(2026, 9, 7, 0, 0, 0, 0, time.UTC)) + names := usageStorageIDNames(context.Background(), api, buckets, "fallback-proj", start, end, nil) + + want := map[string]string{"2028": "backups", "42": "logs", "300": "no-project"} + for sid, name := range want { + if names[sid] != name { + t.Errorf("names[%q] = %q, want %q", sid, names[sid], name) + } + } + // Padding buckets have no storage id in the fake table and all answer "", + // which must not be recorded; failures and empty answers stay unmapped. + if _, ok := names[""]; ok { + t.Errorf("empty storage ids must not be recorded: %v", names) + } + for sid, name := range names { + if _, ok := want[sid]; !ok { + t.Errorf("unexpected mapping %q -> %q", sid, name) + } + } + if peak > usageLookupConcurrency { + t.Errorf("peak concurrency %d exceeds %d", peak, usageLookupConcurrency) + } + if peak < 2 { + t.Errorf("expected parallel requests, peak was %d", peak) + } + if p, _ := projects.Load("bkt_1"); p != "proj" { + t.Errorf("bkt_1 queried with project %v, want proj", p) + } + if p, _ := projects.Load("bkt_3"); p != "fallback-proj" { + t.Errorf("bucket without project should use the command's project, got %v", p) + } + if _, asked := projects.Load(""); asked { + t.Errorf("a bucket without id must not trigger a lookup") + } + + // The mapped names resolve the rows of the unfiltered call. + samples := usageSamples(usageFixtureData(), "proj", names, "") + if samples[0].Bucket != "backups" || samples[1].Bucket != "logs" || samples[5].Bucket != "7" { + t.Errorf("resolved buckets = %q %q %q", samples[0].Bucket, samples[1].Bucket, samples[5].Bucket) + } +} + +func TestAggregateUsageRawStorageIDs(t *testing.T) { + // No names at all (too many buckets for the lookup): the table shows the + // API's storage_id under "Storage ID" instead of an id-filled Bucket column. + raw := aggregateUsage(usageSamples(usageFixtureData(), "p", nil, ""), groupByBucket, false) + if len(raw) != 3 { + t.Fatalf("got %d rows, want 3", len(raw)) + } + row := raw[0].TableRow() + if _, ok := row["bucket"]; ok { + t.Errorf("unresolved rows must not have a Bucket column: %+v", row) + } + if got := row["storage_id"]; got.Label != "Storage ID" || got.Value != "2028" { + t.Errorf("storage id column = %+v", got) + } + // JSON keeps both fields; bucket falls back to the storage id. + out, _ := json.Marshal(raw[0]) + if !strings.Contains(string(out), `"storage_id":"2028"`) || !strings.Contains(string(out), `"bucket":"2028"`) { + t.Errorf("json = %s", out) + } + + // As soon as one name resolves the Bucket column is back (unknown ids are + // shown as-is inside it). + named := aggregateUsage(usageFixture(), groupByBucket, false) + for _, r := range named { + row := r.TableRow() + if _, ok := row["storage_id"]; ok { + t.Errorf("resolved rows must not have a Storage ID column: %+v", row) + } + if row["bucket"].Label != "Bucket" { + t.Errorf("bucket column = %+v", row["bucket"]) + } + } + if named[2].TableRow()["bucket"].Value != "7" { + t.Errorf("unknown id inside the Bucket column = %q", named[2].TableRow()["bucket"].Value) + } +} + +func TestAggregateUsageByDay(t *testing.T) { + rows := aggregateUsage(usageFixture(), groupByDay, false) + if len(rows) != 3 { + t.Fatalf("got %d rows, want 3: %+v", len(rows), rows) + } + wantBytes := map[string]int64{"2026-09-01": 1100, "2026-09-02": 2200, "2026-09-03": 3050} + for i, r := range rows { + if r.Bytes != wantBytes[r.Date] { + t.Errorf("%s bytes = %d, want %d", r.Date, r.Bytes, wantBytes[r.Date]) + } + if r.Project != "my-project" { + t.Errorf("project = %q", r.Project) + } + // Several buckets per day: bucket is never uniform; tier/region are + // blank when mixed (09-01, 09-02) and kept when shared (09-03). + if r.Bucket != "" || r.StorageID != "" { + t.Errorf("%s: mixed buckets should be blank: %+v", r.Date, r) + } + if r.Date == "2026-09-03" { + if r.Tier != "standard" || r.Region != "DAL" { + t.Errorf("%s: uniform tier/region should be kept: %+v", r.Date, r) + } + } else if r.Tier != "" || r.Region != "" { + t.Errorf("%s: mixed tier/region should be blank: %+v", r.Date, r) + } + if i > 0 && rows[i-1].Date > r.Date { + t.Errorf("rows not sorted by date: %v", rows) + } + if r.Days != 0 || r.AvgBytes != 0 { + t.Errorf("day rows must not carry days/avg: %+v", r) + } + row := r.TableRow() + if _, ok := row["date"]; !ok { + t.Errorf("day grouping should have a date column") + } + if _, ok := row["days"]; ok { + t.Errorf("day grouping should not have a days column") + } + } +} + +func TestAggregateUsageByBucket(t *testing.T) { + rows := aggregateUsage(usageFixture(), groupByBucket, true) + if len(rows) != 3 { + t.Fatalf("got %d rows, want 3: %+v", len(rows), rows) + } + // Sorted by bytes desc. + if rows[0].Bucket != "backups" || rows[0].Bytes != 6000 || rows[0].Days != 3 || rows[0].AvgBytes != 2000 { + t.Errorf("backups row = %+v", rows[0]) + } + if rows[0].Tier != "standard" || rows[0].Region != "DAL" || rows[0].StorageID != "2028" || rows[0].Date != "" { + t.Errorf("uniform dimensions should be kept: %+v", rows[0]) + } + if rows[1].Bucket != "logs" || rows[1].Bytes != 300 || rows[1].Days != 2 || rows[1].AvgBytes != 150 { + t.Errorf("logs row = %+v", rows[1]) + } + if rows[2].Bucket != "7" || rows[2].Bytes != 50 || rows[2].Days != 1 { + t.Errorf("unknown bucket row = %+v", rows[2]) + } + // -H fills the human sizes and the table shows them. + if rows[0].Size != "5.9 KiB" || rows[0].AvgSize != "2.0 KiB" { + t.Errorf("human sizes = %q %q", rows[0].Size, rows[0].AvgSize) + } + row := rows[0].TableRow() + if row["bytes"].Value != "5.9 KiB" || row["avg_bytes"].Value != "2.0 KiB" || row["days"].Value != "3" { + t.Errorf("table row = %+v", row) + } + if _, ok := row["date"]; ok { + t.Errorf("bucket grouping should not have a date column") + } +} + +func TestAggregateUsageByTierAndRegion(t *testing.T) { + tiers := aggregateUsage(usageFixture(), groupByTier, false) + if len(tiers) != 2 || tiers[0].Tier != "standard" || tiers[0].Bytes != 6050 || tiers[1].Tier != "high" || tiers[1].Bytes != 300 { + t.Errorf("tier rows = %+v", tiers) + } + // standard spans two buckets: bucket is blank, region is uniform. + if tiers[0].Bucket != "" || tiers[0].Region != "DAL" { + t.Errorf("standard tier dimensions = %+v", tiers[0]) + } + regions := aggregateUsage(usageFixture(), groupByRegion, false) + if len(regions) != 2 || regions[0].Region != "DAL" || regions[0].Bytes != 6050 || regions[1].Region != "TYO4" || regions[1].Bytes != 300 { + t.Errorf("region rows = %+v", regions) + } + + // Projects stay separate under --all-projects. + mixed := append(usageFixture(), usageSample{Date: "2026-09-01", Project: "other", StorageID: "x", Bucket: "x", Tier: "standard", Region: "DAL", Bytes: 1}) + byTier := aggregateUsage(mixed, groupByTier, false) + if len(byTier) != 3 { + t.Errorf("expected a separate row per project, got %+v", byTier) + } + if empty := aggregateUsage(nil, groupByDay, false); len(empty) != 0 { + t.Errorf("no samples should yield no rows, got %+v", empty) + } +} + +func TestUsageRowJSONShape(t *testing.T) { + rows := aggregateUsage(usageFixture(), groupByBucket, false) + raw, err := json.Marshal(rows[0]) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"project":"my-project"`, `"bucket":"backups"`, `"storage_id":"2028"`, `"tier":"standard"`, `"region":"DAL"`, `"bytes":6000`, `"days":3`, `"avg_bytes":2000`} { + if !strings.Contains(string(raw), want) { + t.Errorf("json %s lacks %s", raw, want) + } + } + if strings.Contains(string(raw), `"size"`) || strings.Contains(string(raw), `"date"`) || strings.Contains(string(raw), "groupBy") { + t.Errorf("unexpected fields in %s", raw) + } +} + +// TestUsageSamplesSkipsOtherStorageTypes covers the aggregation fix: the +// endpoint has no storage_type filter and returns object, file and block rows, +// so summing everything would inflate object storage usage. +func TestUsageSamplesSkipsOtherStorageTypes(t *testing.T) { + str := func(s string) *string { return &s } + i64 := func(n int64) *int64 { return &n } + data := []components.StorageUsageData{ + {Attributes: &components.StorageUsageAttributes{StorageID: str("1"), StorageType: str("object"), Bytes: i64(100)}}, + {Attributes: &components.StorageUsageAttributes{StorageID: str("2"), StorageType: str("file"), Bytes: i64(900)}}, + {Attributes: &components.StorageUsageAttributes{StorageID: str("3"), StorageType: str("block"), Bytes: i64(500)}}, + // An older payload without the field is kept: this endpoint is the + // object storage one, so an unset type is treated as object. + {Attributes: &components.StorageUsageAttributes{StorageID: str("4"), Bytes: i64(7)}}, + } + got := usageSamples(data, "proj_1", nil, "") + if len(got) != 2 { + t.Fatalf("kept %d samples, want the 2 object rows: %+v", len(got), got) + } + var total int64 + for _, s := range got { + total += s.Bytes + } + if total != 107 { + t.Errorf("total = %d bytes, want 107 (file/block rows must not be summed)", total) + } +} diff --git a/cmd/servers/rescue.go b/cmd/servers/rescue.go index 6c624b5..2ea5726 100644 --- a/cmd/servers/rescue.go +++ b/cmd/servers/rescue.go @@ -5,7 +5,6 @@ import ( "fmt" "os" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" "github.com/latitudesh/lsh/cmd/lsh" "github.com/latitudesh/lsh/internal/utils" @@ -57,8 +56,8 @@ func runRescueMode(cmd *cobra.Command, args []string) error { // dashboard. stderr keeps stdout clean for structured output. fmt.Fprintln(os.Stderr, "Note: rescue login credentials are available on the server's page in the dashboard.") - want := []components.ServerDataStatus{components.ServerDataStatusRescueMode} - fail := []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} + want := []wait.ServerStatus{wait.ServerStatusRescueMode} + fail := []wait.ServerStatus{wait.ServerStatusFailedDeployment} // Idempotent wait: a server already in the target state is already done — // requiring a transition here would hang until timeout. return waitForServerState(cmd, serverID, want, fail, false) @@ -104,11 +103,11 @@ func runExitRescueMode(cmd *cobra.Command, args []string) error { } // Leaving rescue mode reboots into the installed OS; it settles on/off. - want := []components.ServerDataStatus{ - components.ServerDataStatusOn, - components.ServerDataStatusOff, + want := []wait.ServerStatus{ + wait.ServerStatusOn, + wait.ServerStatusOff, } - fail := []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} + fail := []wait.ServerStatus{wait.ServerStatusFailedDeployment} // Idempotent wait: a server already in the target state is already done — // requiring a transition here would hang until timeout. return waitForServerState(cmd, serverID, want, fail, false) diff --git a/cmd/servers/servers_test.go b/cmd/servers/servers_test.go index 5c1547e..5350f97 100644 --- a/cmd/servers/servers_test.go +++ b/cmd/servers/servers_test.go @@ -5,6 +5,7 @@ import ( "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/wait" ) func strp(s string) *string { return &s } @@ -48,17 +49,17 @@ func TestActionsCmdRequiresArg(t *testing.T) { // TestPowerActionTargets verifies each action waits for the correct terminal // power state. func TestPowerActionTargets(t *testing.T) { - cases := map[string]components.ServerDataStatus{ - "power_on": components.ServerDataStatusOn, - "reboot": components.ServerDataStatusOn, - "power_off": components.ServerDataStatusOff, + cases := map[string]wait.ServerStatus{ + "power_on": wait.ServerStatusOn, + "reboot": wait.ServerStatusOn, + "power_off": wait.ServerStatusOff, } for action, wantState := range cases { want, fail := powerActionTargets(action) if len(want) != 1 || want[0] != wantState { t.Errorf("powerActionTargets(%q) want = %v, expected [%v]", action, want, wantState) } - if len(fail) != 1 || fail[0] != components.ServerDataStatusFailedDeployment { + if len(fail) != 1 || fail[0] != wait.ServerStatusFailedDeployment { t.Errorf("powerActionTargets(%q) fail = %v, expected [failed_deployment]", action, fail) } } diff --git a/cmd/servers/wait.go b/cmd/servers/wait.go index 6094ee6..2697afa 100644 --- a/cmd/servers/wait.go +++ b/cmd/servers/wait.go @@ -6,7 +6,6 @@ import ( "fmt" "os" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" "github.com/latitudesh/lsh/cmd/lsh" "github.com/latitudesh/lsh/internal/wait" @@ -21,7 +20,7 @@ import ( // requireTransition guards operations that act on a server which may already // sit in a target state (e.g. power_on on an already-on server) so the wait // does not return before the operation has actually taken effect. -func waitForServerState(cmd *cobra.Command, serverID string, want, fail []components.ServerDataStatus, requireTransition bool) error { +func waitForServerState(cmd *cobra.Command, serverID string, want, fail []wait.ServerStatus, requireTransition bool) error { o := wait.OptionsFrom(cmd) if !o.Enabled { if cmd.Flags().Changed("timeout") { @@ -58,13 +57,13 @@ func waitForServerState(cmd *cobra.Command, serverID string, want, fail []compon // powerActionTargets maps a power action to the server states that satisfy the // wait (want) and the states that abort it (fail). -func powerActionTargets(action string) (want, fail []components.ServerDataStatus) { - fail = []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} +func powerActionTargets(action string) (want, fail []wait.ServerStatus) { + fail = []wait.ServerStatus{wait.ServerStatusFailedDeployment} switch action { case "power_on", "reboot": - return []components.ServerDataStatus{components.ServerDataStatusOn}, fail + return []wait.ServerStatus{wait.ServerStatusOn}, fail case "power_off": - return []components.ServerDataStatus{components.ServerDataStatusOff}, fail + return []wait.ServerStatus{wait.ServerStatusOff}, fail default: return nil, fail } diff --git a/cmd/sshkeys/create.go b/cmd/sshkeys/create.go index 9cb45b9..c0eab49 100644 --- a/cmd/sshkeys/create.go +++ b/cmd/sshkeys/create.go @@ -49,8 +49,8 @@ func (o *CreateSSHKeyOperation) run(cmd *cobra.Command, args []string) error { Data: operations.PostSSHKeySSHKeysData{ Type: operations.PostSSHKeySSHKeysTypeSSHKeys, Attributes: &operations.PostSSHKeySSHKeysAttributes{ - Name: &name, - PublicKey: &publicKey, + Name: name, + PublicKey: publicKey, }, }, } diff --git a/cmd/sshkeys/sshkeys_test.go b/cmd/sshkeys/sshkeys_test.go index aa6f0b8..b9c5b16 100644 --- a/cmd/sshkeys/sshkeys_test.go +++ b/cmd/sshkeys/sshkeys_test.go @@ -106,15 +106,15 @@ func TestCreateBuildsRequestBody(t *testing.T) { Data: operations.PostSSHKeySSHKeysData{ Type: operations.PostSSHKeySSHKeysTypeSSHKeys, Attributes: &operations.PostSSHKeySSHKeysAttributes{ - Name: &name, - PublicKey: &publicKey, + Name: name, + PublicKey: publicKey, }, }, } if request.Data.Type != "ssh_keys" { t.Errorf("type = %q, want ssh_keys", request.Data.Type) } - if *request.Data.Attributes.Name != name || *request.Data.Attributes.PublicKey != publicKey { + if request.Data.Attributes.Name != name || request.Data.Attributes.PublicKey != publicKey { t.Error("attributes not wired through") } } diff --git a/cmd/sshkeys/update.go b/cmd/sshkeys/update.go index 7daed4f..2cc1f53 100644 --- a/cmd/sshkeys/update.go +++ b/cmd/sshkeys/update.go @@ -59,7 +59,7 @@ func (o *UpdateSSHKeyOperation) run(cmd *cobra.Command, args []string) error { request := operations.PutSSHKeySSHKeysRequestBody{ Data: operations.PutSSHKeySSHKeysData{ - ID: &sshKeyID, + ID: sshKeyID, Type: operations.PutSSHKeySSHKeysTypeSSHKeys, Attributes: attributes, }, diff --git a/cmd/storage_filesystems/update.go b/cmd/storage_filesystems/update.go index 1c5cc57..4df8331 100644 --- a/cmd/storage_filesystems/update.go +++ b/cmd/storage_filesystems/update.go @@ -33,7 +33,7 @@ type UpdateFilesystemOperation struct{} func buildUpdateRequest(sizeSet bool, size int64) operations.PatchStorageFilesystemsFilesystemStorageRequestBody { attributes := operations.PatchStorageFilesystemsFilesystemStorageAttributes{} if sizeSet { - attributes.SizeInGb = &size + attributes.SizeInGb = size } return operations.PatchStorageFilesystemsFilesystemStorageRequestBody{ diff --git a/cmd/storage_filesystems/update_test.go b/cmd/storage_filesystems/update_test.go index 201cd70..9259456 100644 --- a/cmd/storage_filesystems/update_test.go +++ b/cmd/storage_filesystems/update_test.go @@ -11,15 +11,15 @@ func TestBuildUpdateRequestSizeSet(t *testing.T) { if req.Data.Type != operations.PatchStorageFilesystemsFilesystemStorageTypeFilesystems { t.Errorf("Type = %q, want filesystems", req.Data.Type) } - if req.Data.Attributes.SizeInGb == nil || *req.Data.Attributes.SizeInGb != 3000 { + if req.Data.Attributes.SizeInGb != 3000 { t.Errorf("SizeInGb = %v, want 3000", req.Data.Attributes.SizeInGb) } } func TestBuildUpdateRequestSizeOmitted(t *testing.T) { req := buildUpdateRequest(false, 0) - if req.Data.Attributes.SizeInGb != nil { - t.Errorf("SizeInGb = %v, want nil when --size not set", req.Data.Attributes.SizeInGb) + if req.Data.Attributes.SizeInGb != 0 { + t.Errorf("SizeInGb = %v, want 0 when --size not set", req.Data.Attributes.SizeInGb) } } diff --git a/cmd/storage_objects/create.go b/cmd/storage_objects/create.go deleted file mode 100644 index 4d93ad2..0000000 --- a/cmd/storage_objects/create.go +++ /dev/null @@ -1,134 +0,0 @@ -package storage_objects - -import ( - "context" - "fmt" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" - "github.com/latitudesh/lsh/cmd/lsh" - "github.com/latitudesh/lsh/internal/tui" - "github.com/latitudesh/lsh/internal/utils" - cobra "github.com/spf13/cobra" -) - -func NewCreateCmd() *cobra.Command { - o := CreateBucketOperation{} - cmd := &cobra.Command{ - Long: "Create an object storage bucket in a project.\n\n" + - "Buckets are S3-compatible. Optional flags enable versioning, object\n" + - "lock (WORM) and a higher-performance storage class where available.\n", - RunE: o.run, - Short: "Create a bucket", - Example: ` lsh storage-objects create --project my-project --name my-bucket --region SAO2 - lsh storage-objects create --project my-project --name logs --region DAL --storage-class high_performance --versioning`, - Use: "create", - } - - cmd.Flags().String("project", "", "Project ID or slug to create the bucket in") - cmd.Flags().String("name", "", "Bucket name (no special characters or spaces)") - cmd.Flags().String("region", "", "Site slug representing the region (e.g. DAL, SAO2)") - cmd.Flags().String("storage-class", "", "Storage tier: standard or high_performance") - cmd.Flags().Bool("versioning", false, "Enable S3 object versioning") - cmd.Flags().Bool("locking", false, "Enable S3 Object Lock (WORM); implies versioning") - - return cmd -} - -type CreateBucketOperation struct{} - -// buildCreateRequest turns parsed flags into the SDK request body. Project, -// name and region are required. Optional toggles are only set when supplied so -// the API defaults apply otherwise. Split out for unit testing. -func buildCreateRequest(project, name, region, storageClass string, versioningSet, versioning, lockingSet, locking bool) (operations.PostStorageBucketsRequestBody, error) { - if project == "" { - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("--project is required") - } - if name == "" { - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("--name is required") - } - if region == "" { - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("--region is required") - } - - attributes := operations.PostStorageBucketsAttributes{ - Project: project, - Name: name, - Region: region, - } - - if storageClass != "" { - switch storageClass { - case string(operations.StorageClassStandard), string(operations.StorageClassHighPerformance): - sc := operations.StorageClass(storageClass) - attributes.StorageClass = &sc - default: - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("invalid --storage-class %q (expected standard or high_performance)", storageClass) - } - } - if versioningSet { - attributes.Versioning = &versioning - } - if lockingSet { - // The API requires versioning on locked buckets; enforce the documented - // implication client-side instead of surfacing an opaque server error. - if locking && versioningSet && !versioning { - return operations.PostStorageBucketsRequestBody{}, fmt.Errorf("--locking requires versioning; drop --versioning=false or enable it") - } - if locking && !versioningSet { - enabled := true - attributes.Versioning = &enabled - } - attributes.Locking = &locking - } - - return operations.PostStorageBucketsRequestBody{ - Data: operations.PostStorageBucketsData{ - Type: operations.PostStorageBucketsTypeObjects, - Attributes: attributes, - }, - }, nil -} - -func (o *CreateBucketOperation) run(cmd *cobra.Command, args []string) error { - project, _ := cmd.Flags().GetString("project") - name, _ := cmd.Flags().GetString("name") - region, _ := cmd.Flags().GetString("region") - storageClass, _ := cmd.Flags().GetString("storage-class") - versioning, _ := cmd.Flags().GetBool("versioning") - locking, _ := cmd.Flags().GetBool("locking") - - request, err := buildCreateRequest( - project, name, region, storageClass, - cmd.Flags().Changed("versioning"), versioning, - cmd.Flags().Changed("locking"), locking, - ) - if err != nil { - utils.PrintError(err) - return err - } - - if lsh.DryRun { - lsh.LogDebugf("dry-run flag specified. Skip sending request.") - return nil - } - - client := lsh.NewClient() - ctx := context.Background() - - response, err := client.ObjectStorage.PostStorageBuckets(ctx, request, operations.WithRetries(lsh.RetryConfig())) - if err != nil { - utils.PrintError(err) - return err - } - - if response.Object != nil && response.Object.Data != nil { - fmt.Println(tui.SuccessStyle.Render("✓ Bucket created successfully!")) - - if !lsh.Debug { - bucket := Bucket{ObjectStorageData: *response.Object.Data} - utils.RenderStatic(bucket.GetData()) - } - } - - return nil -} diff --git a/cmd/storage_objects/create_test.go b/cmd/storage_objects/create_test.go deleted file mode 100644 index 941931d..0000000 --- a/cmd/storage_objects/create_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package storage_objects - -import ( - "testing" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" -) - -func TestBuildCreateRequest(t *testing.T) { - req, err := buildCreateRequest("my-project", "my-bucket", "SAO2", "high_performance", true, true, false, false) - if err != nil { - t.Fatalf("buildCreateRequest returned error: %v", err) - } - - if req.Data.Type != operations.PostStorageBucketsTypeObjects { - t.Errorf("Type = %q, want objects", req.Data.Type) - } - attr := req.Data.Attributes - if attr.Project != "my-project" || attr.Name != "my-bucket" || attr.Region != "SAO2" { - t.Errorf("unexpected core attributes: %+v", attr) - } - if attr.StorageClass == nil || *attr.StorageClass != operations.StorageClassHighPerformance { - t.Errorf("StorageClass = %v, want high_performance", attr.StorageClass) - } - if attr.Versioning == nil || !*attr.Versioning { - t.Errorf("Versioning = %v, want true", attr.Versioning) - } - if attr.Locking != nil { - t.Errorf("Locking = %v, want nil when --locking not set", attr.Locking) - } -} - -func TestBuildCreateRequestDefaults(t *testing.T) { - req, err := buildCreateRequest("p", "b", "DAL", "", false, false, false, false) - if err != nil { - t.Fatalf("buildCreateRequest returned error: %v", err) - } - attr := req.Data.Attributes - if attr.StorageClass != nil { - t.Errorf("StorageClass = %v, want nil by default", attr.StorageClass) - } - if attr.Versioning != nil || attr.Locking != nil { - t.Errorf("optional toggles should be nil by default, got versioning=%v locking=%v", attr.Versioning, attr.Locking) - } -} - -func TestBuildCreateRequestValidation(t *testing.T) { - cases := []struct { - name string - project, bucket, region, sc string - }{ - {"missing project", "", "b", "DAL", ""}, - {"missing name", "p", "", "DAL", ""}, - {"missing region", "p", "b", "", ""}, - {"invalid storage-class", "p", "b", "DAL", "turbo"}, - } - for _, c := range cases { - if _, err := buildCreateRequest(c.project, c.bucket, c.region, c.sc, false, false, false, false); err == nil { - t.Errorf("%s: expected error, got nil", c.name) - } - } -} - -func TestCreateCmdFlags(t *testing.T) { - cmd := NewCreateCmd() - if err := cmd.Flags().Parse([]string{ - "--project", "p1", - "--name", "b1", - "--region", "SAO2", - "--storage-class", "standard", - "--versioning", - }); err != nil { - t.Fatalf("flag parse error: %v", err) - } - if !cmd.Flags().Changed("versioning") { - t.Error("expected --versioning to be marked changed") - } - if cmd.Flags().Changed("locking") { - t.Error("expected --locking to be unset") - } -} diff --git a/cmd/storage_objects/delete.go b/cmd/storage_objects/delete.go deleted file mode 100644 index 557b1c7..0000000 --- a/cmd/storage_objects/delete.go +++ /dev/null @@ -1,62 +0,0 @@ -package storage_objects - -import ( - "context" - "fmt" - "net/http" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" - "github.com/latitudesh/lsh/cmd/lsh" - "github.com/latitudesh/lsh/internal/utils" - cobra "github.com/spf13/cobra" -) - -func NewDeleteCmd() *cobra.Command { - op := DeleteBucketOperation{} - cmd := &cobra.Command{ - Long: "Delete an object storage bucket by ID.\n", - RunE: op.run, - Short: "Delete a bucket", - Example: ` lsh storage-objects delete bucket_xxxxxxxx`, - Use: "delete ", - Aliases: []string{"rm"}, - Args: cobra.ExactArgs(1), - } - - return cmd -} - -type DeleteBucketOperation struct{} - -func (o *DeleteBucketOperation) run(cmd *cobra.Command, args []string) error { - id := args[0] - - if lsh.DryRun { - lsh.LogDebugf("dry-run flag specified. Skip sending request.") - return nil - } - - client := lsh.NewClient() - ctx := context.Background() - - resp, err := client.ObjectStorage.DeleteStorageBuckets(ctx, id, operations.WithRetries(lsh.RetryConfig())) - if err != nil { - utils.PrintError(err) - return err - } - - if !lsh.Debug { - // The API answers deletes with 200 or 204 depending on the path. - status := 0 - if resp.HTTPMeta.Response != nil { - status = resp.HTTPMeta.Response.StatusCode - } - if status == http.StatusOK || status == http.StatusNoContent { - fmt.Printf("\nBucket deleted successfully!\n") - } else { - fmt.Printf("Warning: Unexpected status code: %d\n", status) - } - } - - return nil -} diff --git a/cmd/storage_objects/get.go b/cmd/storage_objects/get.go deleted file mode 100644 index a7780b9..0000000 --- a/cmd/storage_objects/get.go +++ /dev/null @@ -1,53 +0,0 @@ -package storage_objects - -import ( - "context" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" - "github.com/latitudesh/lsh/cmd/lsh" - "github.com/latitudesh/lsh/internal/utils" - cobra "github.com/spf13/cobra" -) - -func NewGetCmd() *cobra.Command { - op := GetBucketOperation{} - cmd := &cobra.Command{ - Long: "Retrieve a single object storage bucket by ID.\n", - RunE: op.run, - Short: "Get a bucket", - Example: ` lsh storage-objects get bucket_xxxxxxxx`, - Use: "get ", - Args: cobra.ExactArgs(1), - } - - return cmd -} - -type GetBucketOperation struct{} - -func (o *GetBucketOperation) run(cmd *cobra.Command, args []string) error { - id := args[0] - - if lsh.DryRun { - lsh.LogDebugf("dry-run flag specified. Skip sending request.") - return nil - } - - client := lsh.NewClient() - ctx := context.Background() - - response, err := client.ObjectStorage.GetStorageBucket(ctx, id, operations.WithRetries(lsh.RetryConfig())) - if err != nil { - utils.PrintError(err) - return err - } - - if response.Object != nil && response.Object.Data != nil { - if !lsh.Debug { - bucket := Bucket{ObjectStorageData: *response.Object.Data} - utils.Render(bucket.GetData()) - } - } - - return nil -} diff --git a/cmd/storage_objects/list.go b/cmd/storage_objects/list.go deleted file mode 100644 index ee5d5be..0000000 --- a/cmd/storage_objects/list.go +++ /dev/null @@ -1,66 +0,0 @@ -package storage_objects - -import ( - "context" - - "github.com/latitudesh/latitudesh-go-sdk/models/operations" - "github.com/latitudesh/lsh/cli" - "github.com/latitudesh/lsh/cmd/lsh" - "github.com/latitudesh/lsh/internal/utils" - cobra "github.com/spf13/cobra" -) - -func NewListCmd() *cobra.Command { - op := ListBucketsOperation{} - cmd := &cobra.Command{ - Long: "List object storage buckets in the project.\n", - RunE: op.run, - Short: "List buckets", - Example: ` lsh storage-objects list --project my-project - lsh storage-objects list`, - Use: "list", - Aliases: []string{"ls"}, - Annotations: map[string]string{cli.ProjectOptionalAnnotation: "true"}, - } - - cmd.Flags().String("project", "", "Project ID or slug to filter by") - - return cmd -} - -type ListBucketsOperation struct{} - -func (o *ListBucketsOperation) run(cmd *cobra.Command, args []string) error { - var filterProject *string - if cmd.Flags().Changed("project") { - value, _ := cmd.Flags().GetString("project") - filterProject = &value - } - - if lsh.DryRun { - lsh.LogDebugf("dry-run flag specified. Skip sending request.") - return nil - } - - client := lsh.NewClient() - ctx := context.Background() - - response, err := client.ObjectStorage.GetStorageBuckets(ctx, filterProject, operations.WithRetries(lsh.RetryConfig())) - if err != nil { - utils.PrintError(err) - return err - } - - buckets := Buckets{} - if response.ObjectStorages != nil { - for i := range response.ObjectStorages.Data { - buckets.Data = append(buckets.Data, &Bucket{ObjectStorageData: response.ObjectStorages.Data[i]}) - } - } - - if !lsh.Debug { - utils.Render(buckets.GetData()) - } - - return nil -} diff --git a/cmd/storage_objects/render_test.go b/cmd/storage_objects/render_test.go deleted file mode 100644 index 4f1b539..0000000 --- a/cmd/storage_objects/render_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package storage_objects - -import ( - "encoding/json" - "testing" - - "github.com/latitudesh/latitudesh-go-sdk/models/components" -) - -const bucketsFixture = `{ - "data": [ - { - "id": "bucket_abc123", - "type": "object_storages", - "attributes": { - "name": "my-bucket", - "bucket_name": "my-bucket-xyz", - "storage_class": "high_performance", - "endpoint": "https://s3.example.com", - "created_at": "2026-06-01T12:00:00Z", - "region": {"id": "SAO2", "city": "Sao Paulo", "country": "Brazil"} - } - } - ] -}` - -// TestBucketsPayloadDecoding pins the SDK's typed list envelope to the API -// shape and asserts the rendered row for the first bucket. -func TestBucketsPayloadDecoding(t *testing.T) { - var payload components.ObjectStorages - if err := json.Unmarshal([]byte(bucketsFixture), &payload); err != nil { - t.Fatalf("could not unmarshal buckets fixture: %v", err) - } - if len(payload.Data) != 1 { - t.Fatalf("expected 1 bucket, got %d", len(payload.Data)) - } - - bucket := Bucket{ObjectStorageData: payload.Data[0]} - row := bucket.TableRow() - - expectations := map[string]string{ - "id": "bucket_abc123", - "name": "my-bucket", - "bucket_name": "my-bucket-xyz", - "region": "SAO2", - "storage_class": "high_performance", - "endpoint": "https://s3.example.com", - "created_at": "2026-06-01T12:00:00Z", - } - for key, want := range expectations { - if got := row[key].Value; got != want { - t.Errorf("row[%q] = %q, want %q", key, got, want) - } - } - // Live gets return region without an id ({city, country}); the cell must - // fall back to the city so the column is not blank. - city := "Miami" - noID := Bucket{ObjectStorageData: components.ObjectStorageData{ - Attributes: &components.ObjectStorageDataAttributes{ - Region: &components.ObjectStorageDataRegion{City: &city}, - }, - }} - if got := noID.TableRow()["region"].Value; got != "Miami" { - t.Errorf("region fallback = %q, want Miami", got) - } -} - -func TestBucketTableRowEmpty(t *testing.T) { - bucket := Bucket{ObjectStorageData: components.ObjectStorageData{}} - row := bucket.TableRow() - if row["name"].Value != "" { - t.Errorf("expected blank cells for empty bucket, got %+v", row) - } -} - -func TestGetAndDeleteArgs(t *testing.T) { - if err := NewGetCmd().Args(NewGetCmd(), []string{}); err == nil { - t.Error("get: expected error with no args") - } - if err := NewGetCmd().Args(NewGetCmd(), []string{"id1"}); err != nil { - t.Errorf("get: unexpected error with one arg: %v", err) - } - if err := NewDeleteCmd().Args(NewDeleteCmd(), []string{"a", "b"}); err == nil { - t.Error("delete: expected error with two args") - } -} diff --git a/cmd/storage_objects/storage_objects.go b/cmd/storage_objects/storage_objects.go deleted file mode 100644 index dd3554b..0000000 --- a/cmd/storage_objects/storage_objects.go +++ /dev/null @@ -1,96 +0,0 @@ -package storage_objects - -import ( - "github.com/latitudesh/latitudesh-go-sdk/models/components" - "github.com/latitudesh/lsh/internal/output/table" - "github.com/latitudesh/lsh/internal/renderer" -) - -// Buckets is the renderable collection of object storage buckets. -type Buckets struct { - Data []*Bucket -} - -func (m *Buckets) GetData() []renderer.ResponseData { - var data []renderer.ResponseData - for _, v := range m.Data { - data = append(data, v) - } - return data -} - -// Bucket wraps the SDK ObjectStorageData so it can be rendered by the shared -// renderer (table / -o json,yaml / --query). -type Bucket struct { - components.ObjectStorageData -} - -func (m *Bucket) GetData() []renderer.ResponseData { - return []renderer.ResponseData{m} -} - -func (m *Bucket) TableRow() table.Row { - b := m.ObjectStorageData - - getStr := func(s *string) string { - if s != nil { - return *s - } - return "" - } - - var name, bucketName, region, storageClass, endpoint, createdAt string - if attr := b.Attributes; attr != nil { - name = getStr(attr.Name) - bucketName = getStr(attr.BucketName) - endpoint = getStr(attr.Endpoint) - if attr.StorageClass != nil { - storageClass = string(*attr.StorageClass) - } - // The API sends region as {city, country} on get (id is not populated - // and list omits the attribute entirely); prefer the most specific field. - if attr.Region != nil { - region = getStr(attr.Region.ID) - if region == "" { - region = getStr(attr.Region.City) - } - if region == "" { - region = getStr(attr.Region.Country) - } - } - if attr.CreatedAt != nil { - createdAt = attr.CreatedAt.Format("2006-01-02T15:04:05Z07:00") - } - } - - return table.Row{ - "id": table.Cell{ - Label: "ID", - Value: table.String(getStr(b.ID)), - }, - "name": table.Cell{ - Label: "Name", - Value: table.String(name), - }, - "bucket_name": table.Cell{ - Label: "Bucket Name", - Value: table.String(bucketName), - }, - "region": table.Cell{ - Label: "Region", - Value: table.String(region), - }, - "storage_class": table.Cell{ - Label: "Storage Class", - Value: table.String(storageClass), - }, - "endpoint": table.Cell{ - Label: "Endpoint", - Value: table.String(endpoint), - }, - "created_at": table.Cell{ - Label: "Created At", - Value: table.String(createdAt), - }, - } -} diff --git a/cmd/tags/create.go b/cmd/tags/create.go index bdf5778..3b00fb8 100644 --- a/cmd/tags/create.go +++ b/cmd/tags/create.go @@ -88,9 +88,9 @@ func (o *CreateTagOperation) run(cmd *cobra.Command, args []string) error { Data: &operations.CreateTagTagsData{ Type: &createTagType, Attributes: &operations.CreateTagTagsAttributes{ - Name: &name, + Name: name, Description: &description, - Color: &color, + Color: color, }, }, } diff --git a/cmd/virtualmachines/get.go b/cmd/virtualmachines/get.go index 4b17c48..9143542 100644 --- a/cmd/virtualmachines/get.go +++ b/cmd/virtualmachines/get.go @@ -36,7 +36,7 @@ func (o *GetVirtualMachineOperation) run(cmd *cobra.Command, args []string) erro client := lsh.NewClient() ctx := context.Background() - response, err := client.VirtualMachines.Get(ctx, id, operations.WithRetries(lsh.RetryConfig())) + response, err := client.VirtualMachines.Get(ctx, id, nil, operations.WithRetries(lsh.RetryConfig())) if err != nil { utils.PrintError(err) return err diff --git a/cmd/virtualmachines/list.go b/cmd/virtualmachines/list.go index 11f0f6b..59ead7d 100644 --- a/cmd/virtualmachines/list.go +++ b/cmd/virtualmachines/list.go @@ -53,7 +53,7 @@ func (o *ListVirtualMachineOperation) run(cmd *cobra.Command, args []string) err client := lsh.NewClient() ctx := context.Background() - response, err := client.VirtualMachines.List(ctx, filterProject, filterTags, nil, operations.WithRetries(lsh.RetryConfig())) + response, err := client.VirtualMachines.List(ctx, operations.IndexVirtualMachineRequest{FilterProject: filterProject, FilterTags: filterTags}, operations.WithRetries(lsh.RetryConfig())) if err != nil { utils.PrintError(err) return err diff --git a/cmd/virtualmachines/wait.go b/cmd/virtualmachines/wait.go index f13c66a..f55155e 100644 --- a/cmd/virtualmachines/wait.go +++ b/cmd/virtualmachines/wait.go @@ -6,7 +6,6 @@ import ( "fmt" "os" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" "github.com/latitudesh/lsh/cmd/lsh" "github.com/latitudesh/lsh/internal/utils" @@ -37,7 +36,7 @@ func waitForVirtualMachine(cmd *cobra.Command, vmID string) error { fmt.Fprintf(os.Stderr, "Waiting for virtual machine %s to finish provisioning… (Ctrl+C to stop)\n", vmID) - want := []components.VirtualMachineAttributesStatus{components.VirtualMachineAttributesStatusRunning} + want := []wait.VirtualMachineStatus{wait.VirtualMachineStatusRunning} status, err := wait.ForVirtualMachineState(ctx, client, vmID, want, o, operations.WithRetries(lsh.RetryConfig())) switch { case errors.Is(err, wait.ErrCanceled): @@ -54,7 +53,7 @@ func waitForVirtualMachine(cmd *cobra.Command, vmID string) error { // real, final state. The wait itself already succeeded, so a failed // re-fetch only degrades the display — surface it on stderr. if !lsh.Debug { - resp, err := client.VirtualMachines.Get(ctx, vmID, operations.WithRetries(lsh.RetryConfig())) + resp, err := client.VirtualMachines.Get(ctx, vmID, nil, operations.WithRetries(lsh.RetryConfig())) if err == nil && resp.VirtualMachine != nil && resp.VirtualMachine.Data != nil { vm := VirtualMachine{VirtualMachineAttributes: *resp.VirtualMachine.Data} utils.RenderStatic(vm.GetData()) diff --git a/go.mod b/go.mod index 2c28e73..d36b1dd 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,9 @@ require ( github.com/go-openapi/swag v0.22.7 github.com/go-openapi/validate v0.22.6 github.com/jmespath/go-jmespath v0.4.0 - github.com/latitudesh/latitudesh-go-sdk v1.16.5 + github.com/latitudesh/latitudesh-go-sdk v1.19.20 github.com/manifoldco/promptui v0.9.0 + github.com/minio/minio-go/v7 v7.1.0 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pb33f/libopenapi v0.15.14 @@ -22,8 +23,8 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 - golang.org/x/term v0.37.0 - golang.org/x/text v0.17.0 + golang.org/x/term v0.45.0 + golang.org/x/text v0.41.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -33,14 +34,18 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/ansi v0.10.1 // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect - github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.22.0 // indirect @@ -48,43 +53,56 @@ require ( github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/loads v0.21.5 // indirect github.com/go-openapi/spec v0.20.13 // indirect - github.com/google/uuid v1.5.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.3.1 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sahilm/fuzzy v0.1.1 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spyzhov/ajson v0.8.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.mongodb.org/mongo-driver v1.17.7 // indirect go.opentelemetry.io/otel v1.17.0 // indirect go.opentelemetry.io/otel/metric v1.17.0 // indirect go.opentelemetry.io/otel/trace v1.17.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/sys v0.38.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect ) diff --git a/go.sum b/go.sum index 3059508..fe0ceed 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPn github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= @@ -24,14 +26,16 @@ github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0G github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= -github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= -github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/dave/jennifer v1.7.0 h1:uRbSBH9UTS64yXbh4FrMHfgfY762RD+C7bUPKODpSJE= github.com/dave/jennifer v1.7.0/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= @@ -42,6 +46,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -50,6 +56,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -89,11 +97,11 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -106,6 +114,13 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -115,10 +130,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/latitudesh/latitudesh-go-sdk v1.16.5 h1:lPaGS8Xz1DNQPOj2bHm6htx0j9S3G6iGRw/WNe1ckjQ= -github.com/latitudesh/latitudesh-go-sdk v1.16.5/go.mod h1:Uso3PR7vjaRxfpAzlDt6LUYLvBdiL4ZZBrYmKm64sp8= -github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= -github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/latitudesh/latitudesh-go-sdk v1.19.20 h1:hxEGwDu2l4Sb6uGELxz8nSbhDrrl/aMowYTDlHatM9A= +github.com/latitudesh/latitudesh-go-sdk v1.19.20/go.mod h1:ckEY2Redz9dr7OVvdLA6uUMvP57WAX2Dt5RP2sQdSo8= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -130,8 +145,14 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= -github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.1.0 h1:QEt5IStDpxgGjEdtOgpiZ5QhmSl3ax7qy61vi2SwHO8= +github.com/minio/minio-go/v7 v7.1.0/go.mod h1:Dm7WS1AgLmBa0NcQD6SeJnJf+K/EUW3GR7Ks6olB3OA= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -165,18 +186,21 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pb33f/libopenapi v0.15.14 h1:A0fn45jbthDyFGXfu5bYIZVsWyPI6hJYm3wG143MT8o= github.com/pb33f/libopenapi v0.15.14/go.mod h1:PEXNwvtT4KNdjrwudp5OYnD1ryqK6uJ68aMNyWvoMuc= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= +github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= @@ -188,8 +212,8 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= @@ -203,16 +227,20 @@ github.com/spyzhov/ajson v0.8.0/go.mod h1:63V+CGM6f1Bu/p4nLIN8885ojBdt88TbLoSFzy github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= @@ -220,6 +248,10 @@ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+x github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver v1.17.7 h1:a9w+U3Vt67eYzcfq3k/OAv284/uUUkL0uP75VE5rCOU= go.mongodb.org/mongo-driver v1.17.7/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM= @@ -230,13 +262,15 @@ go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0 go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ= go.opentelemetry.io/otel/trace v1.17.0 h1:/SWhSRHmDPOImIAetP1QAeMnZYiQXrTy4fMMYOdSKWQ= go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -247,13 +281,13 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -272,18 +306,18 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -304,8 +338,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/config/lock.go b/internal/config/lock.go new file mode 100644 index 0000000..6efeeda --- /dev/null +++ b/internal/config/lock.go @@ -0,0 +1,45 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" +) + +// Update applies mutate to the configuration file as one transaction: the file +// is locked, loaded, mutated and written back before the lock is released. +// +// Load/Save on their own are not enough. Save writes a temp file and renames it +// atomically, so a single write can never be torn — but two processes that both +// load, mutate and save (two `lsh s3 access-keys create --save` runs, say) each +// hold a full snapshot, and the second rename silently drops the first one's +// change. For an S3 access key that means losing a secret the API only ever +// returns once, while the credential stays live on the server. Serialising the +// whole read-modify-write is the only way to avoid it. +// +// The lock is advisory and per-file; readers do not take it, so a concurrent +// reader may still observe the pre-update file (which is consistent, thanks to +// the atomic rename). +func Update(mutate func(*File) error) error { + path, err := Path() + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), dirPerm); err != nil { + return fmt.Errorf("config: mkdir: %w", err) + } + unlock, err := lockConfig(path + ".lock") + if err != nil { + return err + } + defer unlock() + + f, err := Load() + if err != nil { + return err + } + if err := mutate(f); err != nil { + return err + } + return Save(f) +} diff --git a/internal/config/lock_test.go b/internal/config/lock_test.go new file mode 100644 index 0000000..ecec146 --- /dev/null +++ b/internal/config/lock_test.go @@ -0,0 +1,96 @@ +package config + +import ( + "path/filepath" + "sync" + "testing" +) + +// TestUpdateSerializesConcurrentWriters covers the transaction guarantee: two +// processes saving different keys at the same time must both survive. Without +// the lock each writer holds its own snapshot and the last rename drops the +// other's entry — losing a secret the API only returns once. +func TestUpdateSerializesConcurrentWriters(t *testing.T) { + t.Setenv("LSH_CONFIG_PATH", filepath.Join(t.TempDir(), "config.json")) + if err := Update(func(f *File) error { + f.SetProfile("p", Profile{Authorization: "token"}) + return nil + }); err != nil { + t.Fatal(err) + } + + const writers = 8 + var wg sync.WaitGroup + errs := make(chan error, writers) + for i := 0; i < writers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + errs <- Update(func(f *File) error { + _, p, err := f.Resolve("p") + if err != nil { + return err + } + p.SetObjectStorageKey(string(rune('a'+i)), StoredAccessKey{AccessKeyID: "AK", Scope: ScopeFullAccess}) + f.SetProfile("p", p) + return nil + }) + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("Update: %v", err) + } + } + + f, err := Load() + if err != nil { + t.Fatal(err) + } + _, p, err := f.Resolve("p") + if err != nil { + t.Fatal(err) + } + keys := p.ObjectStorageKeys() + if len(keys) != writers { + t.Errorf("%d keys survived, want %d — a concurrent save overwrote another: %v", len(keys), writers, keys) + } + if p.Authorization != "token" { + t.Errorf("the rest of the profile must be preserved, got %+v", p) + } +} + +// TestUpdateMutateErrorLeavesFileAlone keeps a failed transaction from writing. +func TestUpdateMutateErrorLeavesFileAlone(t *testing.T) { + t.Setenv("LSH_CONFIG_PATH", filepath.Join(t.TempDir(), "config.json")) + if err := Update(func(f *File) error { + f.SetProfile("p", Profile{Authorization: "first"}) + return nil + }); err != nil { + t.Fatal(err) + } + sentinel := errNoWrite{} + if err := Update(func(f *File) error { + f.SetProfile("p", Profile{Authorization: "second"}) + return sentinel + }); err != sentinel { + t.Fatalf("Update returned %v, want the mutate error", err) + } + f, err := Load() + if err != nil { + t.Fatal(err) + } + _, p, err := f.Resolve("p") + if err != nil { + t.Fatal(err) + } + if p.Authorization != "first" { + t.Errorf("a failed transaction must not be written, got %q", p.Authorization) + } +} + +type errNoWrite struct{} + +func (errNoWrite) Error() string { return "no write" } diff --git a/internal/config/lock_unix.go b/internal/config/lock_unix.go new file mode 100644 index 0000000..a8c1a9d --- /dev/null +++ b/internal/config/lock_unix.go @@ -0,0 +1,27 @@ +//go:build !windows + +package config + +import ( + "fmt" + "os" + "syscall" +) + +// lockConfig takes an exclusive advisory lock on path, blocking until it is +// available. The kernel releases flock locks when the file descriptor closes, +// so a process that dies mid-update never leaves the lock behind. +func lockConfig(path string) (func(), error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, filePerm) + if err != nil { + return nil, fmt.Errorf("config: open lock: %w", err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + f.Close() + return nil, fmt.Errorf("config: lock: %w", err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + }, nil +} diff --git a/internal/config/lock_windows.go b/internal/config/lock_windows.go new file mode 100644 index 0000000..3035b19 --- /dev/null +++ b/internal/config/lock_windows.go @@ -0,0 +1,31 @@ +//go:build windows + +package config + +import ( + "fmt" + "os" + + "golang.org/x/sys/windows" +) + +// lockConfig takes an exclusive lock on path, blocking until it is available. +// Windows releases the range lock when the handle closes, matching the flock +// behaviour used on the other platforms. +func lockConfig(path string) (func(), error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, filePerm) + if err != nil { + return nil, fmt.Errorf("config: open lock: %w", err) + } + handle := windows.Handle(f.Fd()) + var overlapped windows.Overlapped + if err := windows.LockFileEx(handle, windows.LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &overlapped); err != nil { + f.Close() + return nil, fmt.Errorf("config: lock: %w", err) + } + return func() { + var release windows.Overlapped + _ = windows.UnlockFileEx(handle, 0, 1, 0, &release) + _ = f.Close() + }, nil +} diff --git a/internal/config/objectstorage.go b/internal/config/objectstorage.go new file mode 100644 index 0000000..e10fb1e --- /dev/null +++ b/internal/config/objectstorage.go @@ -0,0 +1,158 @@ +package config + +import ( + "sort" + "time" +) + +// Access-key scopes as reported by the API. +const ( + ScopeFullAccess = "fullaccess" + ScopeLimitedAccess = "limited_access" + // ScopeUnknown marks keys imported without being matched against the + // API. They are only used when selected explicitly with --access-key. + ScopeUnknown = "unknown" +) + +// Per-bucket permissions of a limited_access key. +const ( + PermissionRW = "rw" + PermissionReadOnly = "readonly" +) + +// Origins of a stored key (informational). +const ( + KeySourceConfigure = "configure" + KeySourceCreate = "access-key-create" + KeySourceMakeBkt = "mb" + KeySourceImport = "import" +) + +// ObjectStorageConfig is the per-profile store of S3 access keys. +type ObjectStorageConfig struct { + // DefaultKey breaks ties when several saved keys cover a bucket equally. + DefaultKey string `json:"default_key,omitempty"` + // Keys is indexed by the logical key name the user passes to --access-key. + Keys map[string]StoredAccessKey `json:"keys,omitempty"` +} + +// StoredAccessKey is one S3 access key saved locally. The scope metadata is +// what lets the CLI pick the least-privileged key covering a bucket without +// asking the API again. +type StoredAccessKey struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + // StorageClass is standard (Wasabi) or high_performance (VAST). + StorageClass string `json:"storage_class"` + // Site is the Latitude site slug (e.g. TYO4). Set for high_performance + // keys, which are bound to one VAST cluster; empty for standard keys, + // which cover every site of the project. + Site string `json:"site,omitempty"` + ProjectID string `json:"project_id,omitempty"` + // Scope is fullaccess, limited_access or unknown. + Scope string `json:"scope"` + // Buckets maps bkt_ IDs to rw|readonly for limited_access keys. + Buckets map[string]string `json:"buckets,omitempty"` + // Username is the backend identity the API needs on DELETE. + Username string `json:"username,omitempty"` + Source string `json:"source,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// String renders the key without its secret so accidental %v/%s formatting +// never leaks it into logs or errors. +func (k StoredAccessKey) String() string { + return "access_key_id=" + k.AccessKeyID + " secret=[redacted]" +} + +// GoString mirrors String for %#v. +func (k StoredAccessKey) GoString() string { return k.String() } + +// Covers reports whether the key grants access to bucketID with the +// requested permission ("rw" or "readonly"); a fullaccess key covers every +// bucket, a limited key only the listed ones. +func (k StoredAccessKey) Covers(bucketID string, needWrite bool) bool { + switch k.Scope { + case ScopeFullAccess: + return true + case ScopeLimitedAccess: + perm, ok := k.Buckets[bucketID] + if !ok { + return false + } + return !needWrite || perm == PermissionRW + } + return false +} + +// Permission returns rw, readonly or "" for bucketID. +func (k StoredAccessKey) Permission(bucketID string) string { + if k.Scope == ScopeFullAccess { + return PermissionRW + } + return k.Buckets[bucketID] +} + +// ObjectStorageKeys returns the saved keys of the profile (never nil). +func (p Profile) ObjectStorageKeys() map[string]StoredAccessKey { + if p.ObjectStorage == nil || p.ObjectStorage.Keys == nil { + return map[string]StoredAccessKey{} + } + return p.ObjectStorage.Keys +} + +// SortedObjectStorageKeyNames returns the saved key names alphabetically. +func (p Profile) SortedObjectStorageKeyNames() []string { + keys := p.ObjectStorageKeys() + names := make([]string, 0, len(keys)) + for n := range keys { + names = append(names, n) + } + sort.Strings(names) + return names +} + +// SetObjectStorageKey inserts or replaces a saved key. +func (p *Profile) SetObjectStorageKey(name string, k StoredAccessKey) { + if p.ObjectStorage == nil { + p.ObjectStorage = &ObjectStorageConfig{} + } + if p.ObjectStorage.Keys == nil { + p.ObjectStorage.Keys = map[string]StoredAccessKey{} + } + p.ObjectStorage.Keys[name] = k +} + +// RemoveObjectStorageKey deletes a saved key by name, clearing DefaultKey if +// it pointed at it. Returns false when the name was not stored. +func (p *Profile) RemoveObjectStorageKey(name string) bool { + if p.ObjectStorage == nil || p.ObjectStorage.Keys == nil { + return false + } + if _, ok := p.ObjectStorage.Keys[name]; !ok { + return false + } + delete(p.ObjectStorage.Keys, name) + if p.ObjectStorage.DefaultKey == name { + p.ObjectStorage.DefaultKey = "" + } + return true +} + +// FindObjectStorageKeyByID looks a saved key up by its access key ID. +func (p Profile) FindObjectStorageKeyByID(accessKeyID string) (string, StoredAccessKey, bool) { + for name, k := range p.ObjectStorageKeys() { + if k.AccessKeyID == accessKeyID { + return name, k, true + } + } + return "", StoredAccessKey{}, false +} + +// DefaultObjectStorageKey returns the configured tie-breaker key name. +func (p Profile) DefaultObjectStorageKey() string { + if p.ObjectStorage == nil { + return "" + } + return p.ObjectStorage.DefaultKey +} diff --git a/internal/config/profile.go b/internal/config/profile.go index 477d18b..f1a4ac2 100644 --- a/internal/config/profile.go +++ b/internal/config/profile.go @@ -13,6 +13,10 @@ type Profile struct { Email string `json:"email,omitempty"` Source string `json:"source,omitempty"` APIVersion string `json:"api_version,omitempty"` + + // ObjectStorage holds the S3 access keys saved for `lsh s3`. Optional: + // profiles written before this field existed load unchanged. + ObjectStorage *ObjectStorageConfig `json:"object_storage,omitempty"` } // SourceBrowser is set on profiles created via the browser-assisted diff --git a/internal/exitcode/exitcode.go b/internal/exitcode/exitcode.go new file mode 100644 index 0000000..5995b33 --- /dev/null +++ b/internal/exitcode/exitcode.go @@ -0,0 +1,89 @@ +// Package exitcode defines the process exit codes used by the newer command +// groups (starting with `lsh s3`) and the error type that carries them. +// +// Historically every lsh failure exited with 1. Object storage commands are +// used heavily in scripts and CI, where distinguishing "not found" from +// "no credentials" from "refused for safety" matters, so they attach an +// explicit code to their errors. main() maps the error to os.Exit through +// Of(); commands that do not opt in keep exiting with 1. +package exitcode + +import ( + "errors" + "fmt" +) + +const ( + // OK is a successful run (including empty listings). + OK = 0 + // Generic is an unexpected error, or one or more transfers failed. + Generic = 1 + // Usage is invalid usage: bad URI/flag, ambiguous bucket, --recursive + // without --all, and other argument errors. + Usage = 2 + // NotFound covers a missing bucket, object, access key or lifecycle rule. + NotFound = 3 + // Credentials means no usable S3 credential was found, or the backend + // rejected the credential (InvalidAccessKeyId, SignatureDoesNotMatch). + Credentials = 4 + // Permission is an authenticated request the backend refused (403). + Permission = 5 + // Partial means the operation completed for some objects and failed for + // others (recursive delete, bucket emptying). + Partial = 6 + // Refused is a safety refusal: non-empty bucket without --force, + // --max-delete exceeded, object lock, prompt declined, or a confirmation + // required in a non-interactive session without --yes. + Refused = 7 + // Interrupted is returned after a SIGINT once cleanup finished. + Interrupted = 130 +) + +// OptInAnnotation marks a cobra command whose subtree uses these exit codes. +// Command groups that predate them keep exiting 1 for every failure, so shared +// pre-run validation has to know which contract applies. Commands set it on +// themselves (see cmd/s3.Finalize) and callers walk up the parents. +const OptInAnnotation = "lsh.exit_codes" + +// Error pairs an error with the exit code the process should terminate with. +type Error struct { + Code int + Err error +} + +func (e *Error) Error() string { + if e.Err == nil { + return fmt.Sprintf("exit %d", e.Code) + } + return e.Err.Error() +} + +// Unwrap exposes the wrapped error to errors.Is/As. +func (e *Error) Unwrap() error { return e.Err } + +// New wraps err with an exit code. A nil err yields nil. +func New(code int, err error) error { + if err == nil { + return nil + } + return &Error{Code: code, Err: err} +} + +// Errorf formats an error and attaches an exit code. +func Errorf(code int, format string, a ...interface{}) error { + return &Error{Code: code, Err: fmt.Errorf(format, a...)} +} + +// Of returns the exit code carried by err (the outermost *Error in the chain +// wins, as errors.As walks from the outside in), Generic when err carries +// none, and OK for nil. +func Of(err error) int { + if err == nil { + return OK + } + var e *Error + if errors.As(err, &e) { + return e.Code + } + return Generic +} diff --git a/internal/objectstorage/apiclient.go b/internal/objectstorage/apiclient.go new file mode 100644 index 0000000..d5e3db5 --- /dev/null +++ b/internal/objectstorage/apiclient.go @@ -0,0 +1,93 @@ +package objectstorage + +import ( + "bytes" + "io" + "net/http" + "regexp" + "strings" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/spf13/viper" +) + +// NewAPIClient returns the Latitude SDK client used by the object storage +// commands. It is the regular authenticated client with one addition: a +// transport that normalizes known API/SDK mismatches in /storage responses +// before the generated models decode them. +// +// Today the API serializes `retention_period` as "" (or a numeric string) for +// buckets without object lock, while the v1.19.20 model declares it as +// *int64, which makes every bucket list/get fail with "cannot unmarshal string +// into Go value of type int64". Until the API returns null/number, the +// transport rewrites the field. Everything else passes through untouched. +func NewAPIClient() *sdk.Latitudesh { + token := viper.GetString("Authorization") + opts := []sdk.SDKOption{ + sdk.WithSecurity(token), + sdk.WithClient(&normalizingHTTPClient{inner: http.DefaultClient}), + } + // Honour the global --hostname/--scheme overrides (dev/staging, tests) + // the same way the legacy client does; the SDK default is production. + if host := viper.GetString("hostname"); host != "" && host != "api.latitude.sh" { + scheme := viper.GetString("scheme") + if scheme == "" { + scheme = "https" + } + opts = append(opts, sdk.WithServerURL(scheme+"://"+host)) + } + return sdk.New(opts...) +} + +type normalizingHTTPClient struct { + inner *http.Client +} + +// reRetentionString matches retention_period serialized as any JSON string. +var reRetentionString = regexp.MustCompile(`"retention_period"\s*:\s*"((?:[^"\\]|\\.)*)"`) + +// Do executes the request and rewrites storage payloads. +func (c *normalizingHTTPClient) Do(req *http.Request) (*http.Response, error) { + resp, err := c.inner.Do(req) + if err != nil || resp == nil || resp.Body == nil { + return resp, err + } + if !strings.Contains(req.URL.Path, "/storage/") || !strings.Contains(resp.Header.Get("Content-Type"), "json") { + return resp, nil + } + body, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + if readErr != nil { + return nil, readErr + } + fixed := NormalizeStorageJSON(body) + resp.Body = io.NopCloser(bytes.NewReader(fixed)) + resp.ContentLength = int64(len(fixed)) + resp.Header.Del("Content-Length") + return resp, nil +} + +// NormalizeStorageJSON applies the known field fixes to a /storage payload. +func NormalizeStorageJSON(body []byte) []byte { + if !bytes.Contains(body, []byte(`"retention_period"`)) { + return body + } + return reRetentionString.ReplaceAllFunc(body, func(m []byte) []byte { + sub := reRetentionString.FindSubmatch(m) + if len(sub) == 2 { + if v := strings.TrimSpace(string(sub[1])); v != "" && isDigits(v) { + return []byte(`"retention_period":` + v) + } + } + return []byte(`"retention_period":null`) + }) +} + +func isDigits(s string) bool { + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return s != "" +} diff --git a/internal/objectstorage/apiclient_test.go b/internal/objectstorage/apiclient_test.go new file mode 100644 index 0000000..e3dea8f --- /dev/null +++ b/internal/objectstorage/apiclient_test.go @@ -0,0 +1,21 @@ +package objectstorage + +import "testing" + +func TestNormalizeStorageJSON(t *testing.T) { + cases := map[string]string{ + `{"retention_period":""}`: `{"retention_period":null}`, + `{"retention_period": ""}`: `{"retention_period":null}`, + `{"retention_period":"30"}`: `{"retention_period":30}`, + `{"retention_period":"n/a"}`: `{"retention_period":null}`, + `{"retention_period":30}`: `{"retention_period":30}`, + `{"retention_period":null}`: `{"retention_period":null}`, + `{"name":"x"}`: `{"name":"x"}`, + `{"a":{"retention_period":"7"},"b":{"retention_period":""}}`: `{"a":{"retention_period":7},"b":{"retention_period":null}}`, + } + for in, want := range cases { + if got := string(NormalizeStorageJSON([]byte(in))); got != want { + t.Errorf("NormalizeStorageJSON(%s) = %s, want %s", in, got, want) + } + } +} diff --git a/internal/objectstorage/client.go b/internal/objectstorage/client.go new file mode 100644 index 0000000..7e27397 --- /dev/null +++ b/internal/objectstorage/client.go @@ -0,0 +1,138 @@ +package objectstorage + +import ( + "context" + "io" + "net" + "net/http" + "os" + "os/signal" + "regexp" + "syscall" + "time" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/version" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// Addressing styles for the escape hatch flag. +const ( + AddressingPath = "path" + AddressingVirtual = "virtual" +) + +// ClientOptions tunes NewS3Client. +type ClientOptions struct { + // Debug enables HTTP tracing (redacted) to Trace (stderr when nil). + Debug bool + Trace io.Writer + // Addressing is path (default) or virtual. + Addressing string + // MaxRetries caps request retries (minio default 10 when zero). + MaxRetries int +} + +// NewS3Client builds the minio client for a bucket with the settings every +// S3-compatible backend needs: explicit endpoint and signing region (never +// discovered through GetBucketLocation), path-style addressing, no trailing +// checksums (Content-MD5 is used instead), sane timeouts without a total +// deadline so long uploads are not killed. +func NewS3Client(b *Bucket, c Credential, o ClientOptions) (*minio.Client, error) { + if err := b.Validate(); err != nil { + return nil, err + } + host, secure, err := EndpointHost(b.Endpoint) + if err != nil { + return nil, exitcode.New(exitcode.Usage, err) + } + lookup := minio.BucketLookupPath + if o.Addressing == AddressingVirtual { + lookup = minio.BucketLookupDNS + } + region := b.SigningRegion + if region == "" { + region = DefaultSigningRegion + } + client, err := minio.New(host, &minio.Options{ + Creds: credentials.NewStaticV4(c.AccessKeyID, c.Secret(), ""), + Secure: secure, + Region: region, + BucketLookup: lookup, + TrailingHeaders: false, + Transport: newTransport(), + MaxRetries: o.MaxRetries, + }) + if err != nil { + return nil, exitcode.Errorf(exitcode.Generic, "could not create S3 client for %s: %v", host, err) + } + client.SetAppInfo("lsh", version.Version) + if o.Debug { + w := o.Trace + if w == nil { + w = os.Stderr + } + client.TraceOn(&redactingWriter{w: w}) + } + return client, nil +} + +// newTransport mirrors the minio default transport but with explicit +// connection timeouts and no overall request deadline. +func newTransport() http.RoundTripper { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 15 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 64, + MaxIdleConnsPerHost: 16, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 15 * time.Second, + ResponseHeaderTimeout: 90 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ForceAttemptHTTP2: true, + DisableCompression: true, + } +} + +var redactPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?i)(Signature=)[0-9a-f]+`), + regexp.MustCompile(`(?i)(X-Amz-Signature=)[0-9a-f]+`), + regexp.MustCompile(`(?i)(X-Amz-Security-Token=)[^&\s]+`), + regexp.MustCompile(`(?i)(x-amz-security-token: )\S+`), + regexp.MustCompile(`(?i)("?secret_access_key"?\s*[:=]\s*"?)[^",\s]+`), + regexp.MustCompile(`(?i)("?secret_key"?\s*[:=]\s*"?)[^",\s]+`), +} + +// redactingWriter strips signatures and secrets from minio's HTTP trace. +type redactingWriter struct{ w io.Writer } + +func (r *redactingWriter) Write(p []byte) (int, error) { + out := p + for _, re := range redactPatterns { + out = re.ReplaceAll(out, []byte("${1}[redacted]")) + } + if _, err := r.w.Write(out); err != nil { + return 0, err + } + return len(p), nil +} + +// Redact applies the same redaction to an arbitrary string (error messages, +// debug lines). +func Redact(s string) string { + for _, re := range redactPatterns { + s = re.ReplaceAllString(s, "${1}[redacted]") + } + return s +} + +// SignalContext returns a context cancelled on SIGINT/SIGTERM so transfers can +// abort cleanly (multipart uploads are aborted by minio on error). The +// returned stop function releases the signal handler. +func SignalContext(parent context.Context) (context.Context, context.CancelFunc) { + return signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM) +} diff --git a/internal/objectstorage/client_test.go b/internal/objectstorage/client_test.go new file mode 100644 index 0000000..7304286 --- /dev/null +++ b/internal/objectstorage/client_test.go @@ -0,0 +1,138 @@ +package objectstorage + +import ( + "bytes" + "context" + "io" + "regexp" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/objectstorage/s3test" + "github.com/minio/minio-go/v7" +) + +// TestClientAgainstFakeS3 exercises the client settings that matter for +// S3-compatible backends: path-style addressing, the derived signing region, +// no checksum trailers on writes, and that the fake server round-trips +// list/put/get/delete. +func TestClientAgainstFakeS3(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("backups-7f3a") + srv.AddObject("backups-7f3a", "2026/09/a.sql", []byte("aaa"), "application/sql") + srv.AddObject("backups-7f3a", "2026/09/b.sql", []byte("bbbb"), "application/sql") + srv.AddObject("backups-7f3a", "readme.txt", []byte("hi"), "text/plain") + + b := &Bucket{ID: "bkt_1", Name: "backups", BucketName: "backups-7f3a", Endpoint: srv.URL(), StorageClass: "standard", SigningRegion: "eu-west-2"} + var trace bytes.Buffer + client, err := NewS3Client(b, NewCredential("AKIAEXAMPLE", "topsecret", "test"), ClientOptions{Debug: true, Trace: &trace, MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + // Non-recursive listing: one prefix + one object. + var keys []string + var prefixes []string + for info := range client.ListObjects(ctx, b.BucketName, minio.ListObjectsOptions{Recursive: false}) { + if info.Err != nil { + t.Fatal(info.Err) + } + if strings.HasSuffix(info.Key, "/") { + prefixes = append(prefixes, info.Key) + } else { + keys = append(keys, info.Key) + } + } + if len(prefixes) != 1 || prefixes[0] != "2026/" || len(keys) != 1 || keys[0] != "readme.txt" { + t.Fatalf("listing: prefixes=%v keys=%v", prefixes, keys) + } + + // Upload, download, delete. + if _, err := client.PutObject(ctx, b.BucketName, "new/file.bin", bytes.NewReader([]byte("payload")), 7, minio.PutObjectOptions{ContentType: "application/octet-stream"}); err != nil { + t.Fatal(err) + } + obj, err := client.GetObject(ctx, b.BucketName, "new/file.bin", minio.GetObjectOptions{}) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(obj) + if err != nil || string(data) != "payload" { + t.Fatalf("download = %q, %v", data, err) + } + if err := client.RemoveObject(ctx, b.BucketName, "new/file.bin", minio.RemoveObjectOptions{}); err != nil { + t.Fatal(err) + } + if srv.Object("backups-7f3a", "new/file.bin") != nil { + t.Fatal("object still present after delete") + } + // Deleting a missing key is idempotent (S3 semantics). + if err := client.RemoveObject(ctx, b.BucketName, "missing", minio.RemoveObjectOptions{}); err != nil { + t.Fatalf("delete of missing key must succeed: %v", err) + } + + reqs := srv.Requests() + if len(reqs) == 0 { + t.Fatal("no requests recorded") + } + for _, r := range reqs { + if !strings.HasPrefix(r.Path, "/backups-7f3a") { + t.Errorf("expected path-style request, got %s", r.Path) + } + if region := s3test.SigningRegionOf(r); region != "eu-west-2" { + t.Errorf("signing region = %q, want eu-west-2 (%s %s)", region, r.Method, r.Path) + } + } + if srv.HasChecksumHeaders() { + t.Error("writes must not carry x-amz-checksum-*/aws-chunked headers (Wasabi rejects them)") + } + if strings.Contains(trace.String(), "topsecret") { + t.Error("trace leaked the secret") + } + // minio already masks the SigV4 signature in its trace; our writer must + // keep it that way (no 64-hex signature anywhere in the output). + if regexp.MustCompile(`Signature=[0-9a-f]{64}`).MatchString(trace.String()) { + t.Error("trace did not redact the signature") + } +} + +func TestHumanizeMapsS3Errors(t *testing.T) { + srv := s3test.New() + defer srv.Close() + srv.CreateBucket("bkt-test") + b := &Bucket{ID: "bkt_1", Name: "b", BucketName: "bkt-test", Endpoint: srv.URL(), SigningRegion: "us-east-1"} + cred := NewCredential("AK", "SK", "saved key \"x\"") + client, err := NewS3Client(b, cred, ClientOptions{MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + _, err = client.StatObject(ctx, "bkt-test", "nope", minio.StatObjectOptions{}) + if h := Humanize(err, b, &cred); h == nil || !strings.Contains(h.Error(), "not found") { + t.Errorf("NoSuchKey → %v", h) + } + + srv.DenyAll = true + _, err = client.StatObject(ctx, "bkt-test", "x", minio.StatObjectOptions{}) + h := Humanize(err, b, &cred) + if h == nil || !strings.Contains(h.Error(), "access denied") { + t.Errorf("AccessDenied → %v", h) + } + srv.DenyAll = false + + // HEAD responses carry no body, so use a GET (listing) to exercise the + // XML error mapping. + srv.FailNext = &s3test.ErrorResponse{Code: "AuthorizationHeaderMalformed", Message: "wrong region", Region: "eu-west-2", Status: 400} + err = nil + for info := range client.ListObjects(ctx, "bkt-test", minio.ListObjectsOptions{}) { + if info.Err != nil { + err = info.Err + } + } + h = Humanize(err, b, &cred) + if h == nil || !strings.Contains(h.Error(), "eu-west-2") { + t.Errorf("AuthorizationHeaderMalformed → %v", h) + } +} diff --git a/internal/objectstorage/creds.go b/internal/objectstorage/creds.go new file mode 100644 index 0000000..cea0622 --- /dev/null +++ b/internal/objectstorage/creds.go @@ -0,0 +1,399 @@ +package objectstorage + +import ( + "errors" + "fmt" + "os" + "sort" + "strings" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" +) + +// Environment variables understood by the S3 layer. +const ( + EnvAccessKeyID = "LSH_S3_ACCESS_KEY_ID" + EnvSecretAccessKey = "LSH_S3_SECRET_ACCESS_KEY" + // EnvUseAWSEnv opts in to reading AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY + // as a fallback. Off by default: those variables almost always point at + // the real AWS and would produce opaque 403s against Latitude. + EnvUseAWSEnv = "LSH_S3_USE_AWS_ENV" + EnvEndpointURL = "LSH_S3_ENDPOINT_URL" + EnvSigningRegion = "LSH_S3_SIGNING_REGION" +) + +// Credential is a resolved S3 credential plus where it came from. The secret +// is unexported so that formatting the value never prints it. +type Credential struct { + // Name is the saved key name; empty for environment credentials. + Name string + AccessKeyID string + secret string + // Source describes the origin for messages and --debug: + // "LSH_S3_ACCESS_KEY_ID (environment)", "saved key \"ci\" (profile acme)". + Source string + // FromEnv is true when the credential came from environment variables. + FromEnv bool + // Key carries the scope metadata of a saved key (zero for env). + Key config.StoredAccessKey + // Profile is the profile name the key was loaded from. + Profile string +} + +// NewCredential builds a credential from explicit values (tests, imports). +func NewCredential(id, secret, source string) Credential { + return Credential{AccessKeyID: id, secret: secret, Source: source} +} + +// Secret returns the secret access key. +func (c Credential) Secret() string { return c.secret } + +// String never includes the secret. +func (c Credential) String() string { + return fmt.Sprintf("access_key_id=%s source=%s secret=[redacted]", c.AccessKeyID, c.Source) +} + +// GoString mirrors String for %#v. +func (c Credential) GoString() string { return c.String() } + +// Describe renders the credential for user-facing messages, including its +// permission on bucketID when known. +func (c Credential) Describe(bucketID string) string { + if c.FromEnv || c.Name == "" { + return c.Source + } + perm := c.Key.Permission(bucketID) + switch { + case c.Key.Scope == config.ScopeFullAccess: + return fmt.Sprintf("saved key %q (fullaccess)", c.Name) + case perm != "": + return fmt.Sprintf("saved key %q (limited_access, %s on this bucket)", c.Name, perm) + default: + return fmt.Sprintf("saved key %q (%s)", c.Name, c.Key.Scope) + } +} + +// CredentialOptions controls ResolveCredential. +type CredentialOptions struct { + // AccessKeyName is the --access-key flag: force a saved key by name. + AccessKeyName string + // ProfileOverride is the --profile flag. + ProfileOverride string + // Write is true when the operation needs write permission on the bucket + // (upload, delete, presign PUT). + Write bool +} + +// EnvCredential reads LSH_S3_* (and, when opted in, AWS_*) from the +// environment. ok is false when nothing is set; err reports a half-set pair. +func EnvCredential() (Credential, bool, error) { + id, secret := os.Getenv(EnvAccessKeyID), os.Getenv(EnvSecretAccessKey) + switch { + case id != "" && secret != "": + return Credential{AccessKeyID: id, secret: secret, Source: EnvAccessKeyID + " (environment)", FromEnv: true}, true, nil + case id != "" && secret == "": + return Credential{}, false, exitcode.Errorf(exitcode.Credentials, "%s is set but %s is missing", EnvAccessKeyID, EnvSecretAccessKey) + case id == "" && secret != "": + return Credential{}, false, exitcode.Errorf(exitcode.Credentials, "%s is set but %s is missing", EnvSecretAccessKey, EnvAccessKeyID) + } + if os.Getenv(EnvUseAWSEnv) == "1" || strings.EqualFold(os.Getenv(EnvUseAWSEnv), "true") { + if os.Getenv("AWS_SESSION_TOKEN") != "" { + // STS credentials are never Latitude access keys. + return Credential{}, false, nil + } + aid, asecret := os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY") + if aid != "" && asecret != "" { + fmt.Fprintf(os.Stderr, "warning: using AWS_ACCESS_KEY_ID from the environment (%s=1)\n", EnvUseAWSEnv) + return Credential{AccessKeyID: aid, secret: asecret, Source: "AWS_ACCESS_KEY_ID (environment, opt-in fallback)", FromEnv: true}, true, nil + } + } + return Credential{}, false, nil +} + +// ActiveProfile loads the config file and resolves the active profile +// (--profile > LSH_PROFILE > default_profile). +func ActiveProfile(override string) (*config.File, string, config.Profile, error) { + f, err := config.Load() + if err != nil { + return nil, "", config.Profile{}, err + } + name, p, err := f.Resolve(override) + if err != nil { + if errors.Is(err, config.ErrProfileNotFound) { + if override != "" { + return f, name, p, exitcode.Errorf(exitcode.Usage, "profile %q not found — run 'lsh profile list'", override) + } + return f, "", config.Profile{}, exitcode.Errorf(exitcode.Credentials, "no active profile — run 'lsh login' (or use LSH_S3_ACCESS_KEY_ID/LSH_S3_SECRET_ACCESS_KEY)") + } + return f, name, p, err + } + return f, name, p, nil +} + +// ResolveCredential picks the credential for an operation on b: +// +// 1. LSH_S3_ACCESS_KEY_ID + LSH_S3_SECRET_ACCESS_KEY (environment) +// 2. --access-key (saved key, must cover the bucket) +// 3. the least-privileged saved key that covers the bucket +// +// In endpoint-override mode only the environment is consulted: saved keys are +// never sent to a host the API did not vouch for. +func ResolveCredential(b *Bucket, o CredentialOptions) (Credential, error) { + if c, ok, err := EnvCredential(); err != nil { + return Credential{}, err + } else if ok { + return c, nil + } + if b != nil && b.EndpointOverride { + return Credential{}, exitcode.Errorf(exitcode.Credentials, + "saved access keys are not sent to custom endpoints; set %s and %s in the environment", EnvAccessKeyID, EnvSecretAccessKey) + } + _, profileName, profile, err := ActiveProfile(o.ProfileOverride) + if err != nil { + return Credential{}, err + } + keys := profile.ObjectStorageKeys() + + if o.AccessKeyName != "" { + k, ok := keys[o.AccessKeyName] + if !ok { + return Credential{}, exitcode.Errorf(exitcode.NotFound, "no saved access key named %q in profile %s; run 'lsh s3 access-keys list --saved'", o.AccessKeyName, profileName) + } + c := fromStored(o.AccessKeyName, k, profileName) + if b != nil && !b.EndpointOverride && b.ID != "" { + // Covers() is true for every bucket ID on a fullaccess key, so + // without this the class/site/project constraints that automatic + // selection enforces would never be checked for an explicit key: + // the request would go out and fail at the backend instead. + if err := explainIncompatibleKey(o.AccessKeyName, k, b); err != nil { + return Credential{}, err + } + if k.Scope != config.ScopeUnknown && !k.Covers(b.ID, false) { + return Credential{}, exitcode.Errorf(exitcode.Permission, "saved key %q does not cover bucket %s", o.AccessKeyName, b.Display()) + } + if o.Write && k.Scope == config.ScopeLimitedAccess && k.Permission(b.ID) != config.PermissionRW { + return Credential{}, exitcode.Errorf(exitcode.Permission, "saved key %q is readonly on bucket %s; pick a key with rw permission (lsh s3 access-keys list) or create one: lsh s3 access-keys create --bucket %s --save", o.AccessKeyName, b.Display(), b.Name) + } + } + return c, nil + } + + if b == nil { + return Credential{}, exitcode.Errorf(exitcode.Credentials, "no credential selected; pass --access-key or set %s/%s", EnvAccessKeyID, EnvSecretAccessKey) + } + name, k, ok := SelectKey(keys, profile.DefaultObjectStorageKey(), b, o.Write) + if !ok { + return Credential{}, NoCredentialError(b, profileName, o.Write, keys) + } + return fromStored(name, k, profileName), nil +} + +func fromStored(name string, k config.StoredAccessKey, profile string) Credential { + return Credential{ + Name: name, + AccessKeyID: k.AccessKeyID, + secret: k.SecretAccessKey, + Source: fmt.Sprintf("saved key %q (profile %s)", name, profile), + Key: k, + Profile: profile, + } +} + +// explainIncompatibleKey reports why a saved key cannot serve b, or nil when +// nothing rules it out. It applies the constraints keyMatchesBucket uses for +// automatic selection — with the same "empty means unknown, so do not judge" +// rule, which keeps imported keys with partial metadata usable — but names the +// mismatch instead of silently skipping the key, because here the user picked +// it explicitly. +func explainIncompatibleKey(name string, k config.StoredAccessKey, b *Bucket) error { + switch { + case k.StorageClass != "" && b.StorageClass != "" && k.StorageClass != b.StorageClass: + return exitcode.Errorf(exitcode.Usage, + "saved key %q is a %s key and bucket %s is %s; the two backends do not share credentials — run 'lsh s3 access-keys list --saved' to pick another, or drop --access-key to let the CLI choose", + name, k.StorageClass, b.Display(), b.StorageClass) + case k.Site != "" && b.Site != "" && !strings.EqualFold(k.Site, b.Site): + return exitcode.Errorf(exitcode.Usage, + "saved key %q belongs to site %s and bucket %s is in %s; a %s key only works in its own site", + name, strings.ToUpper(k.Site), b.Display(), strings.ToUpper(b.Site), ClassHighPerformance) + case k.ProjectID != "" && b.ProjectID != "" && k.ProjectID != b.ProjectID: + return exitcode.Errorf(exitcode.Usage, + "saved key %q belongs to project %s and bucket %s to project %s", + name, k.ProjectID, b.Display(), b.ProjectID) + } + return nil +} + +// SelectKey applies the automatic selection rules over the saved keys: +// same storage class, compatible site and project, covering the bucket with +// the needed permission. Ranking: limited rw > limited readonly (reads only) +// > fullaccess; ties go to defaultKey, then to the newest key, then by name. +func SelectKey(keys map[string]config.StoredAccessKey, defaultKey string, b *Bucket, write bool) (string, config.StoredAccessKey, bool) { + type cand struct { + name string + key config.StoredAccessKey + rank int + } + var cands []cand + for name, k := range keys { + if !keyMatchesBucket(k, b) { + continue + } + if !k.Covers(b.ID, write) { + continue + } + rank := 2 + if k.Scope == config.ScopeLimitedAccess { + if k.Permission(b.ID) == config.PermissionRW { + rank = 0 + } else { + rank = 1 + } + } + cands = append(cands, cand{name, k, rank}) + } + if len(cands) == 0 { + return "", config.StoredAccessKey{}, false + } + sort.Slice(cands, func(i, j int) bool { + a, c := cands[i], cands[j] + if a.rank != c.rank { + return a.rank < c.rank + } + if (a.name == defaultKey) != (c.name == defaultKey) { + return a.name == defaultKey + } + if !a.key.CreatedAt.Equal(c.key.CreatedAt) { + return a.key.CreatedAt.After(c.key.CreatedAt) + } + return a.name < c.name + }) + return cands[0].name, cands[0].key, true +} + +// keyMatchesBucket checks class, site and project compatibility. Empty values +// on either side are treated as wildcards so keys saved before the site was +// known keep working. +func keyMatchesBucket(k config.StoredAccessKey, b *Bucket) bool { + if k.Scope == config.ScopeUnknown { + return false + } + if k.StorageClass != "" && b.StorageClass != "" && k.StorageClass != b.StorageClass { + return false + } + if k.Site != "" && b.Site != "" && !strings.EqualFold(k.Site, b.Site) { + return false + } + if k.ProjectID != "" && b.ProjectID != "" && k.ProjectID != b.ProjectID { + return false + } + return true +} + +// NoCredentialError explains how to obtain a credential for b. +func NoCredentialError(b *Bucket, profile string, write bool, keys map[string]config.StoredAccessKey) error { + what := "no saved S3 access key covers" + if write { + // Point out read-only keys that exist but cannot write. + for name, k := range keys { + if keyMatchesBucket(k, b) && k.Covers(b.ID, false) { + what = fmt.Sprintf("saved key %q is readonly on this bucket; no saved key can write to", name) + break + } + } + } + return exitcode.Errorf(exitcode.Credentials, + "%s bucket %s (%s, profile %s).\n create one: lsh s3 access-keys create --bucket %s --save\n or run: lsh s3 configure\n or export: %s and %s", + what, b.Display(), b.StorageClass, profile, b.Name, EnvAccessKeyID, EnvSecretAccessKey) +} + +// SaveKey stores k under name in the active profile and persists the file. +// +// The read-modify-write runs inside config.Update so two processes saving keys +// at the same time cannot drop each other's entry: a lost entry would strand a +// live credential whose secret the API never returns again. +func SaveKey(profileOverride, name string, k config.StoredAccessKey) (string, error) { + var profileName string + err := config.Update(func(f *config.File) error { + var p config.Profile + var resolveErr error + profileName, p, resolveErr = resolveProfileIn(f, profileOverride) + if resolveErr != nil { + return resolveErr + } + p.SetObjectStorageKey(name, k) + f.SetProfile(profileName, p) + return nil + }) + return profileName, err +} + +// resolveProfileIn resolves the active profile inside an already-loaded file, +// mapping the same errors ActiveProfile reports. It exists so a config.Update +// transaction works on the file it locked instead of loading a second copy. +func resolveProfileIn(f *config.File, override string) (string, config.Profile, error) { + name, p, err := f.Resolve(override) + if err != nil { + if errors.Is(err, config.ErrProfileNotFound) { + if override != "" { + return name, p, exitcode.Errorf(exitcode.Usage, "profile %q not found — run 'lsh profile list'", override) + } + return "", config.Profile{}, exitcode.Errorf(exitcode.Credentials, "no active profile — run 'lsh login' (or use LSH_S3_ACCESS_KEY_ID/LSH_S3_SECRET_ACCESS_KEY)") + } + return name, p, err + } + return name, p, nil +} + +// ForgetKey removes a saved key by name from the active profile. +func ForgetKey(profileOverride, name string) (string, bool, error) { + var profileName string + var removed bool + err := config.Update(func(f *config.File) error { + var p config.Profile + var resolveErr error + profileName, p, resolveErr = resolveProfileIn(f, profileOverride) + if resolveErr != nil { + return resolveErr + } + removed = p.RemoveObjectStorageKey(name) + if removed { + f.SetProfile(profileName, p) + } + return nil + }) + if err != nil { + return profileName, false, err + } + return profileName, removed, nil +} + +// ForgetKeyByID removes every saved key with the given access key ID. +func ForgetKeyByID(profileOverride, accessKeyID string) (string, []string, error) { + var profileName string + var removed []string + err := config.Update(func(f *config.File) error { + var p config.Profile + var resolveErr error + profileName, p, resolveErr = resolveProfileIn(f, profileOverride) + if resolveErr != nil { + return resolveErr + } + removed = nil + for name, k := range p.ObjectStorageKeys() { + if k.AccessKeyID == accessKeyID { + p.RemoveObjectStorageKey(name) + removed = append(removed, name) + } + } + if len(removed) > 0 { + f.SetProfile(profileName, p) + } + return nil + }) + if err != nil { + return profileName, nil, err + } + sort.Strings(removed) + return profileName, removed, nil +} diff --git a/internal/objectstorage/creds_test.go b/internal/objectstorage/creds_test.go new file mode 100644 index 0000000..b67ce09 --- /dev/null +++ b/internal/objectstorage/creds_test.go @@ -0,0 +1,169 @@ +package objectstorage + +import ( + "os" + "strings" + "testing" + "time" + + "github.com/latitudesh/lsh/internal/config" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func TestEnvCredential(t *testing.T) { + t.Setenv(EnvAccessKeyID, "") + t.Setenv(EnvSecretAccessKey, "") + t.Setenv(EnvUseAWSEnv, "") + t.Setenv("AWS_ACCESS_KEY_ID", "AKIAREAL") + t.Setenv("AWS_SECRET_ACCESS_KEY", "realsecret") + t.Setenv("AWS_SESSION_TOKEN", "") + + if _, ok, err := EnvCredential(); ok || err != nil { + t.Fatalf("AWS_* must be ignored without opt-in: ok=%v err=%v", ok, err) + } + + t.Setenv(EnvUseAWSEnv, "1") + c, ok, err := EnvCredential() + if !ok || err != nil || c.AccessKeyID != "AKIAREAL" || c.Secret() != "realsecret" || !c.FromEnv { + t.Fatalf("opt-in fallback failed: %+v ok=%v err=%v", c, ok, err) + } + t.Setenv("AWS_SESSION_TOKEN", "sts") + if _, ok, _ := EnvCredential(); ok { + t.Fatal("STS credentials must never be used as Latitude keys") + } + + t.Setenv(EnvAccessKeyID, "LSHID") + if _, _, err := EnvCredential(); err == nil || exitcode.Of(err) != exitcode.Credentials { + t.Fatalf("half-set pair must fail with exit %d, got %v", exitcode.Credentials, err) + } + t.Setenv(EnvSecretAccessKey, "LSHSECRET") + c, ok, err = EnvCredential() + if !ok || err != nil || c.AccessKeyID != "LSHID" || c.Secret() != "LSHSECRET" { + t.Fatalf("LSH_S3_* must win: %+v", c) + } + if strings.Contains(c.String(), "LSHSECRET") { + t.Fatal("String() leaked the secret") + } +} + +func TestSelectKeyLeastPrivilege(t *testing.T) { + b := &Bucket{ID: "bkt_1", StorageClass: config.ScopeFullAccess, Site: "DAL", ProjectID: "proj_1"} + b.StorageClass = "standard" + now := time.Now() + keys := map[string]config.StoredAccessKey{ + "full-std": {AccessKeyID: "F", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now.Add(-time.Hour)}, + "ro-1": {AccessKeyID: "R", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": config.PermissionReadOnly}, CreatedAt: now}, + "rw-1": {AccessKeyID: "W", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_1": config.PermissionRW}, CreatedAt: now.Add(-2 * time.Hour)}, + "other": {AccessKeyID: "O", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeLimitedAccess, Buckets: map[string]string{"bkt_9": config.PermissionRW}, CreatedAt: now}, + "hp": {AccessKeyID: "H", StorageClass: "high_performance", Site: "DAL", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now}, + "unknown": {AccessKeyID: "U", StorageClass: "standard", Scope: config.ScopeUnknown, CreatedAt: now}, + } + + name, _, ok := SelectKey(keys, "", b, false) + if !ok || name != "rw-1" { + t.Fatalf("read: want rw-1 (limited rw ranks first), got %q ok=%v", name, ok) + } + name, _, ok = SelectKey(keys, "", b, true) + if !ok || name != "rw-1" { + t.Fatalf("write: want rw-1, got %q", name) + } + + delete(keys, "rw-1") + name, _, _ = SelectKey(keys, "", b, false) + if name != "ro-1" { + t.Fatalf("read without rw key: want ro-1 over fullaccess, got %q", name) + } + name, _, _ = SelectKey(keys, "", b, true) + if name != "full-std" { + t.Fatalf("write must skip readonly and use fullaccess, got %q", name) + } + + delete(keys, "full-std") + if _, _, ok := SelectKey(keys, "", b, true); ok { + t.Fatal("no writable key must yield no selection") + } + + // Fullaccess ties: default key wins, then newest. + keys = map[string]config.StoredAccessKey{ + "old": {AccessKeyID: "A", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now.Add(-time.Hour)}, + "new": {AccessKeyID: "B", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now}, + "dflt": {AccessKeyID: "C", StorageClass: "standard", ProjectID: "proj_1", Scope: config.ScopeFullAccess, CreatedAt: now.Add(-2 * time.Hour)}, + } + if name, _, _ := SelectKey(keys, "dflt", b, false); name != "dflt" { + t.Fatalf("default key must break ties, got %q", name) + } + if name, _, _ := SelectKey(keys, "", b, false); name != "new" { + t.Fatalf("newest key must break ties, got %q", name) + } + + // A high_performance bucket in another site is not covered by a DAL key. + hp := &Bucket{ID: "bkt_2", StorageClass: "high_performance", Site: "NYC", ProjectID: "proj_1"} + keys = map[string]config.StoredAccessKey{"hp-dal": {AccessKeyID: "H", StorageClass: "high_performance", Site: "DAL", ProjectID: "proj_1", Scope: config.ScopeFullAccess}} + if _, _, ok := SelectKey(keys, "", hp, false); ok { + t.Fatal("DAL key must not cover NYC high_performance bucket") + } + hp.Site = "dal" + if _, _, ok := SelectKey(keys, "", hp, false); !ok { + t.Fatal("site comparison must be case-insensitive") + } +} + +func TestResolveCredentialEndpointOverrideRequiresEnv(t *testing.T) { + t.Setenv(EnvAccessKeyID, "") + t.Setenv(EnvSecretAccessKey, "") + t.Setenv(EnvUseAWSEnv, "") + os.Unsetenv("AWS_ACCESS_KEY_ID") + b := &Bucket{BucketName: "raw", Endpoint: "https://s3.example.test", EndpointOverride: true} + _, err := ResolveCredential(b, CredentialOptions{}) + if err == nil || exitcode.Of(err) != exitcode.Credentials { + t.Fatalf("saved keys must not be used with --endpoint-url; got %v", err) + } + if !strings.Contains(err.Error(), EnvAccessKeyID) { + t.Errorf("error should tell the user which variables to set: %v", err) + } +} + +// TestExplicitKeyChecksCompatibility covers the --access-key path: Covers() is +// true for every bucket on a fullaccess key, so without an explicit +// compatibility check a key from another backend, site or project would be +// sent to the bucket and fail at the server instead of locally. +func TestExplicitKeyChecksCompatibility(t *testing.T) { + hp := &Bucket{ID: "bkt_1", Name: "fast", StorageClass: ClassHighPerformance, Site: "TYO4", ProjectID: "proj_1"} + full := func(class, site, project string) config.StoredAccessKey { + return config.StoredAccessKey{AccessKeyID: "AK", StorageClass: class, Site: site, ProjectID: project, Scope: config.ScopeFullAccess} + } + cases := []struct { + name string + key config.StoredAccessKey + want string + }{ + {"other storage class", full(ClassStandard, "TYO4", "proj_1"), "do not share credentials"}, + {"other site", full(ClassHighPerformance, "DAL", "proj_1"), "only works in its own site"}, + {"other project", full(ClassHighPerformance, "TYO4", "proj_2"), "belongs to project"}, + } + for _, c := range cases { + err := explainIncompatibleKey("ci-key", c.key, hp) + if err == nil { + t.Errorf("%s: expected the selection to be rejected", c.name) + continue + } + if code := exitcode.Of(err); code != exitcode.Usage { + t.Errorf("%s: exit code = %d, want %d", c.name, code, exitcode.Usage) + } + if !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: error %q should explain the mismatch (%q)", c.name, err, c.want) + } + } + + // A compatible key passes, and partial metadata is not judged: an imported + // key with no class/site/project stays usable. + for _, ok := range []config.StoredAccessKey{ + full(ClassHighPerformance, "tyo4", "proj_1"), + full("", "", ""), + {AccessKeyID: "AK", Scope: config.ScopeUnknown}, + } { + if err := explainIncompatibleKey("ci-key", ok, hp); err != nil { + t.Errorf("key %+v must be accepted, got %v", ok, err) + } + } +} diff --git a/internal/objectstorage/errors.go b/internal/objectstorage/errors.go new file mode 100644 index 0000000..fb5e99c --- /dev/null +++ b/internal/objectstorage/errors.go @@ -0,0 +1,327 @@ +package objectstorage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "strconv" + "strings" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/tui" + "github.com/minio/minio-go/v7" +) + +// Humanize rewrites an S3 or API error into an actionable message with the +// right exit code. Errors that already carry an exit code pass through. +func Humanize(err error, b *Bucket, c *Credential) error { + if err == nil { + return nil + } + var already *exitcode.Error + if errors.As(err, &already) { + return err + } + if errors.Is(err, context.Canceled) { + return exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + if errors.Is(err, context.DeadlineExceeded) { + return exitcode.Errorf(exitcode.Generic, "timed out: %v", err) + } + + bucketName := "" + if b != nil { + bucketName = b.Display() + } + cred := "the credential" + if c != nil { + if b != nil { + cred = c.Describe(b.ID) + } else { + cred = c.Source + } + } + + if resp := minio.ToErrorResponse(err); resp.Code != "" || resp.StatusCode != 0 { + switch resp.Code { + case "NoSuchBucket": + endpoint := "" + backend := resp.BucketName + if b != nil { + endpoint = b.Endpoint + backend = b.BucketName + } + return exitcode.Errorf(exitcode.NotFound, "bucket %s (backend name %q) was not found at %s; it may still be provisioning — check 'lsh s3 get %s'", bucketName, backend, endpoint, displayOrBackend(b)) + case "NoSuchKey", "NotFound": + key := resp.Key + if key == "" { + return exitcode.Errorf(exitcode.NotFound, "object not found in bucket %s", bucketName) + } + return exitcode.Errorf(exitcode.NotFound, "object %q not found in bucket %s", key, bucketName) + case "AccessDenied", "AllAccessDisabled": + hint := "" + if b != nil { + hint = fmt.Sprintf("; use a key with rw permission (lsh s3 access-keys list) or create one: lsh s3 access-keys create --bucket %s --save", displayOrBackend(b)) + } + return exitcode.Errorf(exitcode.Permission, "access denied to %s using %s%s", bucketName, cred, hint) + case "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidToken", "ExpiredToken": + hint := "" + if c != nil && !c.FromEnv && c.Name != "" { + hint = fmt.Sprintf("; if it was deleted or rotated, run 'lsh s3 access-keys forget %s' and create a new one", c.Name) + } + return exitcode.Errorf(exitcode.Credentials, "the backend rejected %s (%s)%s", cred, resp.Code, hint) + case "AuthorizationHeaderMalformed": + expected := resp.Region + signed := "" + if b != nil { + signed = b.SigningRegion + } + if expected != "" { + return exitcode.Errorf(exitcode.Credentials, "signing region mismatch: the backend expects %q but the request was signed for %q; retry with --signing-region %s (or %s=%s)", expected, signed, expected, EnvSigningRegion, expected) + } + return exitcode.Errorf(exitcode.Credentials, "signing region mismatch (signed for %q); retry with --signing-region ", signed) + case "BucketNotEmpty": + return exitcode.Errorf(exitcode.Refused, "bucket %s is not empty; re-run with --force to delete its objects first", bucketName) + case "BucketAlreadyOwnedByYou", "BucketAlreadyExists": + return exitcode.Errorf(exitcode.Usage, "bucket %s already exists", bucketName) + case "EntityTooLarge": + return exitcode.Errorf(exitcode.Generic, "object exceeds the backend size limit: %s", resp.Message) + case "RequestTimeTooSkewed": + return exitcode.Errorf(exitcode.Generic, "the local clock is off by more than 15 minutes; fix the system time and retry") + case "InvalidArgument", "InvalidRequest", "XMinioInvalidObjectName", "KeyTooLongError", "InvalidBucketName": + return exitcode.Errorf(exitcode.Usage, "%s: %s", resp.Code, resp.Message) + case "ObjectLocked", "InvalidRetentionPeriod", "ObjectLockConfigurationNotFoundError": + return exitcode.Errorf(exitcode.Refused, "%s: %s", resp.Code, resp.Message) + case "PreconditionFailed": + return exitcode.Errorf(exitcode.Generic, "precondition failed: %s", resp.Message) + case "SlowDown", "ServiceUnavailable", "InternalError": + return exitcode.Errorf(exitcode.Generic, "the backend is busy (%s); retry in a moment", resp.Code) + } + if resp.StatusCode == 403 { + return exitcode.Errorf(exitcode.Permission, "access denied to %s using %s", bucketName, cred) + } + if resp.StatusCode == 404 { + return exitcode.Errorf(exitcode.NotFound, "not found: %s", firstNonEmpty(resp.Message, resp.Code, "404")) + } + msg := firstNonEmpty(resp.Message, resp.Code) + if resp.Code != "" && resp.Message != "" { + msg = resp.Code + ": " + resp.Message + } + return exitcode.Errorf(exitcode.Generic, "%s", Redact(msg)) + } + + if e := humanizeLatitudeError(err, ""); e != nil { + return e + } + + var urlErr *url.Error + var netErr net.Error + var opErr *net.OpError + if errors.As(err, &urlErr) || errors.As(err, &netErr) || errors.As(err, &opErr) { + target := "" + if b != nil { + target = b.Endpoint + } + return exitcode.Errorf(exitcode.Generic, "could not reach %s: %v", target, Redact(err.Error())) + } + return exitcode.New(exitcode.Generic, errors.New(Redact(err.Error()))) +} + +func displayOrBackend(b *Bucket) string { + if b == nil { + return "" + } + if b.Name != "" { + return b.Name + } + return b.BucketName +} + +// humanizeAPIError maps Latitude API errors for control-plane calls. +func humanizeAPIError(err error, what string) error { + if e := humanizeLatitudeError(err, what); e != nil { + return e + } + // Same interruption contract as Humanize: the exit code must not depend on + // whether the signal landed during an API call or an S3 call. + if errors.Is(err, context.Canceled) { + return exitcode.Errorf(exitcode.Interrupted, "interrupted") + } + if errors.Is(err, context.DeadlineExceeded) { + return exitcode.Errorf(exitcode.Generic, "timed out: %v", err) + } + var already *exitcode.Error + if errors.As(err, &already) { + return err + } + return exitcode.New(exitcode.Generic, err) +} + +// HumanizeAPI is the exported form of humanizeAPIError for commands that +// only talk to the Latitude API (access keys, lifecycle, metrics, usage). +func HumanizeAPI(err error, what string) error { + if err == nil { + return nil + } + return humanizeAPIError(err, what) +} + +// humanizeLatitudeError recognizes the two error shapes the SDK produces: +// *components.APIError (unmodeled statuses; carries the HTTP status and raw +// body) and *components.ErrorObject (JSON:API errors for the statuses the +// spec models — 403/404/422/500 — where the status is a string per entry). +// It returns nil for any other error. +func humanizeLatitudeError(err error, what string) error { + var apiErr *components.APIError + if errors.As(err, &apiErr) { + return humanizeAPIStatus(apiErr.StatusCode, []byte(apiErr.Body), what) + } + var eo *components.ErrorObject + if errors.As(err, &eo) { + status := 0 + var parts []string + for _, e := range eo.Errors { + if status == 0 && e.Status != nil { + if n, convErr := strconv.Atoi(strings.TrimSpace(*e.Status)); convErr == nil { + status = n + } + } + msg := firstNonEmpty(str(e.Detail), str(e.Title), str(e.Code)) + if msg == "" { + continue + } + if e.Source != nil && e.Source.Pointer != nil && *e.Source.Pointer != "" { + msg = strings.TrimPrefix(*e.Source.Pointer, "/data/attributes/") + " " + msg + } + parts = append(parts, msg) + } + detail := strings.Join(parts, "; ") + if status == 0 { + switch { + case strings.Contains(strings.ToLower(detail), "not found"): + status = 404 + case strings.Contains(strings.ToLower(detail), "forbidden"), strings.Contains(strings.ToLower(detail), "permission"): + status = 403 + default: + status = 422 + } + } + body, _ := json.Marshal(map[string]interface{}{"errors": []map[string]string{{"detail": detail}}}) + return humanizeAPIStatus(status, body, what) + } + return nil +} + +// humanizeAPIStatus turns an HTTP status + JSON:API body into an exit-coded +// error with the API's own detail message when present. +func humanizeAPIStatus(status int, body []byte, what string) error { + detail := apiDetail(body) + switch status { + case 401: + return exitcode.Errorf(exitcode.Credentials, "your API token is invalid or revoked; run 'lsh login' to sign in again") + case 403: + return exitcode.Errorf(exitcode.Permission, "your API token does not have permission for this action%s", suffix(detail)) + case 404: + if what != "" { + return exitcode.Errorf(exitcode.NotFound, "%s not found%s", what, suffix(detail)) + } + return exitcode.Errorf(exitcode.NotFound, "not found%s", suffix(detail)) + case 409: + if strings.Contains(strings.ToLower(detail), "not empty") { + return exitcode.Errorf(exitcode.Refused, "%s", detail) + } + return exitcode.Errorf(exitcode.Usage, "conflict%s", suffix(detail)) + case 422, 400: + return exitcode.Errorf(exitcode.Usage, "the API rejected the request%s", suffix(detail)) + case 429: + return exitcode.Errorf(exitcode.Generic, "rate limited by the API; retry in a moment") + } + if status >= 500 { + return exitcode.Errorf(exitcode.Generic, "the API returned %d%s", status, suffix(detail)) + } + return exitcode.Errorf(exitcode.Generic, "API error %d%s", status, suffix(detail)) +} + +func suffix(detail string) string { + if detail == "" { + return "" + } + return ": " + detail +} + +// apiDetail extracts the most useful text from a JSON:API error body. +func apiDetail(body []byte) string { + if len(body) == 0 { + return "" + } + var env struct { + Errors []struct { + Title string `json:"title"` + Detail string `json:"detail"` + Code string `json:"code"` + Source struct { + Pointer string `json:"pointer"` + } `json:"source"` + } `json:"errors"` + Error string `json:"error"` + Message string `json:"message"` + } + if err := json.Unmarshal(body, &env); err != nil { + s := strings.TrimSpace(string(body)) + if len(s) > 200 { + s = s[:200] + "…" + } + return s + } + var parts []string + for _, e := range env.Errors { + msg := firstNonEmpty(e.Detail, e.Title, e.Code) + if msg == "" { + continue + } + if e.Source.Pointer != "" { + msg = strings.TrimPrefix(e.Source.Pointer, "/data/attributes/") + " " + msg + } + parts = append(parts, msg) + } + if len(parts) > 0 { + return strings.Join(parts, "; ") + } + return firstNonEmpty(env.Message, env.Error) +} + +// PrintedError marks an error that was already written to stderr by +// PrintError, so the command wrapper in cmd/s3 does not print it a second +// time. It unwraps to the original error (exit codes are preserved). +type PrintedError struct{ Err error } + +func (p *PrintedError) Error() string { return p.Err.Error() } + +// Unwrap exposes the wrapped error to errors.Is/As. +func (p *PrintedError) Unwrap() error { return p.Err } + +// PrintError writes a humanized error to stderr in the CLI's error style and +// returns it wrapped in PrintedError (with its exit code) so RunE can +// propagate it without cobra echoing it again. +func PrintError(err error) error { + if err == nil { + return nil + } + var already *PrintedError + if errors.As(err, &already) { + return err + } + fmt.Fprintln(os.Stderr, tui.ErrorStyle.Render("✗ Error: ")+err.Error()) + return &PrintedError{Err: err} +} + +// IsPrinted reports whether err was already printed by PrintError. +func IsPrinted(err error) bool { + var p *PrintedError + return errors.As(err, &p) +} diff --git a/internal/objectstorage/filter.go b/internal/objectstorage/filter.go new file mode 100644 index 0000000..8d3e9a6 --- /dev/null +++ b/internal/objectstorage/filter.go @@ -0,0 +1,120 @@ +package objectstorage + +import ( + "fmt" + "regexp" + "strings" + + "github.com/spf13/pflag" +) + +// Filters implements the aws s3 --exclude/--include semantics: everything is +// included by default, rules apply in the order given on the command line and +// the last matching rule wins. Patterns are shell globs where `*` also +// matches `/` (fnmatch-style), evaluated against the path relative to the +// source directory or prefix. +type Filters struct { + rules []filterRule +} + +type filterRule struct { + include bool + pattern string + re *regexp.Regexp +} + +// Add appends a rule; returns an error for an invalid pattern. +func (f *Filters) Add(include bool, pattern string) error { + re, err := globToRegexp(pattern) + if err != nil { + return err + } + f.rules = append(f.rules, filterRule{include: include, pattern: pattern, re: re}) + return nil +} + +// Empty reports whether no rules were given. +func (f *Filters) Empty() bool { return f == nil || len(f.rules) == 0 } + +// Include decides whether relPath passes the filters. +func (f *Filters) Include(relPath string) bool { + if f == nil { + return true + } + include := true + for _, r := range f.rules { + if r.re.MatchString(relPath) { + include = r.include + } + } + return include +} + +// Describe lists the rules (for --dry-run/--debug). +func (f *Filters) Describe() string { + if f.Empty() { + return "(no filters)" + } + parts := make([]string, 0, len(f.rules)) + for _, r := range f.rules { + kind := "exclude" + if r.include { + kind = "include" + } + parts = append(parts, fmt.Sprintf("--%s %q", kind, r.pattern)) + } + return strings.Join(parts, " ") +} + +// globToRegexp translates a fnmatch-style pattern into an anchored regexp. +func globToRegexp(pattern string) (*regexp.Regexp, error) { + var sb strings.Builder + sb.WriteString("^") + for i := 0; i < len(pattern); i++ { + c := pattern[i] + switch c { + case '*': + sb.WriteString(".*") + case '?': + sb.WriteString(".") + case '[': + j := strings.IndexByte(pattern[i:], ']') + if j < 0 { + sb.WriteString(regexp.QuoteMeta(string(c))) + continue + } + class := pattern[i+1 : i+j] + if strings.HasPrefix(class, "!") { + class = "^" + class[1:] + } + sb.WriteString("[" + strings.ReplaceAll(class, `\`, `\\`) + "]") + i += j + default: + sb.WriteString(regexp.QuoteMeta(string(c))) + } + } + sb.WriteString("$") + re, err := regexp.Compile(sb.String()) + if err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", pattern, err) + } + return re, nil +} + +// FilterFlags registers --exclude and --include on a flag set, preserving the +// relative order in which they appear (cobra's per-flag slices cannot). +func FilterFlags(fs *pflag.FlagSet, f *Filters) { + fs.Var(&filterValue{f: f, include: false}, "exclude", "exclude paths matching this pattern (repeatable; rules apply in order, last match wins)") + fs.Var(&filterValue{f: f, include: true}, "include", "re-include paths matching this pattern after an --exclude (repeatable)") +} + +type filterValue struct { + f *Filters + include bool +} + +func (v *filterValue) String() string { return "" } +func (v *filterValue) Type() string { return "pattern" } +func (v *filterValue) Set(s string) error { + return v.f.Add(v.include, s) +} diff --git a/internal/objectstorage/filter_test.go b/internal/objectstorage/filter_test.go new file mode 100644 index 0000000..e6f3d84 --- /dev/null +++ b/internal/objectstorage/filter_test.go @@ -0,0 +1,64 @@ +package objectstorage + +import ( + "testing" + + "github.com/spf13/pflag" +) + +// The three examples from the aws user guide ("Use of Exclude and Include +// Filters"): include alone is a no-op, exclude-all then include narrows, +// order matters. +func TestFiltersAWSSemantics(t *testing.T) { + var f Filters + if !f.Include("anything.txt") { + t.Fatal("no rules must include everything") + } + + var onlyInclude Filters + _ = onlyInclude.Add(true, "*.log") + if !onlyInclude.Include("a.txt") { + t.Error("--include alone must not exclude other files (aws semantics)") + } + + var narrow Filters + _ = narrow.Add(false, "*") + _ = narrow.Add(true, "*.log") + if narrow.Include("a.txt") || !narrow.Include("dir/b.log") { + t.Error("--exclude '*' --include '*.log' must include only .log files") + } + + var reversed Filters + _ = reversed.Add(true, "*.log") + _ = reversed.Add(false, "*") + if reversed.Include("b.log") { + t.Error("last matching rule wins: --include then --exclude '*' excludes everything") + } + + var dirOnly Filters + _ = dirOnly.Add(false, "*") + _ = dirOnly.Add(true, "2026/09/*") + if !dirOnly.Include("2026/09/dump.sql") || dirOnly.Include("2026/08/dump.sql") { + t.Error("star must match across slashes like fnmatch in aws") + } +} + +func TestFilterFlagsPreserveOrder(t *testing.T) { + var f Filters + fs := pflag.NewFlagSet("t", pflag.ContinueOnError) + FilterFlags(fs, &f) + if err := fs.Parse([]string{"--exclude", "*", "--include", "*.log", "--exclude", "tmp/*"}); err != nil { + t.Fatal(err) + } + if !f.Include("a.log") || f.Include("tmp/a.log") || f.Include("a.txt") { + t.Errorf("order not preserved: %s", f.Describe()) + } +} + +func TestGlobToRegexpInvalid(t *testing.T) { + var f Filters + if err := f.Add(true, "[a-"); err != nil { + // An unterminated class is treated literally, not as an error. + t.Errorf("unterminated class should be literal, got %v", err) + } +} diff --git a/internal/objectstorage/output.go b/internal/objectstorage/output.go new file mode 100644 index 0000000..5d0bcdd --- /dev/null +++ b/internal/objectstorage/output.go @@ -0,0 +1,250 @@ +package objectstorage + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/latitudesh/lsh/internal/output/table" + "github.com/latitudesh/lsh/internal/renderer" + "github.com/minio/minio-go/v7" +) + +// Object is a listed or inspected S3 object (also used for versions). +type Object struct { + Type string `json:"type"` + Key string `json:"key"` + Size int64 `json:"size"` + LastModified time.Time `json:"last_modified"` + ETag string `json:"etag,omitempty"` + ContentType string `json:"content_type,omitempty"` + StorageClass string `json:"storage_class,omitempty"` + VersionID string `json:"version_id,omitempty"` + IsLatest *bool `json:"is_latest,omitempty"` + IsDeleteMarker bool `json:"is_delete_marker,omitempty"` + // Metadata holds x-amz-meta-* values (stat only). + Metadata map[string]string `json:"metadata,omitempty"` +} + +// ObjectFromInfo converts a minio listing entry. +func ObjectFromInfo(info minio.ObjectInfo, withVersions bool) Object { + o := Object{ + Type: "object", + Key: info.Key, + Size: info.Size, + LastModified: info.LastModified, + ETag: strings.Trim(info.ETag, `"`), + ContentType: info.ContentType, + StorageClass: info.StorageClass, + IsDeleteMarker: info.IsDeleteMarker, + } + // VersionID is copied whenever the backend returns one; IsLatest is only + // meaningful in a versions listing (HEAD responses never carry it). + o.VersionID = info.VersionID + if withVersions { + latest := info.IsLatest + o.IsLatest = &latest + } + if info.IsDeleteMarker { + o.Type = "delete_marker" + } + return o +} + +func (o Object) TableRow() table.Row { + row := table.Row{ + "type": {Label: "Type", Value: strings.ToUpper(shortType(o.Type))}, + "key": {Label: "Key", Value: o.Key, MaxLength: 80}, + "size": {Label: "Size", Value: HumanSize(o.Size)}, + "last_modified": {Label: "Last Modified", Value: FormatTime(o.LastModified)}, + } + if o.VersionID != "" { + row["version_id"] = table.Cell{Label: "Version ID", Value: o.VersionID} + } + if o.ContentType != "" { + row["content_type"] = table.Cell{Label: "Content Type", Value: o.ContentType} + } + if o.ETag != "" { + row["etag"] = table.Cell{Label: "ETag", Value: o.ETag} + } + return row +} + +func shortType(t string) string { + switch t { + case "prefix": + return "PRE" + case "delete_marker": + return "DEL" + default: + return "OBJ" + } +} + +// Prefix is a common prefix ("directory") in a non-recursive listing. +type Prefix struct { + Type string `json:"type"` + Prefix string `json:"prefix"` +} + +// NewPrefix builds a Prefix entry. +func NewPrefix(p string) Prefix { return Prefix{Type: "prefix", Prefix: p} } + +func (p Prefix) TableRow() table.Row { + return table.Row{ + "type": {Label: "Type", Value: "PRE"}, + "key": {Label: "Key", Value: p.Prefix, MaxLength: 80}, + "size": {Label: "Size", Value: ""}, + "last_modified": {Label: "Last Modified", Value: ""}, + } +} + +// TransferResult is one cp/mv operation (or its dry-run plan). +type TransferResult struct { + Op string `json:"op"` // upload | download | copy | move + Source string `json:"source"` + Destination string `json:"destination"` + Size int64 `json:"size"` + ETag string `json:"etag,omitempty"` + ContentType string `json:"content_type,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + Error string `json:"error,omitempty"` +} + +func (t TransferResult) TableRow() table.Row { + status := "ok" + if t.DryRun { + status = "dryrun" + } + if t.Error != "" { + status = "error" + } + return table.Row{ + "op": {Label: "Op", Value: t.Op}, + "source": {Label: "Source", Value: t.Source, MaxLength: 60}, + "destination": {Label: "Destination", Value: t.Destination, MaxLength: 60}, + "size": {Label: "Size", Value: HumanSize(t.Size)}, + "status": {Label: "Status", Value: status}, + } +} + +// HumanLine renders the aws-style line: `upload: ./a to s3://b/a`. +func (t TransferResult) HumanLine() string { + prefix := "" + if t.DryRun { + prefix = "(dryrun) " + } + if t.Error != "" { + return fmt.Sprintf("%s%s failed: %s to %s: %s", prefix, t.Op, t.Source, t.Destination, t.Error) + } + return fmt.Sprintf("%s%s: %s to %s", prefix, t.Op, t.Source, t.Destination) +} + +// DeleteResult is one rm/rb deletion (or its dry-run plan). +type DeleteResult struct { + Bucket string `json:"bucket"` + Key string `json:"key,omitempty"` + VersionID string `json:"version_id,omitempty"` + Deleted bool `json:"deleted"` + DryRun bool `json:"dry_run,omitempty"` + Error string `json:"error,omitempty"` +} + +func (d DeleteResult) TableRow() table.Row { + status := "deleted" + if d.DryRun { + status = "dryrun" + } + if d.Error != "" { + status = "error: " + d.Error + } + return table.Row{ + "bucket": {Label: "Bucket", Value: d.Bucket}, + "key": {Label: "Key", Value: d.Key, MaxLength: 80}, + "status": {Label: "Status", Value: status}, + } +} + +// HumanLine renders `delete: s3://b/k` (or `remove_bucket: s3://b`). +func (d DeleteResult) HumanLine() string { + prefix := "" + if d.DryRun { + prefix = "(dryrun) " + } + target := "s3://" + d.Bucket + verb := "remove_bucket" + if d.Key != "" { + target += "/" + d.Key + verb = "delete" + } + if d.VersionID != "" { + target += " (version " + d.VersionID + ")" + } + if d.Error != "" { + return fmt.Sprintf("%s%s failed: %s: %s", prefix, verb, target, d.Error) + } + return fmt.Sprintf("%s%s: %s", prefix, verb, target) +} + +// PresignResult is the output of `presign`. +type PresignResult struct { + URL string `json:"url"` + Method string `json:"method"` + ExpiresAt time.Time `json:"expires_at"` +} + +func (p PresignResult) TableRow() table.Row { + return table.Row{ + "url": {Label: "URL", Value: p.URL}, + "method": {Label: "Method", Value: p.Method}, + "expires_at": {Label: "Expires At", Value: FormatTime(p.ExpiresAt)}, + } +} + +// AsResponseData adapts a slice of any ResponseData implementation. +func AsResponseData[T renderer.ResponseData](items []T) []renderer.ResponseData { + out := make([]renderer.ResponseData, 0, len(items)) + for _, it := range items { + out = append(out, it) + } + return out +} + +// FormatTime renders timestamps the way `aws s3 ls` does, in local time. +func FormatTime(t time.Time) string { + if t.IsZero() { + return "" + } + return t.Local().Format("2006-01-02 15:04:05") +} + +// HumanSize renders sizes exactly like `aws s3 ls --human-readable`: +// "1 Byte", "12 Bytes", then "1.0 KiB", "3.4 MiB"… promoting the unit when the +// rounded value would reach 1024 (so 1048575 bytes is "1.0 MiB"). +func HumanSize(n int64) string { + if n == 1 { + return "1 Byte" + } + if n < 1024 { + return fmt.Sprintf("%d Bytes", n) + } + units := []string{"KiB", "MiB", "GiB", "TiB", "PiB", "EiB"} + unit := 1024.0 + for i, u := range units { + unit *= 1024 + if math.Round(float64(n)/unit*1024) < 1024 || i == len(units)-1 { + return fmt.Sprintf("%.1f %s", 1024*float64(n)/unit, u) + } + } + return fmt.Sprintf("%d Bytes", n) +} + +// LsSummary renders the --summarize footer for object listings. +func LsSummary(count int, bytes int64, human bool) string { + size := fmt.Sprintf("%d", bytes) + if human { + size = HumanSize(bytes) + } + return fmt.Sprintf("\nTotal Objects: %d\n Total Size: %s", count, size) +} diff --git a/internal/objectstorage/prompt.go b/internal/objectstorage/prompt.go new file mode 100644 index 0000000..982fca0 --- /dev/null +++ b/internal/objectstorage/prompt.go @@ -0,0 +1,228 @@ +package objectstorage + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/tui" + "github.com/spf13/cobra" + "golang.org/x/term" +) + +// stdin/stderr are package variables so tests can substitute them. +var ( + promptIn io.Reader = os.Stdin + promptOut io.Writer = os.Stderr + isTTY = func() bool { + return term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stderr.Fd())) + } +) + +// CanPrompt reports whether the command may ask the user something: --no-input +// was not given, and both stdin and stderr are terminals. Prompts are written +// to stderr so stdout stays clean for pipes. +func CanPrompt(cmd *cobra.Command) bool { + if cmd != nil { + if noInput, err := cmd.Flags().GetBool("no-input"); err == nil && noInput { + return false + } + } + return isTTY() +} + +// Confirm asks a yes/no question. In a real terminal it uses the shared +// Bubble Tea confirm widget (the same y/n prompt as the rest of the CLI); +// otherwise (tests, redirected streams) it falls back to a plain [y/N] line so +// the flow stays scriptable. EOF counts as No. +func Confirm(question string) (bool, error) { + if isTTY() { + return tui.RunConfirm(question) + } + return confirmLine(question) +} + +// confirmLine is the non-interactive fallback of Confirm. +func confirmLine(question string) (bool, error) { + fmt.Fprintf(promptOut, "? %s [y/N] ", question) + line, err := readLine() + if err != nil && line == "" { + fmt.Fprintln(promptOut) + return false, nil + } + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + return true, nil + } + return false, nil +} + +// Choose lets the user pick one option and returns its index (0-based), or -1 +// when the selection is cancelled. In a real terminal it uses the shared +// Bubble Tea list (arrow keys, filtering — the same picker as the rest of the +// CLI); otherwise it falls back to a numbered menu read from stdin. +func Choose(question string, options []string, defaultIndex int) (int, error) { + if isTTY() { + choice, err := tui.RunList(question, options, nil) + if err != nil { + // Cancelled (esc/ctrl+c) or no selection. + return -1, nil + } + for i, opt := range options { + if opt == choice { + return i, nil + } + } + return -1, nil + } + return chooseLine(question, options, defaultIndex) +} + +// chooseLine is the non-interactive fallback of Choose. +func chooseLine(question string, options []string, defaultIndex int) (int, error) { + fmt.Fprintln(promptOut, "? "+question) + for i, opt := range options { + marker := " " + if i == defaultIndex { + marker = "*" + } + fmt.Fprintf(promptOut, " %s[%d] %s\n", marker, i+1, opt) + } + fmt.Fprintf(promptOut, "> ") + line, err := readLine() + line = strings.TrimSpace(line) + if err != nil && line == "" { + fmt.Fprintln(promptOut) + return -1, nil + } + if line == "" { + return defaultIndex, nil + } + n, convErr := strconv.Atoi(line) + if convErr != nil || n < 1 || n > len(options) { + return -1, nil + } + return n - 1, nil +} + +// ReadSecret reads a secret from stdin without echo when stdin is a terminal, +// or a single line otherwise (pipes, heredocs). cmd is used to honour +// --no-input: a session that forbids prompts must fail instead of blocking on +// a terminal read. Piping the value on stdin keeps working either way. +func ReadSecret(cmd *cobra.Command, prompt string) (string, error) { + if f, ok := promptIn.(*os.File); ok && term.IsTerminal(int(f.Fd())) && !CanPrompt(cmd) { + return "", exitcode.Errorf(exitcode.Refused, "a secret is required but prompts are disabled; pipe it on stdin instead: echo \"$SECRET\" | lsh ...") + } + fmt.Fprint(promptOut, prompt) + if f, ok := promptIn.(*os.File); ok && term.IsTerminal(int(f.Fd())) { + b, err := term.ReadPassword(int(f.Fd())) + fmt.Fprintln(promptOut) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil + } + line, err := readLine() + if err != nil && line == "" { + return "", err + } + return strings.TrimSpace(line), nil +} + +func readLine() (string, error) { + r := bufio.NewReader(promptIn) + line, err := r.ReadString('\n') + return strings.TrimRight(line, "\r\n"), err +} + +// ConfirmOrRefuse implements the confirmation contract of destructive +// commands: --yes skips the prompt; otherwise an interactive session is asked +// and a non-interactive one fails fast with exit 7 instead of hanging. +func ConfirmOrRefuse(cmd *cobra.Command, yes bool, question string) error { + if yes { + return nil + } + if !CanPrompt(cmd) { + return exitcode.Errorf(exitcode.Refused, "%s — refusing to continue without --yes in a non-interactive session", question) + } + ok, err := Confirm(question) + if err != nil { + return err + } + if !ok { + return exitcode.Errorf(exitcode.Refused, "cancelled") + } + return nil +} + +// Warnf prints a warning line to stderr. +func Warnf(format string, a ...interface{}) { + fmt.Fprintf(os.Stderr, "warning: "+format+"\n", a...) +} + +// Hintf prints an informational line to stderr (never stdout). +func Hintf(format string, a ...interface{}) { + fmt.Fprintf(os.Stderr, format+"\n", a...) +} + +// ReadLine prints a prompt on stderr and reads one line from stdin (the +// empty string on EOF). Use it for free-text answers such as names. +func ReadLine(prompt string) (string, error) { + fmt.Fprint(promptOut, "? "+prompt) + line, err := readLine() + if err != nil && line == "" { + fmt.Fprintln(promptOut) + return "", nil + } + return strings.TrimSpace(line), nil +} + +// ChooseMany lets the user pick several options and returns their indices. In +// a real terminal it uses the shared Bubble Tea checkbox list (space to toggle, +// enter to confirm); otherwise it falls back to reading a comma/space separated +// list of numbers. Aborting (esc/ctrl+c, or EOF in the fallback) is a refusal, +// not a usage error; a confirmed but empty selection returns no indices. +func ChooseMany(question string, options []string) ([]int, error) { + if isTTY() { + idx, err := tui.RunMultiSelect(question, options, nil) + if errors.Is(err, tui.ErrCanceled) { + return nil, exitcode.Errorf(exitcode.Refused, "cancelled") + } + return idx, err + } + return chooseManyLine(question, options) +} + +// chooseManyLine is the non-interactive fallback of ChooseMany. +func chooseManyLine(question string, options []string) ([]int, error) { + // The numbered list only exists in this fallback, so the hint about typing + // numbers belongs here and not in the callers' question text. + fmt.Fprintln(promptOut, "? "+question+" (comma-separated numbers)") + for i, opt := range options { + fmt.Fprintf(promptOut, " [%d] %s\n", i+1, opt) + } + fmt.Fprint(promptOut, "> ") + line, err := readLine() + if err != nil && strings.TrimSpace(line) == "" { + fmt.Fprintln(promptOut) + return nil, exitcode.Errorf(exitcode.Refused, "cancelled") + } + seen := map[int]bool{} + var out []int + for _, tok := range strings.FieldsFunc(line, func(r rune) bool { return r == ',' || r == ' ' || r == ';' }) { + n, convErr := strconv.Atoi(strings.TrimSpace(tok)) + if convErr != nil || n < 1 || n > len(options) { + return nil, nil + } + if !seen[n-1] { + seen[n-1] = true + out = append(out, n-1) + } + } + return out, nil +} diff --git a/internal/objectstorage/prompt_test.go b/internal/objectstorage/prompt_test.go new file mode 100644 index 0000000..89854b5 --- /dev/null +++ b/internal/objectstorage/prompt_test.go @@ -0,0 +1,63 @@ +package objectstorage + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" +) + +// withPrompt swaps the prompt streams for the duration of a test. +func withPrompt(t *testing.T, stdin string) *bytes.Buffer { + t.Helper() + out := &bytes.Buffer{} + inSaved, outSaved := promptIn, promptOut + promptIn, promptOut = strings.NewReader(stdin), out + t.Cleanup(func() { promptIn, promptOut = inSaved, outSaved }) + return out +} + +// TestChooseManyLineCancelIsRefused covers the exit-code fix: aborting a +// selection is a refusal (7), not a usage error (2), and the numbered-list +// hint belongs to this fallback rather than to the callers' question text. +func TestChooseManyLineCancelIsRefused(t *testing.T) { + out := withPrompt(t, "") // EOF: nothing typed + idx, err := chooseManyLine("Which buckets should the key cover?", []string{"a", "b"}) + if idx != nil { + t.Errorf("selected %v on EOF", idx) + } + if code := exitcode.Of(err); code != exitcode.Refused { + t.Errorf("exit code = %d, want %d", code, exitcode.Refused) + } + if !strings.Contains(out.String(), "comma-separated numbers") { + t.Errorf("the fallback must explain how to answer, got %q", out.String()) + } +} + +func TestChooseManyLineSelection(t *testing.T) { + withPrompt(t, "1,3\n") + idx, err := chooseManyLine("Which buckets?", []string{"a", "b", "c"}) + if err != nil { + t.Fatalf("chooseManyLine: %v", err) + } + if len(idx) != 2 || idx[0] != 0 || idx[1] != 2 { + t.Errorf("selected %v, want [0 2]", idx) + } +} + +// TestHumanizeAPIInterruption covers the exit-code fix on the control plane: +// Ctrl-C must yield 130 whichever phase of a command it lands in. +func TestHumanizeAPIInterruption(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := HumanizeAPI(ctx.Err(), "project \"x\"") + if code := exitcode.Of(err); code != exitcode.Interrupted { + t.Errorf("exit code = %d, want %d (err=%v)", code, exitcode.Interrupted, err) + } + // The S3 path already behaved this way; the two must agree. + if code := exitcode.Of(Humanize(ctx.Err(), nil, nil)); code != exitcode.Interrupted { + t.Errorf("Humanize disagrees: %d", code) + } +} diff --git a/internal/objectstorage/region.go b/internal/objectstorage/region.go new file mode 100644 index 0000000..706e7f9 --- /dev/null +++ b/internal/objectstorage/region.go @@ -0,0 +1,60 @@ +package objectstorage + +import ( + "fmt" + "net/url" + "strings" +) + +// DefaultSigningRegion is used when the endpoint hostname does not encode a +// region. VAST clusters accept any region string; Wasabi requires the one in +// the hostname, which SigningRegion extracts. +const DefaultSigningRegion = "us-east-1" + +// SigningRegion derives the SigV4 signing region from a bucket endpoint. +// +// Latitude publishes two endpoint shapes (see the plan, §3): +// +// https://s3..storage.sh standard tier (CNAME to Wasabi) → +// https://objects..storage.sh high_performance tier (VAST) → +// +// Wasabi rejects requests signed with the wrong region, so for `s3.*` hosts +// the second label is authoritative. VAST does not validate the region, so the +// site label is used only for consistency; any value would work there. +// Anything else falls back to DefaultSigningRegion. +func SigningRegion(endpoint string) string { + host, _, err := EndpointHost(endpoint) + if err != nil { + return DefaultSigningRegion + } + labels := strings.Split(host, ".") + if len(labels) >= 3 && (labels[0] == "s3" || labels[0] == "objects") && labels[1] != "" { + return labels[1] + } + return DefaultSigningRegion +} + +// EndpointHost splits an endpoint into the host[:port] minio expects and +// whether TLS should be used. Bare hosts (no scheme) default to https. +func EndpointHost(endpoint string) (host string, secure bool, err error) { + e := strings.TrimSpace(endpoint) + if e == "" { + return "", false, fmt.Errorf("empty endpoint") + } + if !strings.Contains(e, "://") { + e = "https://" + e + } + u, err := url.Parse(e) + if err != nil || u.Host == "" { + return "", false, fmt.Errorf("invalid endpoint %q", endpoint) + } + switch strings.ToLower(u.Scheme) { + case "https": + secure = true + case "http": + secure = false + default: + return "", false, fmt.Errorf("unsupported endpoint scheme %q (use http or https)", u.Scheme) + } + return u.Host, secure, nil +} diff --git a/internal/objectstorage/region_test.go b/internal/objectstorage/region_test.go new file mode 100644 index 0000000..eecd70f --- /dev/null +++ b/internal/objectstorage/region_test.go @@ -0,0 +1,41 @@ +package objectstorage + +import "testing" + +func TestSigningRegion(t *testing.T) { + cases := map[string]string{ + "https://s3.us-east-1.storage.sh": "us-east-1", + "https://s3.eu-west-2.storage.sh": "eu-west-2", + "https://s3.ap-northeast-1.storage.sh": "ap-northeast-1", + "https://objects.tyo.storage.sh": "tyo", + "https://objects.nyc.storage.sh": "nyc", + "objects.lon2.storage.sh": "lon2", + "https://127.0.0.1:9000": DefaultSigningRegion, + "http://localhost:9000": DefaultSigningRegion, + "https://example.com": DefaultSigningRegion, + "": DefaultSigningRegion, + } + for in, want := range cases { + if got := SigningRegion(in); got != want { + t.Errorf("SigningRegion(%q) = %q, want %q", in, got, want) + } + } +} + +func TestEndpointHost(t *testing.T) { + host, secure, err := EndpointHost("https://s3.us-east-1.storage.sh") + if err != nil || host != "s3.us-east-1.storage.sh" || !secure { + t.Errorf("https: %q %v %v", host, secure, err) + } + host, secure, err = EndpointHost("http://127.0.0.1:9000") + if err != nil || host != "127.0.0.1:9000" || secure { + t.Errorf("http: %q %v %v", host, secure, err) + } + host, secure, err = EndpointHost("objects.tyo.storage.sh") + if err != nil || host != "objects.tyo.storage.sh" || !secure { + t.Errorf("bare: %q %v %v", host, secure, err) + } + if _, _, err := EndpointHost("ftp://x"); err == nil { + t.Error("ftp should be rejected") + } +} diff --git a/internal/objectstorage/resolver.go b/internal/objectstorage/resolver.go new file mode 100644 index 0000000..ed35131 --- /dev/null +++ b/internal/objectstorage/resolver.go @@ -0,0 +1,506 @@ +package objectstorage + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + sdk "github.com/latitudesh/latitudesh-go-sdk" + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/latitudesh-go-sdk/models/operations" + "github.com/latitudesh/lsh/internal/exitcode" + "github.com/latitudesh/lsh/internal/version" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/spf13/viper" +) + +// Storage classes as reported by the API. +const ( + ClassStandard = "standard" + ClassHighPerformance = "high_performance" +) + +// Bucket is the resolved view of an object storage bucket: everything the S3 +// layer needs (endpoint, real bucket name, signing region) plus the metadata +// shown by `stat` and used for safety checks (versioning, object lock). +type Bucket struct { + ID string + Name string // display name + BucketName string // real name on the S3 endpoint + Endpoint string // https://… + StorageClass string + // Site is the Latitude site slug (DAL, TYO4…). The SDK model drops it, so + // it is filled lazily by Resolver.FillSite when needed. + Site string + City string + Country string + ProjectID string + ProjectSlug string + ProjectName string + Versioning bool + Locking bool + // RetentionMode is NONE, GOVERNANCE or COMPLIANCE when object lock is on. + RetentionMode string + RetentionDays int64 + // Source is "default" (created through the API) or "synchronized" + // (imported from the provider; may lack an endpoint). + Source string + CreatedAt *time.Time + SigningRegion string + // EndpointOverride is true when the bucket was not looked up in the API + // (--endpoint-url / LSH_S3_ENDPOINT_URL): only BucketName, Endpoint and + // SigningRegion are meaningful. + EndpointOverride bool + // Data keeps the SDK payload for table/JSON rendering. + Data components.ObjectStorageData +} + +// ProjectRef returns the best human identifier for the bucket's project. +func (b *Bucket) ProjectRef() string { + if b.ProjectSlug != "" { + return b.ProjectSlug + } + return b.ProjectID +} + +// Display returns "name (bkt_…)" for messages. +func (b *Bucket) Display() string { + if b.EndpointOverride { + return b.BucketName + } + if b.ID == "" { + return b.Name + } + return fmt.Sprintf("%s (%s)", b.Name, b.ID) +} + +// Validate makes sure the bucket can be addressed over S3. +func (b *Bucket) Validate() error { + if b.BucketName == "" { + return exitcode.Errorf(exitcode.Generic, "bucket %s has no backend bucket name yet (it may still be provisioning); retry in a few seconds", b.Display()) + } + if b.Endpoint == "" { + return exitcode.Errorf(exitcode.Generic, "bucket %s has no S3 endpoint (source: %s); pass --endpoint-url if you know where it lives", b.Display(), b.Source) + } + // Validate up front so an invalid backend name is a usage error (exit 2) + // on every command instead of surfacing as a generic minio error. + if err := s3utils.CheckValidBucketName(b.BucketName); err != nil { + return exitcode.Errorf(exitcode.Usage, "invalid bucket name %q: %v", b.BucketName, err) + } + return nil +} + +// BucketFromData converts the SDK model into a Bucket. +func BucketFromData(d components.ObjectStorageData) *Bucket { + b := &Bucket{Data: d} + if d.ID != nil { + b.ID = *d.ID + } + a := d.Attributes + if a == nil { + return b + } + b.Name = str(a.Name) + b.BucketName = str(a.BucketName) + b.Endpoint = str(a.Endpoint) + if a.StorageClass != nil { + b.StorageClass = string(*a.StorageClass) + } + if a.Versioning != nil { + b.Versioning = *a.Versioning + } + if a.Locking != nil { + b.Locking = *a.Locking + } + if a.RetentionMode != nil { + b.RetentionMode = string(*a.RetentionMode) + } + if a.RetentionPeriod != nil { + b.RetentionDays = *a.RetentionPeriod + } + b.Source = str(a.Source) + b.CreatedAt = a.CreatedAt + if a.Region != nil { + b.Site = str(a.Region.ID) + b.City = str(a.Region.City) + b.Country = str(a.Region.Country) + } + if a.Project != nil { + b.ProjectID = str(a.Project.ID) + b.ProjectSlug = str(a.Project.Slug) + b.ProjectName = str(a.Project.Name) + } + b.SigningRegion = SigningRegion(b.Endpoint) + return b +} + +func str(s *string) string { + if s == nil { + return "" + } + return *s +} + +// Resolver turns a bucket token (display name, bkt_ ID or backend name) into +// a Bucket using the Latitude API, or bypasses the API when an endpoint +// override is in effect. +type Resolver struct { + API *sdk.Latitudesh + // Project narrows name lookups to one project (ID or slug). Empty means + // the whole team. + Project string + // EndpointURL enables the API-less mode: the token is the real bucket name + // on that endpoint. Saved credentials are never used in this mode. + EndpointURL string + // SigningRegion overrides the region derived from the endpoint. + SigningRegion string + // ClassFilter, when set, keeps only buckets of this storage class when a + // name matches several (already normalized to standard|high_performance). + ClassFilter string + // SiteFilter, when set, keeps only buckets in this Latitude site (slug, + // e.g. DAL, TYO4) when a name matches several. + SiteFilter string + // HasFilterFlags reports whether the command registered + // --storage-class/-c and --site. The ambiguity error only advertises them + // when it does; cp/mv/sync spend --storage-class on a different meaning. + HasFilterFlags bool + // FilterErr, when set, is returned by Resolve/ListBuckets before any call + // (e.g. an invalid --storage-class value). + FilterErr error + // RetryOptions are passed to every SDK call. + RetryOptions []operations.Option +} + +// Resolve resolves ref.Bucket. +func (r *Resolver) Resolve(ctx context.Context, token string) (*Bucket, error) { + if r.FilterErr != nil { + return nil, r.FilterErr + } + token = strings.TrimSpace(token) + if token == "" { + return nil, exitcode.Errorf(exitcode.Usage, "missing bucket name") + } + if r.EndpointURL != "" { + b := &Bucket{ + Name: token, + BucketName: token, + Endpoint: r.EndpointURL, + EndpointOverride: true, + SigningRegion: SigningRegion(r.EndpointURL), + } + if r.SigningRegion != "" { + b.SigningRegion = r.SigningRegion + } + return b, nil + } + if r.API == nil { + return nil, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login', or set LSH_S3_ENDPOINT_URL to address the bucket without the API") + } + + var b *Bucket + if strings.HasPrefix(token, "bkt_") { + resp, err := r.API.ObjectStorage.GetStorageBucket(ctx, token, r.RetryOptions...) + if err != nil { + return nil, humanizeAPIError(err, fmt.Sprintf("bucket %q", token)) + } + if resp.Object == nil || resp.Object.Data == nil { + return nil, bucketNotFound(token, r.Project) + } + b = BucketFromData(*resp.Object.Data) + } else { + list, err := r.ListBuckets(ctx) + if err != nil { + return nil, err + } + var matches []components.ObjectStorageData + for _, d := range list { + if d.Attributes == nil { + continue + } + if str(d.Attributes.Name) == token || str(d.Attributes.BucketName) == token { + matches = append(matches, d) + } + } + nameMatches := len(matches) + matches, err = r.applyFilters(ctx, matches) + if err != nil { + return nil, err + } + switch len(matches) { + case 0: + if nameMatches > 0 { + // The name existed but the --storage-class/--site filters ruled + // every candidate out. + return nil, exitcode.Errorf(exitcode.NotFound, "no bucket named %q matches the given filters (%s); run 'lsh s3 list' to see the buckets", token, r.filterDesc()) + } + return nil, bucketNotFound(token, r.Project) + case 1: + b = BucketFromData(matches[0]) + default: + return nil, ambiguousBucket(token, matches, r.HasFilterFlags) + } + // The list payload omits some attributes; fetch the full record. + if b.ID != "" && (b.Endpoint == "" || b.BucketName == "") { + resp, err := r.API.ObjectStorage.GetStorageBucket(ctx, b.ID, r.RetryOptions...) + if err == nil && resp.Object != nil && resp.Object.Data != nil { + b = BucketFromData(*resp.Object.Data) + } + } + } + if r.SigningRegion != "" { + b.SigningRegion = r.SigningRegion + } + return b, nil +} + +// ListBuckets returns the team's buckets (filtered by r.Project when set). +func (r *Resolver) ListBuckets(ctx context.Context) ([]components.ObjectStorageData, error) { + if r.FilterErr != nil { + return nil, r.FilterErr + } + if r.API == nil { + return nil, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login' first") + } + var filter *string + if r.Project != "" { + p := r.Project + filter = &p + } + resp, err := r.API.ObjectStorage.GetStorageBuckets(ctx, filter, r.RetryOptions...) + if err != nil { + return nil, humanizeAPIError(err, "buckets") + } + if resp.ObjectStorages == nil { + return nil, nil + } + return resp.ObjectStorages.Data, nil +} + +// applyFilters narrows name matches by storage class and site. Class comes +// straight from the list payload; the site slug does not (the SDK model drops +// it), so it is fetched once from the API only when a site filter is set. +func (r *Resolver) applyFilters(ctx context.Context, matches []components.ObjectStorageData) ([]components.ObjectStorageData, error) { + if (r.ClassFilter == "" && r.SiteFilter == "") || len(matches) == 0 { + return matches, nil + } + var sites map[string]string + if r.SiteFilter != "" { + var err error + if sites, err = RawBucketSitesForProject(ctx, "", r.Project); err != nil { + return nil, err + } + } + out := make([]components.ObjectStorageData, 0, len(matches)) + for _, d := range matches { + b := BucketFromData(d) + if r.ClassFilter != "" && !strings.EqualFold(b.StorageClass, r.ClassFilter) { + continue + } + if r.SiteFilter != "" { + site := b.Site + if site == "" && d.ID != nil { + site = sites[*d.ID] + } + if !strings.EqualFold(site, r.SiteFilter) { + continue + } + } + out = append(out, d) + } + return out, nil +} + +// filterDesc renders the active filters for error messages. +func (r *Resolver) filterDesc() string { + var parts []string + if r.ClassFilter != "" { + parts = append(parts, "--storage-class "+r.ClassFilter) + } + if r.SiteFilter != "" { + parts = append(parts, "--site "+r.SiteFilter) + } + return strings.Join(parts, ", ") +} + +func ambiguousBucket(token string, matches []components.ObjectStorageData, hasFilterFlags bool) error { + lines := make([]string, 0, len(matches)) + for _, d := range matches { + b := BucketFromData(d) + lines = append(lines, fmt.Sprintf(" %s project=%s class=%s site=%s", b.ID, b.ProjectRef(), b.StorageClass, firstNonEmpty(b.Site, b.City))) + } + sort.Strings(lines) + narrow := "narrow it with --project, or use the bkt_ ID" + if hasFilterFlags { + narrow = "narrow it with --project, --storage-class/-c or --site, or use the bkt_ ID" + } + return exitcode.Errorf(exitcode.Usage, "bucket name %q is ambiguous:\n%s\n%s", token, strings.Join(lines, "\n"), narrow) +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +// FillSite fetches the bucket's site slug (e.g. TYO4) straight from the API +// when the SDK model did not carry it. It is a no-op when already known or in +// endpoint-override mode. +func (r *Resolver) FillSite(ctx context.Context, b *Bucket) error { + if b == nil || b.Site != "" || b.EndpointOverride || b.ID == "" { + return nil + } + sites, err := RawBucketSites(ctx, b.ID) + if err != nil { + return err + } + b.Site = sites[b.ID] + return nil +} + +// RawBucketSites returns bucket ID → site slug for one bucket (bucketID set) +// or for every bucket of the team (bucketID empty). It talks to the API +// directly because the generated model discards `region.site`. +func RawBucketSites(ctx context.Context, bucketID string) (map[string]string, error) { + return RawBucketSitesForProject(ctx, bucketID, "") +} + +// RawBucketSitesForProject is RawBucketSites restricted to one project (ID or +// slug) when bucketID is empty, avoiding a team-wide listing. +func RawBucketSitesForProject(ctx context.Context, bucketID, project string) (map[string]string, error) { + path := "/storage/buckets" + var query url.Values + if bucketID != "" { + path += "/" + url.PathEscape(bucketID) + } else if project != "" { + query = url.Values{"filter[project]": {project}} + } + body, err := rawAPIGet(ctx, path, query) + if err != nil { + return nil, err + } + type envelope struct { + Data json.RawMessage `json:"data"` + } + type item struct { + ID string `json:"id"` + Attributes struct { + Region struct { + Site struct { + Slug string `json:"slug"` + } `json:"site"` + } `json:"region"` + } `json:"attributes"` + } + var env envelope + if err := json.Unmarshal(body, &env); err != nil { + return nil, fmt.Errorf("could not parse bucket response: %w", err) + } + out := map[string]string{} + var many []item + if err := json.Unmarshal(env.Data, &many); err == nil { + for _, it := range many { + out[it.ID] = it.Attributes.Region.Site.Slug + } + return out, nil + } + var one item + if err := json.Unmarshal(env.Data, &one); err != nil { + return nil, fmt.Errorf("could not parse bucket response: %w", err) + } + out[one.ID] = one.Attributes.Region.Site.Slug + return out, nil +} + +// RawAccessKeyScopes returns, per access key ID, the bucket names and access +// level the API reports (`buckets[]` and `access`), which the SDK model does +// not expose. project is the project ID or slug. +func RawAccessKeyScopes(ctx context.Context, project string) (map[string]AccessKeyScope, error) { + body, err := rawAPIGet(ctx, "/storage/access_keys", url.Values{"project": {project}}) + if err != nil { + return nil, err + } + var env struct { + Data map[string][]struct { + AccessKeyID string `json:"access_key_id"` + Username string `json:"username"` + Access string `json:"access"` + Buckets []string `json:"buckets"` + Region string `json:"region"` + } `json:"data"` + } + if err := json.Unmarshal(body, &env); err != nil { + return nil, fmt.Errorf("could not parse access keys response: %w", err) + } + out := map[string]AccessKeyScope{} + for class, keys := range env.Data { + for _, k := range keys { + out[k.AccessKeyID] = AccessKeyScope{Access: k.Access, Buckets: k.Buckets, StorageClass: class, Site: k.Region, Username: k.Username} + } + } + return out, nil +} + +// AccessKeyScope is the scope information the API returns for a key. +type AccessKeyScope struct { + // Access is fullaccess, rw, readonly or "" (unknown). + Access string + Buckets []string // backend bucket names + StorageClass string + Site string + Username string +} + +// rawAPIGet performs an authenticated GET against the Latitude API and returns +// the body. It reuses the same connection settings (hostname, scheme, token, +// API version) the SDK client is configured with. +func rawAPIGet(ctx context.Context, path string, query url.Values) ([]byte, error) { + token := viper.GetString("Authorization") + if token == "" { + return nil, exitcode.Errorf(exitcode.Credentials, "not logged in — run 'lsh login' first") + } + host := viper.GetString("hostname") + if host == "" { + host = "api.latitude.sh" + } + scheme := viper.GetString("scheme") + if scheme == "" { + scheme = "https" + } + u := scheme + "://" + host + path + if len(query) > 0 { + u += "?" + query.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+strings.TrimPrefix(token, "Bearer ")) + req.Header.Set("Accept", "application/vnd.api+json, application/json") + req.Header.Set("User-Agent", "lsh/"+version.Version) + if v := viper.GetString("api-version"); v != "" { + req.Header.Set("API-Version", v) + } + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, exitcode.Errorf(exitcode.Generic, "could not reach %s: %v", host, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, humanizeAPIStatus(resp.StatusCode, body, path) + } + return body, nil +} diff --git a/internal/objectstorage/resolver_filter_test.go b/internal/objectstorage/resolver_filter_test.go new file mode 100644 index 0000000..f5165fb --- /dev/null +++ b/internal/objectstorage/resolver_filter_test.go @@ -0,0 +1,45 @@ +package objectstorage + +import ( + "context" + "testing" + + "github.com/latitudesh/latitudesh-go-sdk/models/components" + "github.com/latitudesh/lsh/internal/exitcode" +) + +func classData(id, name, class string) components.ObjectStorageData { + i, n := id, name + c := components.StorageClass(class) + return components.ObjectStorageData{ID: &i, Attributes: &components.ObjectStorageDataAttributes{Name: &n, StorageClass: &c}} +} + +func TestApplyFiltersByClass(t *testing.T) { + matches := []components.ObjectStorageData{ + classData("bkt_std", "backups", "standard"), + classData("bkt_hp", "backups", "high_performance"), + } + r := &Resolver{ClassFilter: "high_performance"} + out, err := r.applyFilters(context.Background(), matches) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || *out[0].ID != "bkt_hp" { + t.Fatalf("class filter kept %d matches, want just bkt_hp", len(out)) + } + // No filter keeps everything. + out, _ = (&Resolver{}).applyFilters(context.Background(), matches) + if len(out) != 2 { + t.Fatalf("no filter must keep both, got %d", len(out)) + } +} + +func TestResolveFilterErr(t *testing.T) { + r := &Resolver{FilterErr: exitcode.Errorf(exitcode.Usage, "bad class")} + if _, err := r.Resolve(context.Background(), "backups"); exitcode.Of(err) != exitcode.Usage { + t.Fatalf("Resolve must return FilterErr, got %v", err) + } + if _, err := r.ListBuckets(context.Background()); exitcode.Of(err) != exitcode.Usage { + t.Fatalf("ListBuckets must return FilterErr, got %v", err) + } +} diff --git a/internal/objectstorage/s3test/server.go b/internal/objectstorage/s3test/server.go new file mode 100644 index 0000000..ae911f0 --- /dev/null +++ b/internal/objectstorage/s3test/server.go @@ -0,0 +1,808 @@ +// Package s3test is an in-memory, path-style S3 server for unit tests. It +// implements the subset of the API the CLI uses (ListObjectsV2, ListObjectVersions, +// Put/Get/Head/Delete object, multi-delete, multipart upload, server-side copy) +// and records every request so tests can assert on headers (no checksum +// trailers, signing region, path style) without a real backend. +package s3test + +import ( + "crypto/md5" + "encoding/hex" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// Object is a stored object. +type Object struct { + Key string + Data []byte + ContentType string + Metadata map[string]string // x-amz-meta-* (lower-case keys without prefix) + Headers map[string]string // other response headers (Cache-Control…) + LastModified time.Time + ETag string + VersionID string +} + +// Bucket is a stored bucket. +type Bucket struct { + Name string + Objects map[string]*Object + // Versions keeps non-current versions when Versioned is true. + Versions map[string][]*Object + Versioned bool +} + +// Request is a recorded incoming request. +type Request struct { + Method string + Path string + Query url.Values + Header http.Header + Body []byte +} + +// Server is the fake S3 endpoint. +type Server struct { + mu sync.Mutex + buckets map[string]*Bucket + requests []Request + uploads map[string]*multipart + httptest *httptest.Server + // DenyAll makes every request fail with AccessDenied (403). + DenyAll bool + // FailNext, when set, is returned once for the next request. + FailNext *ErrorResponse + // SlowListPages forces ListObjectsV2 to paginate with at most N keys per + // page when max-keys is larger, to exercise continuation tokens. + MaxListKeys int + now func() time.Time +} + +type multipart struct { + bucket, key string + parts map[int][]byte + contentType string + metadata map[string]string + headers map[string]string +} + +// ErrorResponse is an S3 XML error. +type ErrorResponse struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + Resource string `xml:"Resource,omitempty"` + RequestID string `xml:"RequestId"` + Region string `xml:"Region,omitempty"` + Status int `xml:"-"` +} + +// New starts a server. Call Close when done. +func New() *Server { + s := &Server{ + buckets: map[string]*Bucket{}, + uploads: map[string]*multipart{}, + now: time.Now, + } + s.httptest = httptest.NewServer(http.HandlerFunc(s.handle)) + return s +} + +// URL returns the endpoint (http://127.0.0.1:port). +func (s *Server) URL() string { return s.httptest.URL } + +// Close shuts the server down. +func (s *Server) Close() { s.httptest.Close() } + +// CreateBucket adds an empty bucket. +func (s *Server) CreateBucket(name string) *Bucket { + s.mu.Lock() + defer s.mu.Unlock() + b := &Bucket{Name: name, Objects: map[string]*Object{}, Versions: map[string][]*Object{}} + s.buckets[name] = b + return b +} + +// AddObject stores an object (creating the bucket if needed). +func (s *Server) AddObject(bucket, key string, data []byte, contentType string) *Object { + s.mu.Lock() + b := s.buckets[bucket] + if b == nil { + b = &Bucket{Name: bucket, Objects: map[string]*Object{}, Versions: map[string][]*Object{}} + s.buckets[bucket] = b + } + s.mu.Unlock() + return s.put(b, key, data, contentType, nil, nil) +} + +// Object returns a stored object or nil. +func (s *Server) Object(bucket, key string) *Object { + s.mu.Lock() + defer s.mu.Unlock() + b := s.buckets[bucket] + if b == nil { + return nil + } + return b.Objects[key] +} + +// Keys returns the sorted object keys of a bucket. +func (s *Server) Keys(bucket string) []string { + s.mu.Lock() + defer s.mu.Unlock() + b := s.buckets[bucket] + if b == nil { + return nil + } + keys := make([]string, 0, len(b.Objects)) + for k := range b.Objects { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// Requests returns a copy of the recorded requests. +func (s *Server) Requests() []Request { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]Request, len(s.requests)) + copy(out, s.requests) + return out +} + +// WriteRequests returns the recorded requests that mutate state. +func (s *Server) WriteRequests() []Request { + var out []Request + for _, r := range s.Requests() { + if r.Method == http.MethodPut || r.Method == http.MethodPost || r.Method == http.MethodDelete { + out = append(out, r) + } + } + return out +} + +// ResetRequests clears the request log. +func (s *Server) ResetRequests() { + s.mu.Lock() + defer s.mu.Unlock() + s.requests = nil +} + +func etagOf(data []byte) string { + sum := md5.Sum(data) + return `"` + hex.EncodeToString(sum[:]) + `"` +} + +func (s *Server) put(b *Bucket, key string, data []byte, contentType string, meta, headers map[string]string) *Object { + s.mu.Lock() + defer s.mu.Unlock() + if contentType == "" { + contentType = "binary/octet-stream" + } + o := &Object{Key: key, Data: data, ContentType: contentType, Metadata: meta, Headers: headers, LastModified: s.now().UTC().Truncate(time.Second), ETag: etagOf(data)} + if b.Versioned { + if prev := b.Objects[key]; prev != nil { + b.Versions[key] = append(b.Versions[key], prev) + } + o.VersionID = fmt.Sprintf("v%d", len(b.Versions[key])+1) + } + b.Objects[key] = o + return o +} + +func (s *Server) record(r *http.Request, body []byte) { + s.mu.Lock() + defer s.mu.Unlock() + s.requests = append(s.requests, Request{Method: r.Method, Path: r.URL.Path, Query: r.URL.Query(), Header: r.Header.Clone(), Body: body}) +} + +func writeError(w http.ResponseWriter, e ErrorResponse) { + if e.RequestID == "" { + e.RequestID = "s3test" + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(e.Status) + _ = xml.NewEncoder(w).Encode(e) +} + +func (s *Server) handle(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + if isAWSChunked(r.Header) { + body = decodeAWSChunked(body) + } + s.record(r, body) + + s.mu.Lock() + failNext := s.FailNext + s.FailNext = nil + deny := s.DenyAll + s.mu.Unlock() + if failNext != nil { + writeError(w, *failNext) + return + } + if deny { + writeError(w, ErrorResponse{Code: "AccessDenied", Message: "Access Denied", Status: 403}) + return + } + + // Path-style: /{bucket}[/{key}] + path := strings.TrimPrefix(r.URL.Path, "/") + if path == "" { + s.listBuckets(w) + return + } + bucketName, key := path, "" + if i := strings.Index(path, "/"); i >= 0 { + bucketName, key = path[:i], path[i+1:] + } + key, _ = url.PathUnescape(key) + + s.mu.Lock() + b := s.buckets[bucketName] + s.mu.Unlock() + if b == nil { + writeError(w, ErrorResponse{Code: "NoSuchBucket", Message: "The specified bucket does not exist", Resource: "/" + bucketName, Status: 404}) + return + } + q := r.URL.Query() + + if key == "" { + switch { + case r.Method == http.MethodGet && q.Has("versions"): + s.listVersions(w, b, q) + case r.Method == http.MethodGet && q.Get("list-type") == "2": + s.listV2(w, b, q) + case r.Method == http.MethodGet && q.Has("location"): + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, `us-east-1`) + case r.Method == http.MethodGet: + // ListObjects v1 fallback: treat as v2 without tokens. + s.listV2(w, b, q) + case r.Method == http.MethodHead: + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && q.Has("delete"): + s.multiDelete(w, b, body) + default: + writeError(w, ErrorResponse{Code: "MethodNotAllowed", Message: "method not allowed on bucket", Status: 405}) + } + return + } + + switch r.Method { + case http.MethodPut: + if src := r.Header.Get("X-Amz-Copy-Source"); src != "" { + s.copyObject(w, b, key, src, r.Header) + return + } + if q.Has("uploadId") { + s.uploadPart(w, q.Get("uploadId"), q.Get("partNumber"), body) + return + } + meta, headers := extractMeta(r.Header) + o := s.put(b, key, body, r.Header.Get("Content-Type"), meta, headers) + w.Header().Set("ETag", o.ETag) + if o.VersionID != "" { + w.Header().Set("x-amz-version-id", o.VersionID) + } + w.WriteHeader(http.StatusOK) + case http.MethodPost: + switch { + case q.Has("uploads"): + s.initiateMultipart(w, b, key, r.Header) + case q.Has("uploadId"): + s.completeMultipart(w, b, key, q.Get("uploadId"), body) + default: + writeError(w, ErrorResponse{Code: "MethodNotAllowed", Message: "bad POST", Status: 405}) + } + case http.MethodGet, http.MethodHead: + s.getObject(w, r, b, key, q.Get("versionId")) + case http.MethodDelete: + if q.Has("uploadId") { + s.mu.Lock() + delete(s.uploads, q.Get("uploadId")) + s.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + return + } + s.mu.Lock() + if vid := q.Get("versionId"); vid != "" { + s.deleteVersion(b, key, vid) + } else { + delete(b.Objects, key) + } + s.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + writeError(w, ErrorResponse{Code: "MethodNotAllowed", Message: "method not allowed", Status: 405}) + } +} + +func (s *Server) deleteVersion(b *Bucket, key, vid string) { + if cur := b.Objects[key]; cur != nil && cur.VersionID == vid { + vs := b.Versions[key] + if len(vs) > 0 { + b.Objects[key] = vs[len(vs)-1] + b.Versions[key] = vs[:len(vs)-1] + } else { + delete(b.Objects, key) + } + return + } + vs := b.Versions[key] + for i, v := range vs { + if v.VersionID == vid { + b.Versions[key] = append(vs[:i], vs[i+1:]...) + return + } + } +} + +func extractMeta(h http.Header) (map[string]string, map[string]string) { + meta := map[string]string{} + headers := map[string]string{} + for k, v := range h { + lk := strings.ToLower(k) + if strings.HasPrefix(lk, "x-amz-meta-") && len(v) > 0 { + meta[strings.TrimPrefix(lk, "x-amz-meta-")] = v[0] + } + switch lk { + case "cache-control", "content-encoding", "content-disposition", "content-language", "expires": + if len(v) > 0 { + headers[http.CanonicalHeaderKey(lk)] = v[0] + } + } + } + return meta, headers +} + +type listBucketsResult struct { + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Buckets []struct { + Name string `xml:"Name"` + CreationDate string `xml:"CreationDate"` + } `xml:"Buckets>Bucket"` +} + +func (s *Server) listBuckets(w http.ResponseWriter) { + s.mu.Lock() + defer s.mu.Unlock() + var res listBucketsResult + for name := range s.buckets { + res.Buckets = append(res.Buckets, struct { + Name string `xml:"Name"` + CreationDate string `xml:"CreationDate"` + }{name, s.now().UTC().Format(time.RFC3339)}) + } + w.Header().Set("Content-Type", "application/xml") + _ = xml.NewEncoder(w).Encode(res) +} + +type listV2Result struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Delimiter string `xml:"Delimiter,omitempty"` + KeyCount int `xml:"KeyCount"` + MaxKeys int `xml:"MaxKeys"` + IsTruncated bool `xml:"IsTruncated"` + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + Contents []contentsEntry `xml:"Contents"` + CommonPrefixes []commonPrefixes `xml:"CommonPrefixes"` +} + +type contentsEntry struct { + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` +} + +type commonPrefixes struct { + Prefix string `xml:"Prefix"` +} + +func (s *Server) listV2(w http.ResponseWriter, b *Bucket, q url.Values) { + s.mu.Lock() + defer s.mu.Unlock() + prefix := q.Get("prefix") + delimiter := q.Get("delimiter") + maxKeys := 1000 + if mk := q.Get("max-keys"); mk != "" { + if n, err := strconv.Atoi(mk); err == nil && n > 0 { + maxKeys = n + } + } + if s.MaxListKeys > 0 && maxKeys > s.MaxListKeys { + maxKeys = s.MaxListKeys + } + startAfter := q.Get("start-after") + if tok := q.Get("continuation-token"); tok != "" { + startAfter = tok + } + + keys := make([]string, 0, len(b.Objects)) + for k := range b.Objects { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + sort.Strings(keys) + + res := listV2Result{Name: b.Name, Prefix: prefix, Delimiter: delimiter, MaxKeys: maxKeys, ContinuationToken: q.Get("continuation-token")} + seenPrefix := map[string]bool{} + count := 0 + var last string + for _, k := range keys { + if startAfter != "" && k <= startAfter { + continue + } + if count >= maxKeys { + res.IsTruncated = true + res.NextContinuationToken = last + break + } + if delimiter != "" { + rest := k[len(prefix):] + if i := strings.Index(rest, delimiter); i >= 0 { + cp := prefix + rest[:i+len(delimiter)] + if !seenPrefix[cp] { + seenPrefix[cp] = true + res.CommonPrefixes = append(res.CommonPrefixes, commonPrefixes{Prefix: cp}) + count++ + } + last = cp + "￿" // skip everything under this prefix + continue + } + } + o := b.Objects[k] + res.Contents = append(res.Contents, contentsEntry{Key: k, LastModified: o.LastModified.Format("2006-01-02T15:04:05.000Z"), ETag: o.ETag, Size: int64(len(o.Data)), StorageClass: "STANDARD"}) + count++ + last = k + } + if res.IsTruncated && strings.HasSuffix(res.NextContinuationToken, "￿") { + // Keys sorted after a prefix marker: use the marker's prefix as token. + res.NextContinuationToken = strings.TrimSuffix(res.NextContinuationToken, "￿") + "￿" + } + res.KeyCount = count + w.Header().Set("Content-Type", "application/xml") + _ = xml.NewEncoder(w).Encode(res) +} + +type listVersionsResult struct { + XMLName xml.Name `xml:"ListVersionsResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + IsTruncated bool `xml:"IsTruncated"` + Versions []versionEntry `xml:"Version"` + DeleteMarkers []versionEntry `xml:"DeleteMarker"` + CommonPrefixes []commonPrefixes `xml:"CommonPrefixes"` +} + +type versionEntry struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` +} + +func (s *Server) listVersions(w http.ResponseWriter, b *Bucket, q url.Values) { + s.mu.Lock() + defer s.mu.Unlock() + prefix := q.Get("prefix") + res := listVersionsResult{Name: b.Name, Prefix: prefix} + keys := make([]string, 0, len(b.Objects)) + for k := range b.Objects { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + sort.Strings(keys) + for _, k := range keys { + o := b.Objects[k] + vid := o.VersionID + if vid == "" { + vid = "null" + } + res.Versions = append(res.Versions, versionEntry{Key: k, VersionID: vid, IsLatest: true, LastModified: o.LastModified.Format("2006-01-02T15:04:05.000Z"), ETag: o.ETag, Size: int64(len(o.Data))}) + for i := len(b.Versions[k]) - 1; i >= 0; i-- { + v := b.Versions[k][i] + res.Versions = append(res.Versions, versionEntry{Key: k, VersionID: v.VersionID, LastModified: v.LastModified.Format("2006-01-02T15:04:05.000Z"), ETag: v.ETag, Size: int64(len(v.Data))}) + } + } + w.Header().Set("Content-Type", "application/xml") + _ = xml.NewEncoder(w).Encode(res) +} + +func (s *Server) getObject(w http.ResponseWriter, r *http.Request, b *Bucket, key, versionID string) { + s.mu.Lock() + o := b.Objects[key] + if o != nil && versionID != "" && o.VersionID != versionID { + o = nil + for _, v := range b.Versions[key] { + if v.VersionID == versionID { + o = v + } + } + } + s.mu.Unlock() + if o == nil { + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusNotFound) + return + } + writeError(w, ErrorResponse{Code: "NoSuchKey", Message: "The specified key does not exist.", Resource: "/" + b.Name + "/" + key, Status: 404}) + return + } + h := w.Header() + h.Set("Content-Type", o.ContentType) + h.Set("ETag", o.ETag) + h.Set("Last-Modified", o.LastModified.UTC().Format(http.TimeFormat)) + h.Set("Accept-Ranges", "bytes") + if o.VersionID != "" { + h.Set("x-amz-version-id", o.VersionID) + } + for k, v := range o.Metadata { + h.Set("X-Amz-Meta-"+k, v) + } + for k, v := range o.Headers { + h.Set(k, v) + } + data := o.Data + status := http.StatusOK + if rng := r.Header.Get("Range"); rng != "" && strings.HasPrefix(rng, "bytes=") { + parts := strings.SplitN(strings.TrimPrefix(rng, "bytes="), "-", 2) + start, _ := strconv.Atoi(parts[0]) + end := len(data) - 1 + if len(parts) == 2 && parts[1] != "" { + if e, err := strconv.Atoi(parts[1]); err == nil && e < end { + end = e + } + } + if start > end || start >= len(data) { + writeError(w, ErrorResponse{Code: "InvalidRange", Message: "The requested range is not satisfiable", Status: 416}) + return + } + h.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + data = data[start : end+1] + status = http.StatusPartialContent + } + h.Set("Content-Length", strconv.Itoa(len(data))) + w.WriteHeader(status) + if r.Method == http.MethodGet { + _, _ = w.Write(data) + } +} + +type deleteRequest struct { + XMLName xml.Name `xml:"Delete"` + Quiet bool `xml:"Quiet"` + Objects []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + } `xml:"Object"` +} + +type deleteResult struct { + XMLName xml.Name `xml:"DeleteResult"` + Deleted []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId,omitempty"` + } `xml:"Deleted"` + Errors []struct { + Key string `xml:"Key"` + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` +} + +func (s *Server) multiDelete(w http.ResponseWriter, b *Bucket, body []byte) { + var req deleteRequest + if err := xml.Unmarshal(body, &req); err != nil { + writeError(w, ErrorResponse{Code: "MalformedXML", Message: err.Error(), Status: 400}) + return + } + var res deleteResult + s.mu.Lock() + for _, o := range req.Objects { + if o.VersionID != "" { + s.deleteVersion(b, o.Key, o.VersionID) + } else { + delete(b.Objects, o.Key) + } + if !req.Quiet { + res.Deleted = append(res.Deleted, struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId,omitempty"` + }{o.Key, o.VersionID}) + } + } + s.mu.Unlock() + w.Header().Set("Content-Type", "application/xml") + _ = xml.NewEncoder(w).Encode(res) +} + +func (s *Server) copyObject(w http.ResponseWriter, b *Bucket, key, src string, h http.Header) { + src = strings.TrimPrefix(src, "/") + src, _ = url.PathUnescape(src) + srcBucket, srcKey := src, "" + if i := strings.Index(src, "/"); i >= 0 { + srcBucket, srcKey = src[:i], src[i+1:] + } + if i := strings.Index(srcKey, "?versionId="); i >= 0 { + srcKey = srcKey[:i] + } + s.mu.Lock() + sb := s.buckets[srcBucket] + var so *Object + if sb != nil { + so = sb.Objects[srcKey] + } + s.mu.Unlock() + if so == nil { + writeError(w, ErrorResponse{Code: "NoSuchKey", Message: "The specified key does not exist.", Status: 404}) + return + } + meta, headers := so.Metadata, so.Headers + contentType := so.ContentType + if strings.EqualFold(h.Get("X-Amz-Metadata-Directive"), "REPLACE") { + meta, headers = extractMeta(h) + if ct := h.Get("Content-Type"); ct != "" { + contentType = ct + } + } + o := s.put(b, key, append([]byte(nil), so.Data...), contentType, meta, headers) + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, `%s%s`, o.LastModified.Format(time.RFC3339), o.ETag) +} + +func (s *Server) initiateMultipart(w http.ResponseWriter, b *Bucket, key string, h http.Header) { + s.mu.Lock() + id := fmt.Sprintf("upload-%d", len(s.uploads)+1) + meta, headers := extractMeta(h) + s.uploads[id] = &multipart{bucket: b.Name, key: key, parts: map[int][]byte{}, contentType: h.Get("Content-Type"), metadata: meta, headers: headers} + s.mu.Unlock() + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, `%s%s%s`, b.Name, xmlEscape(key), id) +} + +func (s *Server) uploadPart(w http.ResponseWriter, uploadID, partNumber string, body []byte) { + n, err := strconv.Atoi(partNumber) + s.mu.Lock() + up := s.uploads[uploadID] + s.mu.Unlock() + if err != nil || up == nil { + writeError(w, ErrorResponse{Code: "NoSuchUpload", Message: "The specified upload does not exist.", Status: 404}) + return + } + s.mu.Lock() + up.parts[n] = append([]byte(nil), body...) + s.mu.Unlock() + w.Header().Set("ETag", etagOf(body)) + w.WriteHeader(http.StatusOK) +} + +func (s *Server) completeMultipart(w http.ResponseWriter, b *Bucket, key, uploadID string, body []byte) { + s.mu.Lock() + up := s.uploads[uploadID] + if up != nil { + delete(s.uploads, uploadID) + } + s.mu.Unlock() + if up == nil { + writeError(w, ErrorResponse{Code: "NoSuchUpload", Message: "The specified upload does not exist.", Status: 404}) + return + } + nums := make([]int, 0, len(up.parts)) + for n := range up.parts { + nums = append(nums, n) + } + sort.Ints(nums) + var data []byte + for _, n := range nums { + data = append(data, up.parts[n]...) + } + o := s.put(b, key, data, up.contentType, up.metadata, up.headers) + o.ETag = fmt.Sprintf(`"%s-%d"`, strings.Trim(etagOf(data), `"`), len(nums)) + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, `%s%s%s`, b.Name, xmlEscape(key), o.ETag) +} + +func xmlEscape(s string) string { + var sb strings.Builder + _ = xml.EscapeText(&sb, []byte(s)) + return sb.String() +} + +// HasChecksumHeaders reports whether any recorded write request carried +// x-amz-checksum-* / x-amz-sdk-checksum-algorithm headers or declared +// checksum trailers — what S3-compatible backends such as Wasabi reject. +// (Plain SigV4 streaming over http is not a checksum trailer.) +func (s *Server) HasChecksumHeaders() bool { + for _, r := range s.WriteRequests() { + for k := range r.Header { + lk := strings.ToLower(k) + if strings.HasPrefix(lk, "x-amz-checksum-") || lk == "x-amz-sdk-checksum-algorithm" || lk == "x-amz-trailer" { + return true + } + } + if strings.Contains(strings.ToUpper(r.Header.Get("X-Amz-Content-Sha256")), "TRAILER") { + return true + } + } + return false +} + +// SigningRegionOf extracts the region from a request's SigV4 Authorization +// header ("" when absent). +func SigningRegionOf(r Request) string { + auth := r.Header.Get("Authorization") + i := strings.Index(auth, "Credential=") + if i < 0 { + return "" + } + parts := strings.Split(auth[i+len("Credential="):], "/") + if len(parts) < 3 { + return "" + } + return parts[2] +} + +// isAWSChunked reports whether the request body uses the SigV4 streaming +// framing (`aws-chunked`), which minio-go emits over plain http. +func isAWSChunked(h http.Header) bool { + if strings.Contains(strings.ToLower(h.Get("Content-Encoding")), "aws-chunked") { + return true + } + return strings.HasPrefix(strings.ToUpper(h.Get("X-Amz-Content-Sha256")), "STREAMING-") +} + +// decodeAWSChunked strips the `;chunk-signature=…\r\n\r\n` +// framing (and any trailing headers after the terminating 0 chunk). +func decodeAWSChunked(body []byte) []byte { + var out []byte + rest := body + for { + nl := strings.Index(string(rest), "\r\n") + if nl < 0 { + return out + } + header := string(rest[:nl]) + rest = rest[nl+2:] + sizeHex := header + if i := strings.IndexByte(header, ';'); i >= 0 { + sizeHex = header[:i] + } + size, err := strconv.ParseInt(strings.TrimSpace(sizeHex), 16, 64) + if err != nil { + // Not framed after all: return the original body. + return body + } + if size == 0 { + return out + } + if int64(len(rest)) < size { + return out + } + out = append(out, rest[:size]...) + rest = rest[size:] + if len(rest) >= 2 && rest[0] == '\r' && rest[1] == '\n' { + rest = rest[2:] + } + } +} diff --git a/internal/objectstorage/storageclass.go b/internal/objectstorage/storageclass.go new file mode 100644 index 0000000..b456b3a --- /dev/null +++ b/internal/objectstorage/storageclass.go @@ -0,0 +1,39 @@ +package objectstorage + +import ( + "fmt" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" +) + +// ParseStorageClass normalizes a user-supplied storage class. It is the single +// alias table shared by every command so the same spelling is accepted +// everywhere: standard|std|wasabi and high_performance|high-performance|hp| +// high|vast (case-insensitive). An empty value returns "" without error so +// callers can apply their own default. +func ParseStorageClass(s string) (string, error) { + v := strings.ToLower(strings.TrimSpace(s)) + switch v { + case "": + return "", nil + case ClassStandard, "std", "wasabi": + return ClassStandard, nil + case ClassHighPerformance, "high-performance", "highperformance", "hp", "high", "vast": + return ClassHighPerformance, nil + } + return "", exitcode.Errorf(exitcode.Usage, "invalid storage class %q (expected standard or high_performance)", s) +} + +// StorageClassLabel returns a short human label for a storage class. +func StorageClassLabel(class string) string { + switch class { + case ClassHighPerformance: + return "high_performance (VAST)" + case ClassStandard: + return "standard (Wasabi)" + } + return class +} + +var _ = fmt.Sprintf diff --git a/internal/objectstorage/uri.go b/internal/objectstorage/uri.go new file mode 100644 index 0000000..a799843 --- /dev/null +++ b/internal/objectstorage/uri.go @@ -0,0 +1,197 @@ +package objectstorage + +import ( + "fmt" + "strings" + + "github.com/latitudesh/lsh/internal/exitcode" +) + +// Ref is a parsed command-line reference to a local path, stdin/stdout or an +// object storage location. +// +// Remote references follow the aws shape: `s3://[/]`. `lsh://` is +// accepted as a synonym. may be the bucket's display name, its +// `bkt_…` ID or the backend bucket name; the resolver decides which. +type Ref struct { + // Raw is the argument as typed. + Raw string + // Bucket is the bucket token (display name, bkt_ id or backend name). + Bucket string + // Key is the object key or prefix; empty when the reference is the bucket + // itself. + Key string + // Remote is true for s3:// and lsh:// references (or bare bucket tokens + // parsed through ParseRemote). + Remote bool + // Stdio is true when the argument is "-" (stdin for sources, stdout for + // destinations). + Stdio bool + // HadScheme records whether an explicit scheme was present. + HadScheme bool +} + +const ( + schemeS3 = "s3://" + schemeLsh = "lsh://" +) + +// IsDir reports whether the reference denotes a prefix (empty key or a key +// ending in "/"), following the aws convention for trailing slashes. +func (r Ref) IsDir() bool { + return r.Remote && (r.Key == "" || strings.HasSuffix(r.Key, "/")) +} + +// String renders the canonical s3:// form for remote references and the raw +// argument otherwise. +func (r Ref) String() string { + if !r.Remote { + return r.Raw + } + if r.Key == "" { + return schemeS3 + r.Bucket + } + return schemeS3 + r.Bucket + "/" + r.Key +} + +// WithKey returns a copy of r pointing at key. +func (r Ref) WithKey(key string) Ref { + r.Key = key + return r +} + +// stripScheme removes a leading s3:// or lsh:// and reports whether one was +// present. +func stripScheme(s string) (string, bool) { + lower := strings.ToLower(s) + switch { + case strings.HasPrefix(lower, schemeS3): + return s[len(schemeS3):], true + case strings.HasPrefix(lower, schemeLsh): + return s[len(schemeLsh):], true + } + return s, false +} + +// ParseRemote parses an argument that must name a bucket or object. The +// scheme is optional (`ls`, `stat`, `rm`, `presign`, `lifecycle`, `metrics` +// accept `my-bucket/prefix/` as well as `s3://my-bucket/prefix/`). +func ParseRemote(arg string) (Ref, error) { + s := strings.TrimSpace(arg) + if s == "" { + return Ref{}, exitcode.Errorf(exitcode.Usage, "missing bucket: expected s3://[/]") + } + if s == "-" { + return Ref{}, exitcode.Errorf(exitcode.Usage, "%q is not a valid bucket reference", arg) + } + rest, had := stripScheme(s) + if strings.HasPrefix(rest, "/") || strings.HasPrefix(rest, ".") { + return Ref{}, exitcode.Errorf(exitcode.Usage, "%q looks like a local path; expected s3://[/]", arg) + } + bucket, key := rest, "" + if i := strings.Index(rest, "/"); i >= 0 { + bucket, key = rest[:i], rest[i+1:] + } + if bucket == "" { + return Ref{}, exitcode.Errorf(exitcode.Usage, "%q has no bucket name; expected s3://[/]", arg) + } + return Ref{Raw: arg, Bucket: bucket, Key: key, Remote: true, HadScheme: had}, nil +} + +// ParseBucketOnly parses an argument that must name a bucket without a key +// (`mb`, `rb`). It mirrors the aws error for `aws s3 rb s3://b/key`. +func ParseBucketOnly(arg string) (Ref, error) { + r, err := ParseRemote(arg) + if err != nil { + return r, err + } + if r.Key != "" { + return r, exitcode.Errorf(exitcode.Usage, "please specify a valid bucket name only (got %q)", arg) + } + return r, nil +} + +// ParseTransferArg parses a `cp`/`mv`/`sync` operand. Only an explicit +// scheme makes it remote; "-" is stdin/stdout; anything else is a local path. +// This is the aws rule that keeps `cp ./file s3://b/` unambiguous. +func ParseTransferArg(arg string) (Ref, error) { + s := strings.TrimSpace(arg) + if s == "" { + return Ref{}, exitcode.Errorf(exitcode.Usage, "empty path argument") + } + if s == "-" { + return Ref{Raw: arg, Stdio: true}, nil + } + if _, had := stripScheme(s); had { + return ParseRemote(s) + } + return Ref{Raw: arg}, nil +} + +// ObjectRef parses an argument that must name an object (bucket and non-empty +// key). Keys ending in "/" are rejected unless allowPrefix is true. +func ObjectRef(arg string, allowPrefix bool) (Ref, error) { + r, err := ParseRemote(arg) + if err != nil { + return r, err + } + if r.Key == "" { + return r, exitcode.Errorf(exitcode.Usage, "%s names a bucket, not an object; expected s3:///", r) + } + if !allowPrefix && strings.HasSuffix(r.Key, "/") { + return r, exitcode.Errorf(exitcode.Usage, "key %q ends with '/'; use --recursive to operate on a prefix", r.Key) + } + return r, nil +} + +// JoinKey appends name to a prefix, inserting "/" when needed. +func JoinKey(prefix, name string) string { + if prefix == "" { + return name + } + if strings.HasSuffix(prefix, "/") { + return prefix + name + } + return prefix + "/" + name +} + +// BaseName returns the last path segment of key ("" when key ends in "/"). +func BaseName(key string) string { + if i := strings.LastIndex(key, "/"); i >= 0 { + return key[i+1:] + } + return key +} + +// NormalizePrefix ensures a non-empty prefix ends with "/" — the aws rule for +// `--recursive` targets, so `rm s3://b/path --recursive` does not touch +// `path2/`. +func NormalizePrefix(prefix string) string { + if prefix == "" || strings.HasSuffix(prefix, "/") { + return prefix + } + return prefix + "/" +} + +// DescribeRef renders a ref for messages, always in s3:// form for remotes. +func DescribeRef(r Ref) string { + if r.Stdio { + return "-" + } + return r.String() +} + +// ErrUsagef is a convenience for usage errors. +func ErrUsagef(format string, a ...interface{}) error { + return exitcode.Errorf(exitcode.Usage, format, a...) +} + +// bucketNotFound formats the "no such bucket" error shared by the resolver. +func bucketNotFound(ref, project string) error { + if project != "" { + return exitcode.Errorf(exitcode.NotFound, "bucket %q not found in project %s; run 'lsh s3 list' to see available buckets", ref, project) + } + return exitcode.Errorf(exitcode.NotFound, "bucket %q not found; run 'lsh s3 list' to see available buckets", ref) +} + +var _ = fmt.Sprintf diff --git a/internal/objectstorage/uri_test.go b/internal/objectstorage/uri_test.go new file mode 100644 index 0000000..8b2237f --- /dev/null +++ b/internal/objectstorage/uri_test.go @@ -0,0 +1,105 @@ +package objectstorage + +import ( + "testing" + + "github.com/latitudesh/lsh/internal/exitcode" +) + +func TestParseRemote(t *testing.T) { + cases := []struct { + in string + bucket string + key string + hadScheme bool + wantErr bool + wantIsDir bool + wantString string + }{ + {"s3://backups", "backups", "", true, false, true, "s3://backups"}, + {"s3://backups/", "backups", "", true, false, true, "s3://backups"}, + {"S3://backups/2026/09/", "backups", "2026/09/", true, false, true, "s3://backups/2026/09/"}, + {"lsh://bkt_123/a/b.txt", "bkt_123", "a/b.txt", true, false, false, "s3://bkt_123/a/b.txt"}, + {"backups/logs/", "backups", "logs/", false, false, true, "s3://backups/logs/"}, + {"backups", "backups", "", false, false, true, "s3://backups"}, + {"", "", "", false, true, false, ""}, + {"-", "", "", false, true, false, ""}, + {"./file", "", "", false, true, false, ""}, + {"/tmp/x", "", "", false, true, false, ""}, + {"s3:///nobucket", "", "", false, true, false, ""}, + } + for _, c := range cases { + r, err := ParseRemote(c.in) + if c.wantErr { + if err == nil { + t.Errorf("ParseRemote(%q): expected error", c.in) + } else if exitcode.Of(err) != exitcode.Usage { + t.Errorf("ParseRemote(%q): exit code %d, want %d", c.in, exitcode.Of(err), exitcode.Usage) + } + continue + } + if err != nil { + t.Errorf("ParseRemote(%q): unexpected error %v", c.in, err) + continue + } + if r.Bucket != c.bucket || r.Key != c.key || r.HadScheme != c.hadScheme || !r.Remote { + t.Errorf("ParseRemote(%q) = %+v", c.in, r) + } + if r.IsDir() != c.wantIsDir { + t.Errorf("ParseRemote(%q).IsDir() = %v, want %v", c.in, r.IsDir(), c.wantIsDir) + } + if r.String() != c.wantString { + t.Errorf("ParseRemote(%q).String() = %q, want %q", c.in, r.String(), c.wantString) + } + } +} + +func TestParseBucketOnlyRejectsKeys(t *testing.T) { + if _, err := ParseBucketOnly("s3://b/key"); err == nil { + t.Fatal("expected error for key in bucket-only argument") + } + r, err := ParseBucketOnly("b") + if err != nil || r.Bucket != "b" { + t.Fatalf("ParseBucketOnly(b) = %+v, %v", r, err) + } +} + +func TestParseTransferArg(t *testing.T) { + if r, _ := ParseTransferArg("-"); !r.Stdio { + t.Error("- should be stdio") + } + if r, _ := ParseTransferArg("./dir/file.txt"); r.Remote || r.Stdio { + t.Error("bare path should be local") + } + if r, _ := ParseTransferArg("backups/file.txt"); r.Remote { + t.Error("bare bucket/key without scheme must stay local in transfer args") + } + r, err := ParseTransferArg("s3://backups/dir/") + if err != nil || !r.Remote || r.Bucket != "backups" || r.Key != "dir/" { + t.Errorf("ParseTransferArg(s3://backups/dir/) = %+v, %v", r, err) + } +} + +func TestObjectRef(t *testing.T) { + if _, err := ObjectRef("s3://b", false); err == nil { + t.Error("bucket-only must be rejected") + } + if _, err := ObjectRef("s3://b/dir/", false); err == nil { + t.Error("trailing slash must be rejected without allowPrefix") + } + if _, err := ObjectRef("s3://b/dir/", true); err != nil { + t.Errorf("trailing slash with allowPrefix: %v", err) + } +} + +func TestKeyHelpers(t *testing.T) { + if JoinKey("", "a") != "a" || JoinKey("p/", "a") != "p/a" || JoinKey("p", "a") != "p/a" { + t.Error("JoinKey") + } + if BaseName("a/b/c.txt") != "c.txt" || BaseName("c.txt") != "c.txt" || BaseName("a/") != "" { + t.Error("BaseName") + } + if NormalizePrefix("") != "" || NormalizePrefix("a") != "a/" || NormalizePrefix("a/") != "a/" { + t.Error("NormalizePrefix") + } +} diff --git a/internal/output/table/utils.go b/internal/output/table/utils.go index d411e1b..4838935 100644 --- a/internal/output/table/utils.go +++ b/internal/output/table/utils.go @@ -62,6 +62,8 @@ func RenderEmptyState(message string) { var PreferredColumnOrder = []string{ "id", "name", + "bucket_name", + "storage_class", "slug", "environment", "description", @@ -80,9 +82,48 @@ var PreferredColumnOrder = []string{ "primary_ipv4", "region", "endpoint", - "storage_class", + "versioning", + "locking", "created_at", "updated_at", + // Object storage (lsh s3) columns. + "type", + "key", + "size", + "last_modified", + "etag", + "content_type", + "version_id", + "op", + "source", + "destination", + "url", + "method", + "expires_at", + "access_key_id", + "scope", + "buckets", + "site", + "saved", + "project", + "prefix", + "expiration_days", + "noncurrent_days", + "abort_mpu_days", + "enabled", + "bucket", + "current_gb", + "consumed_gb", + "estimated_cost", + "currency", + "period_start", + "period_end", + "date", + "tier", + "bytes", + "days", + "avg_bytes", + "action", } // SortColumnsByPreference orders column ids by PreferredColumnOrder, with diff --git a/internal/renderer/format.go b/internal/renderer/format.go index d852c18..d1a1b50 100644 --- a/internal/renderer/format.go +++ b/internal/renderer/format.go @@ -9,8 +9,8 @@ import ( ) // Format is the output format selected for a command's results. table is the -// human-facing default; json, yaml and csv are the machine-readable formats -// meant for automation, piping and (in the future) an MCP server. +// human-facing default; json, yaml, csv and text are the machine-readable +// formats meant for automation, piping and (in the future) an MCP server. type Format string const ( @@ -18,12 +18,15 @@ const ( FormatJSON Format = "json" FormatYAML Format = "yaml" FormatCSV Format = "csv" + // FormatText prints raw values (no quotes, tab-separated rows), like + // `aws --output text`; combined with --query it feeds other tools directly. + FormatText Format = "text" ) // IsStructured reports whether the format produces machine-readable, queryable // output. Only structured formats support the --query (JMESPath) flag. func (f Format) IsStructured() bool { - return f == FormatJSON || f == FormatYAML || f == FormatCSV + return f == FormatJSON || f == FormatYAML || f == FormatCSV || f == FormatText } // ParseFormat normalizes a user-supplied format string. ok is false for an @@ -38,6 +41,8 @@ func ParseFormat(s string) (Format, bool) { return FormatYAML, true case "csv": return FormatCSV, true + case "text": + return FormatText, true default: return "", false } @@ -73,7 +78,7 @@ func ValidateOutputSelection() error { // Validate the format unconditionally — even when --json is set, a typo'd // --output should surface early rather than be silently ignored. if _, ok := ParseFormat(viper.GetString("output")); !ok { - return fmt.Errorf("invalid --output %q (valid values: table, json, yaml, csv)", viper.GetString("output")) + return fmt.Errorf("invalid --output %q (valid values: table, json, yaml, csv, text)", viper.GetString("output")) } query := strings.TrimSpace(viper.GetString("query")) @@ -81,7 +86,7 @@ func ValidateOutputSelection() error { return nil } if !ResolveFormat().IsStructured() { - return fmt.Errorf("--query requires a structured output format; add -o json, -o yaml, or -o csv") + return fmt.Errorf("--query requires a structured output format; add -o json, -o yaml, -o csv, or -o text") } // Compile the JMESPath now so an invalid expression fails fast with a // non-zero exit code instead of only printing to stderr at render time — diff --git a/internal/renderer/main.go b/internal/renderer/main.go index 1b14199..358da6a 100644 --- a/internal/renderer/main.go +++ b/internal/renderer/main.go @@ -17,7 +17,7 @@ type Renderer interface { // GetRenderer returns the renderer for the active output format. // -// Structured formats (json/yaml/csv) are honored everywhere, including when +// Structured formats (json/yaml/csv/text) are honored everywhere, including when // stdout is piped — that is the whole point of automation output. The table // format additionally chooses between the classic ASCII writer (for CI / // LSH_CLASSIC_OUTPUT / non-TTY pipes) and the interactive Bubble Tea view. @@ -29,6 +29,8 @@ func GetRenderer() Renderer { return YAMLRenderer{} case FormatCSV: return CSVRenderer{} + case FormatText: + return TextRenderer{} } // Human-facing table path. @@ -52,6 +54,8 @@ func GetStaticRenderer() Renderer { return YAMLRenderer{} case FormatCSV: return CSVRenderer{} + case FormatText: + return TextRenderer{} } return TableRenderer{} // plain ASCII, no full-screen takeover } diff --git a/internal/renderer/text.go b/internal/renderer/text.go new file mode 100644 index 0000000..b436384 --- /dev/null +++ b/internal/renderer/text.go @@ -0,0 +1,132 @@ +package renderer + +import ( + "fmt" + "io" + "os" + "sort" + "strings" +) + +// TextRenderer implements the "text" output format, modelled on +// `aws --output text`: values are printed raw, without quotes or structure, +// so a `--query secret_access_key` result can be piped straight into another +// tool (e.g. `gh secret set`). +// +// Shapes, after --query has been applied: +// - scalar -> the value on one line, without quotes +// - list of scalars -> one value per line +// - list of objects -> one tab-separated row per object, keys sorted +// - object -> one "keyvalue" line per key, keys sorted +// +// Nested values (objects or arrays inside a cell) are JSON-encoded so a row +// never spans multiple lines. +type TextRenderer struct{} + +func (tr TextRenderer) Render(data []ResponseData) { + if err := renderText(data); err != nil { + reportRenderError(err) + } +} + +func renderText(data []ResponseData) error { + generic, err := structuredData(data) + if err != nil { + return err + } + return writeText(os.Stdout, generic) +} + +// writeText renders the generic (JSON-shaped) value as plain text. +func writeText(w io.Writer, v interface{}) error { + switch t := v.(type) { + case nil: + return nil + case []interface{}: + if len(t) == 0 { + return nil + } + if textHasObjects(t) { + cols := textColumns(t) + for _, it := range t { + if _, err := fmt.Fprintln(w, textRow(it, cols)); err != nil { + return err + } + } + return nil + } + for _, it := range t { + if _, err := fmt.Fprintln(w, scalarToString(it)); err != nil { + return err + } + } + return nil + case map[string]interface{}: + keys := sortedKeys(t) + for _, k := range keys { + if _, err := fmt.Fprintf(w, "%s\t%s\n", k, scalarToString(t[k])); err != nil { + return err + } + } + return nil + default: + _, err := fmt.Fprintln(w, scalarToString(v)) + return err + } +} + +// textHasObjects reports whether any item in the list is an object; a list +// of objects is rendered as rows while a list of scalars is one per line. +func textHasObjects(items []interface{}) bool { + for _, it := range items { + if _, ok := it.(map[string]interface{}); ok { + return true + } + } + return false +} + +// textColumns returns the sorted union of keys across every object in the +// list. Sorted (rather than preference-ordered) keys keep the layout +// predictable for callers that cut columns with awk/cut, matching aws. +func textColumns(items []interface{}) []string { + seen := make(map[string]struct{}) + for _, it := range items { + obj, ok := it.(map[string]interface{}) + if !ok { + continue + } + for k := range obj { + seen[k] = struct{}{} + } + } + cols := make([]string, 0, len(seen)) + for k := range seen { + cols = append(cols, k) + } + sort.Strings(cols) + return cols +} + +// textRow renders one list item as a tab-separated row. A scalar mixed into +// a list of objects is printed as-is. +func textRow(it interface{}, cols []string) string { + obj, ok := it.(map[string]interface{}) + if !ok { + return scalarToString(it) + } + cells := make([]string, len(cols)) + for i, c := range cols { + cells[i] = scalarToString(obj[c]) + } + return strings.Join(cells, "\t") +} + +func sortedKeys(m map[string]interface{}) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/renderer/text_test.go b/internal/renderer/text_test.go new file mode 100644 index 0000000..1904082 --- /dev/null +++ b/internal/renderer/text_test.go @@ -0,0 +1,146 @@ +package renderer + +import ( + "bytes" + "testing" + + outputTable "github.com/latitudesh/lsh/internal/output/table" + "github.com/spf13/viper" +) + +func TestParseFormatText(t *testing.T) { + f, ok := ParseFormat("text") + if !ok || f != FormatText { + t.Fatalf("ParseFormat(text) = %q, %v", f, ok) + } + if !FormatText.IsStructured() { + t.Fatal("text must be structured so --query works") + } +} + +func TestTextRendererScalar(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + viper.Set("query", "[0].name") + + out := captureStdout(t, func() { TextRenderer{}.Render(sampleData()) }) + if out != "alpha\n" { + t.Fatalf("scalar: got %q, want %q", out, "alpha\n") + } +} + +func TestTextRendererScalarList(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + viper.Set("query", "[].id") + + out := captureStdout(t, func() { TextRenderer{}.Render(sampleData()) }) + if out != "srv_1\nsrv_2\n" { + t.Fatalf("scalar list: got %q", out) + } +} + +func TestTextRendererObjectList(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + + out := captureStdout(t, func() { TextRenderer{}.Render(sampleData()) }) + // Keys are sorted: id, name, status. + want := "srv_1\talpha\ton\nsrv_2\tbeta\toff\n" + if out != want { + t.Fatalf("object list: got %q, want %q", out, want) + } +} + +func TestTextRendererObject(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + viper.Set("query", "[0]") + + out := captureStdout(t, func() { TextRenderer{}.Render(sampleData()) }) + want := "id\tsrv_1\nname\talpha\nstatus\ton\n" + if out != want { + t.Fatalf("object: got %q, want %q", out, want) + } +} + +func TestWriteTextNestedValuesAreJSONEncoded(t *testing.T) { + var buf bytes.Buffer + v := []interface{}{ + map[string]interface{}{ + "name": "ci", + "buckets": []interface{}{map[string]interface{}{"bucket_name": "b-1", "permission": "rw"}}, + "size": float64(1024), + "missing": nil, + }, + } + if err := writeText(&buf, v); err != nil { + t.Fatal(err) + } + // Sorted columns: buckets, missing, name, size. + want := `[{"bucket_name":"b-1","permission":"rw"}]` + "\t\tci\t1024\n" + if buf.String() != want { + t.Fatalf("nested: got %q, want %q", buf.String(), want) + } +} + +func TestWriteTextEmptyAndNil(t *testing.T) { + var buf bytes.Buffer + if err := writeText(&buf, []interface{}{}); err != nil { + t.Fatal(err) + } + if err := writeText(&buf, nil); err != nil { + t.Fatal(err) + } + if buf.Len() != 0 { + t.Fatalf("expected no output, got %q", buf.String()) + } +} + +func TestGetStaticRendererText(t *testing.T) { + defer resetOutputViper() + viper.Set("output", "text") + viper.Set("output_explicit", true) + if _, ok := GetStaticRenderer().(TextRenderer); !ok { + t.Fatalf("GetStaticRenderer() = %T, want TextRenderer", GetStaticRenderer()) + } + if _, ok := GetRenderer().(TextRenderer); !ok { + t.Fatalf("GetRenderer() = %T, want TextRenderer", GetRenderer()) + } +} + +// zzKeyRow stands in for the access-key create result. +type zzKeyRow struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` +} + +func (r zzKeyRow) TableRow() outputTable.Row { return outputTable.Row{} } + +// TestQueryOnSingleResultNeedsIndex documents the shape --query operates on: a +// command's rows are always a list, so a bare identifier yields null and the +// documented pipes must index the element. The `access-keys create` examples in +// the README and the automation help topic depend on this. +func TestQueryOnSingleResultNeedsIndex(t *testing.T) { + defer viper.Set("query", "") + row := []ResponseData{zzKeyRow{AccessKeyID: "AK", SecretAccessKey: "s3cr3t"}} + cases := map[string]string{ + "secret_access_key": "", // a JMESPath identifier on a list + "[0].secret_access_key": "s3cr3t\n", // the form the docs must use + "[].secret_access_key": "s3cr3t\n", // projection, also fine + } + for expr, want := range cases { + viper.Set("query", expr) + generic, err := structuredData(row) + if err != nil { + t.Fatalf("%s: %v", expr, err) + } + var buf bytes.Buffer + if err := writeText(&buf, generic); err != nil { + t.Fatalf("%s: %v", expr, err) + } + if buf.String() != want { + t.Errorf("--query %q rendered %q, want %q", expr, buf.String(), want) + } + } +} diff --git a/internal/tui/confirm.go b/internal/tui/confirm.go index 1077029..9ce5f65 100644 --- a/internal/tui/confirm.go +++ b/internal/tui/confirm.go @@ -2,6 +2,7 @@ package tui import ( "fmt" + "os" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" @@ -64,7 +65,9 @@ func (m ConfirmModel) Result() bool { // RunConfirm é uma função helper func RunConfirm(message string) (bool, error) { - p := tea.NewProgram(NewConfirm(message)) + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewConfirm(message), tea.WithOutput(os.Stderr)) m, err := p.Run() if err != nil { return false, err diff --git a/internal/tui/list.go b/internal/tui/list.go index 92dc736..c0d799c 100644 --- a/internal/tui/list.go +++ b/internal/tui/list.go @@ -3,6 +3,7 @@ package tui import ( "fmt" "io" + "os" "github.com/charmbracelet/bubbles/list" tea "github.com/charmbracelet/bubbletea" @@ -134,7 +135,9 @@ func (m ListModel) Choice() string { // RunList é uma função helper para executar a lista func RunList(title string, items []string, descriptions []string) (string, error) { - p := tea.NewProgram(NewList(title, items, descriptions)) + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewList(title, items, descriptions), tea.WithOutput(os.Stderr)) m, err := p.Run() if err != nil { return "", err @@ -142,7 +145,7 @@ func RunList(title string, items []string, descriptions []string) (string, error if model, ok := m.(ListModel); ok { if model.Choice() == "" { - return "", fmt.Errorf("selection cancelled") + return "", ErrCanceled } return model.Choice(), nil } diff --git a/internal/tui/multiselect.go b/internal/tui/multiselect.go new file mode 100644 index 0000000..e6f517f --- /dev/null +++ b/internal/tui/multiselect.go @@ -0,0 +1,145 @@ +package tui + +import ( + "errors" + "fmt" + "os" + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +// MultiSelectModel is a compact checkbox list: ↑/↓ (or k/j) move, space toggles, +// enter confirms, a toggles all, esc/ctrl+c cancels. It mirrors the look of the +// single-select list (TitleStyle, SelectedStyle, HelpStyle) so multi-selection +// feels like the rest of the CLI. +type MultiSelectModel struct { + title string + items []string + descs []string + cursor int + selected map[int]bool + done bool + canceled bool +} + +// NewMultiSelect builds the model. Descriptions are optional (may be shorter +// than items). +func NewMultiSelect(title string, items, descriptions []string) MultiSelectModel { + return MultiSelectModel{ + title: title, + items: items, + descs: descriptions, + selected: make(map[int]bool), + } +} + +func (m MultiSelectModel) Init() tea.Cmd { return nil } + +func (m MultiSelectModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + if key, ok := msg.(tea.KeyMsg); ok { + switch key.String() { + case "ctrl+c", "esc", "q": + m.canceled = true + return m, tea.Quit + case "up", "k": + if m.cursor > 0 { + m.cursor-- + } + case "down", "j": + if m.cursor < len(m.items)-1 { + m.cursor++ + } + case " ", "x": + m.selected[m.cursor] = !m.selected[m.cursor] + case "a": + // Count the entries that are actually on: toggling an item off + // leaves a false value behind, so len(m.selected) overcounts. + on := 0 + for i := range m.items { + if m.selected[i] { + on++ + } + } + all := on < len(m.items) + m.selected = make(map[int]bool, len(m.items)) + if all { + for i := range m.items { + m.selected[i] = true + } + } + case "enter": + m.done = true + return m, tea.Quit + } + } + return m, nil +} + +func (m MultiSelectModel) View() string { + if m.done || m.canceled { + return "" + } + var b strings.Builder + b.WriteString(TitleStyle.Render(m.title) + "\n\n") + for i, it := range m.items { + cursor := " " + if i == m.cursor { + cursor = SelectedStyle.Render("> ") + } + box := "[ ]" + if m.selected[i] { + box = SelectedStyle.Render("[x]") + } + line := fmt.Sprintf("%s%s %s", cursor, box, it) + if i == m.cursor { + line = SelectedStyle.Render(line) + } + b.WriteString(line + "\n") + if i == m.cursor && i < len(m.descs) && m.descs[i] != "" { + b.WriteString(" " + lipgloss.NewStyle().Foreground(MutedColor).Render(m.descs[i]) + "\n") + } + } + b.WriteString("\n" + HelpStyle.Render("↑/↓: move • space: toggle • a: all • enter: confirm • esc: cancel") + "\n") + return b.String() +} + +// Selected returns the chosen indices in ascending order. +func (m MultiSelectModel) Selected() []int { + out := make([]int, 0, len(m.selected)) + for i := 0; i < len(m.items); i++ { + if m.selected[i] { + out = append(out, i) + } + } + return out +} + +// Canceled reports whether the user aborted the selection. +func (m MultiSelectModel) Canceled() bool { return m.canceled } + +// ErrCanceled is returned when the user aborts a prompt (esc/ctrl+c), so +// callers can exit "refused" instead of reporting a usage error. +var ErrCanceled = errors.New("selection cancelled") + +// RunMultiSelect shows the checkbox list and returns the selected indices, or +// ErrCanceled when the user aborts. An empty (but confirmed) selection returns +// no indices and no error. +func RunMultiSelect(title string, items, descriptions []string) ([]int, error) { + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewMultiSelect(title, items, descriptions), tea.WithOutput(os.Stderr)) + m, err := p.Run() + if err != nil { + return nil, err + } + model, ok := m.(MultiSelectModel) + if !ok { + return nil, fmt.Errorf("unexpected model type") + } + if model.Canceled() { + return nil, ErrCanceled + } + return model.Selected(), nil +} diff --git a/internal/tui/multiselect_test.go b/internal/tui/multiselect_test.go new file mode 100644 index 0000000..ee7a5e5 --- /dev/null +++ b/internal/tui/multiselect_test.go @@ -0,0 +1,100 @@ +package tui + +import ( + "reflect" + "testing" + + tea "github.com/charmbracelet/bubbletea" +) + +func key(s string) tea.KeyMsg { + switch s { + case " ": + return tea.KeyMsg{Type: tea.KeySpace} + case "enter": + return tea.KeyMsg{Type: tea.KeyEnter} + case "up": + return tea.KeyMsg{Type: tea.KeyUp} + case "down": + return tea.KeyMsg{Type: tea.KeyDown} + case "esc": + return tea.KeyMsg{Type: tea.KeyEsc} + } + return tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(s)} +} + +func drive(m MultiSelectModel, keys ...string) MultiSelectModel { + for _, k := range keys { + nm, _ := m.Update(key(k)) + m = nm.(MultiSelectModel) + } + return m +} + +func TestMultiSelectToggleAndConfirm(t *testing.T) { + m := NewMultiSelect("Pick", []string{"a", "b", "c"}, nil) + // select index 0, move to index 2 and select it, confirm. + m = drive(m, " ", "down", "down", " ", "enter") + if !m.done || m.Canceled() { + t.Fatalf("enter should confirm, not cancel") + } + if got := m.Selected(); !reflect.DeepEqual(got, []int{0, 2}) { + t.Fatalf("selected = %v, want [0 2]", got) + } +} + +func TestMultiSelectToggleAll(t *testing.T) { + m := NewMultiSelect("Pick", []string{"a", "b"}, nil) + m = drive(m, "a") // select all + if got := m.Selected(); !reflect.DeepEqual(got, []int{0, 1}) { + t.Fatalf("a should select all, got %v", got) + } + m = drive(m, "a") // clear all + if got := m.Selected(); len(got) != 0 { + t.Fatalf("a again should clear, got %v", got) + } +} + +func TestMultiSelectCancel(t *testing.T) { + m := NewMultiSelect("Pick", []string{"a", "b"}, nil) + m = drive(m, " ", "esc") + if !m.Canceled() { + t.Fatalf("esc should cancel") + } +} + +func TestMultiSelectCursorBounds(t *testing.T) { + m := NewMultiSelect("Pick", []string{"a", "b"}, nil) + m = drive(m, "up", "up") // cannot go below 0 + m = drive(m, "down", "down", "down") // cannot go past last + m = drive(m, " ", "enter") + if got := m.Selected(); !reflect.DeepEqual(got, []int{1}) { + t.Fatalf("cursor should clamp to last item, got %v", got) + } +} + +// TestMultiSelectToggleAllAfterDeselect covers the stale-entry bug: toggling an +// item off leaves a false value in the map, so counting entries instead of +// selections made "a" clear the list when it should have selected everything. +func TestMultiSelectToggleAllAfterDeselect(t *testing.T) { + var m tea.Model = NewMultiSelect("pick", []string{"a", "b", "c"}, nil) + // Turn every item on, then off again: three map entries, none selected. + for _, k := range []string{" ", "down", " ", "down", " "} { + m, _ = m.Update(key(k)) + } + for _, k := range []string{" ", "up", " ", "up", " "} { + m, _ = m.Update(key(k)) + } + if got := m.(MultiSelectModel).Selected(); len(got) != 0 { + t.Fatalf("precondition failed: %v still selected", got) + } + m, _ = m.Update(key("a")) + if got := m.(MultiSelectModel).Selected(); !reflect.DeepEqual(got, []int{0, 1, 2}) { + t.Errorf("'a' selected %v, want every item", got) + } + // Pressing it again with everything on clears the selection. + m, _ = m.Update(key("a")) + if got := m.(MultiSelectModel).Selected(); len(got) != 0 { + t.Errorf("'a' on a full selection left %v", got) + } +} diff --git a/internal/tui/numberinput.go b/internal/tui/numberinput.go index 3c5688d..62fccaa 100644 --- a/internal/tui/numberinput.go +++ b/internal/tui/numberinput.go @@ -2,6 +2,7 @@ package tui import ( "fmt" + "os" "strconv" "github.com/charmbracelet/bubbles/textinput" @@ -99,7 +100,9 @@ func (m NumberInputModel) Submitted() bool { // RunNumberInput is a helper function to run the number input func RunNumberInput(label, placeholder string) (int64, error) { - p := tea.NewProgram(NewNumberInput(label, placeholder)) + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewNumberInput(label, placeholder), tea.WithOutput(os.Stderr)) m, err := p.Run() if err != nil { return 0, err diff --git a/internal/tui/textinput.go b/internal/tui/textinput.go index 68c3b7d..73c6efe 100644 --- a/internal/tui/textinput.go +++ b/internal/tui/textinput.go @@ -2,6 +2,7 @@ package tui import ( "fmt" + "os" "github.com/charmbracelet/bubbles/textinput" tea "github.com/charmbracelet/bubbletea" @@ -72,7 +73,9 @@ func (m TextInputModel) Value() string { // RunTextInput is a helper function to run the input func RunTextInput(label, placeholder string) (string, error) { - p := tea.NewProgram(NewTextInput(label, placeholder)) + // The prompt is UI, not output: it goes to stderr so a command whose stdout + // is piped or redirected (-o json > file) is not corrupted by the widget. + p := tea.NewProgram(NewTextInput(label, placeholder), tea.WithOutput(os.Stderr)) m, err := p.Run() if err != nil { return "", err diff --git a/internal/wait/server.go b/internal/wait/server.go index 7bc5b94..eef2255 100644 --- a/internal/wait/server.go +++ b/internal/wait/server.go @@ -32,12 +32,12 @@ func ForServerState( ctx context.Context, client *sdk.Latitudesh, serverID string, - want, fail []components.ServerDataStatus, + want, fail []ServerStatus, requireTransition bool, o Options, - onStatus func(components.ServerDataStatus), + onStatus func(ServerStatus), opts ...operations.Option, -) (components.ServerDataStatus, error) { +) (ServerStatus, error) { if o.Timeout > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, o.Timeout) @@ -45,7 +45,7 @@ func ForServerState( } var ( - last components.ServerDataStatus + last ServerStatus lastErr error seenTransition bool ) @@ -95,8 +95,8 @@ func ForServerState( // operation acting on a server already in a target state (e.g. a reinstall on a // powered-on server) does not return before it actually begins. func decideServerState( - status components.ServerDataStatus, - want, fail []components.ServerDataStatus, + status ServerStatus, + want, fail []ServerStatus, requireTransition, seenTransition bool, ) (done bool, transitioned bool, err error) { inWant := containsStatus(want, status) @@ -131,14 +131,18 @@ func isTerminalAPIError(err error) bool { // serverStatus extracts the status from a GetServer response, tolerating any // nil link in the data → attributes → status chain. -func serverStatus(resp *operations.GetServerResponse) *components.ServerDataStatus { +func serverStatus(resp *operations.GetServerResponse) *ServerStatus { if resp == nil || resp.Server == nil || resp.Server.Data == nil || resp.Server.Data.Attributes == nil { return nil } - return resp.Server.Data.Attributes.Status + if resp.Server.Data.Attributes.Status == nil { + return nil + } + st := ServerStatus(*resp.Server.Data.Attributes.Status) + return &st } -func containsStatus(set []components.ServerDataStatus, s components.ServerDataStatus) bool { +func containsStatus(set []ServerStatus, s ServerStatus) bool { for _, v := range set { if v == s { return true diff --git a/internal/wait/status.go b/internal/wait/status.go new file mode 100644 index 0000000..888e5ca --- /dev/null +++ b/internal/wait/status.go @@ -0,0 +1,29 @@ +package wait + +// ServerStatus is the `status` attribute of a server as reported by the API. +// The SDK stopped generating a dedicated enum for it (v1.19.x exposes a plain +// string), so the known values live here to keep callers type-safe. +type ServerStatus string + +const ( + ServerStatusOn ServerStatus = "on" + ServerStatusOff ServerStatus = "off" + ServerStatusUnknown ServerStatus = "unknown" + ServerStatusDiskErasing ServerStatus = "disk_erasing" + ServerStatusDeploying ServerStatus = "deploying" + ServerStatusFailedDeployment ServerStatus = "failed_deployment" + ServerStatusRescueMode ServerStatus = "rescue_mode" +) + +// VirtualMachineStatus is the `status` attribute of a virtual machine as +// reported by the API (plain string in the SDK since v1.19.x). +type VirtualMachineStatus string + +const ( + VirtualMachineStatusRunning VirtualMachineStatus = "Running" + VirtualMachineStatusConfiguringNetwork VirtualMachineStatus = "Configuring network" + VirtualMachineStatusStarting VirtualMachineStatus = "Starting" + VirtualMachineStatusScheduling VirtualMachineStatus = "Scheduling" + VirtualMachineStatusScheduled VirtualMachineStatus = "Scheduled" + VirtualMachineStatusDestroying VirtualMachineStatus = "Destroying" +) diff --git a/internal/wait/vm.go b/internal/wait/vm.go index 49f9134..f907ecd 100644 --- a/internal/wait/vm.go +++ b/internal/wait/vm.go @@ -6,7 +6,6 @@ import ( "fmt" sdk "github.com/latitudesh/latitudesh-go-sdk" - "github.com/latitudesh/latitudesh-go-sdk/models/components" "github.com/latitudesh/latitudesh-go-sdk/models/operations" ) @@ -22,10 +21,10 @@ func ForVirtualMachineState( ctx context.Context, client *sdk.Latitudesh, vmID string, - want []components.VirtualMachineAttributesStatus, + want []VirtualMachineStatus, o Options, opts ...operations.Option, -) (components.VirtualMachineAttributesStatus, error) { +) (VirtualMachineStatus, error) { if o.Timeout > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, o.Timeout) @@ -33,12 +32,12 @@ func ForVirtualMachineState( } var ( - last components.VirtualMachineAttributesStatus + last VirtualMachineStatus lastErr error ) err := Poll(ctx, DefaultBackoff(), func(ctx context.Context) (bool, error) { - resp, err := client.VirtualMachines.Get(ctx, vmID, opts...) + resp, err := client.VirtualMachines.Get(ctx, vmID, nil, opts...) if err != nil { if isTerminalAPIError(err) { return false, err @@ -63,14 +62,18 @@ func ForVirtualMachineState( // virtualMachineStatus extracts the status from a ShowVirtualMachine response, // tolerating any nil link in the data → attributes → status chain. -func virtualMachineStatus(resp *operations.ShowVirtualMachineResponse) *components.VirtualMachineAttributesStatus { +func virtualMachineStatus(resp *operations.ShowVirtualMachineResponse) *VirtualMachineStatus { if resp == nil || resp.VirtualMachine == nil || resp.VirtualMachine.Data == nil || resp.VirtualMachine.Data.Attributes == nil { return nil } - return resp.VirtualMachine.Data.Attributes.Status + if resp.VirtualMachine.Data.Attributes.Status == nil { + return nil + } + st := VirtualMachineStatus(*resp.VirtualMachine.Data.Attributes.Status) + return &st } -func containsVMStatus(set []components.VirtualMachineAttributesStatus, s components.VirtualMachineAttributesStatus) bool { +func containsVMStatus(set []VirtualMachineStatus, s VirtualMachineStatus) bool { for _, v := range set { if v == s { return true diff --git a/internal/wait/wait_test.go b/internal/wait/wait_test.go index f3bdb7b..b2c52fb 100644 --- a/internal/wait/wait_test.go +++ b/internal/wait/wait_test.go @@ -108,7 +108,7 @@ func TestServerStatusNilSafe(t *testing.T) { if serverStatus(&operations.GetServerResponse{}) != nil { t.Error("serverStatus with nil Server should be nil") } - on := components.ServerDataStatusOn + on := string(ServerStatusOn) resp := &operations.GetServerResponse{ Server: &components.Server{ Data: &components.ServerData{ @@ -117,37 +117,37 @@ func TestServerStatusNilSafe(t *testing.T) { }, } got := serverStatus(resp) - if got == nil || *got != components.ServerDataStatusOn { + if got == nil || *got != ServerStatusOn { t.Errorf("serverStatus = %v, want on", got) } } func TestContainsStatus(t *testing.T) { - set := []components.ServerDataStatus{ - components.ServerDataStatusOn, - components.ServerDataStatusDeploying, + set := []ServerStatus{ + ServerStatusOn, + ServerStatusDeploying, } - if !containsStatus(set, components.ServerDataStatusOn) { + if !containsStatus(set, ServerStatusOn) { t.Error("expected on to be in set") } - if containsStatus(set, components.ServerDataStatusOff) { + if containsStatus(set, ServerStatusOff) { t.Error("did not expect off to be in set") } } func TestDecideServerState(t *testing.T) { - want := []components.ServerDataStatus{components.ServerDataStatusOn, components.ServerDataStatusOff} - fail := []components.ServerDataStatus{components.ServerDataStatusFailedDeployment} + want := []ServerStatus{ServerStatusOn, ServerStatusOff} + fail := []ServerStatus{ServerStatusFailedDeployment} t.Run("want hit without requireTransition is done", func(t *testing.T) { - done, _, err := decideServerState(components.ServerDataStatusOn, want, fail, false, false) + done, _, err := decideServerState(ServerStatusOn, want, fail, false, false) if err != nil || !done { t.Fatalf("done=%v err=%v, want done=true err=nil", done, err) } }) t.Run("want hit is gated until a transition is seen", func(t *testing.T) { - done, transitioned, err := decideServerState(components.ServerDataStatusOn, want, fail, true, false) + done, transitioned, err := decideServerState(ServerStatusOn, want, fail, true, false) if err != nil || done { t.Fatalf("done=%v err=%v, want done=false err=nil (gated)", done, err) } @@ -157,18 +157,18 @@ func TestDecideServerState(t *testing.T) { }) t.Run("transition state flips seenTransition then want succeeds", func(t *testing.T) { - _, transitioned, _ := decideServerState(components.ServerDataStatusDeploying, want, fail, true, false) + _, transitioned, _ := decideServerState(ServerStatusDeploying, want, fail, true, false) if !transitioned { t.Fatal("deploying should set seenTransition") } - done, _, err := decideServerState(components.ServerDataStatusOn, want, fail, true, transitioned) + done, _, err := decideServerState(ServerStatusOn, want, fail, true, transitioned) if err != nil || !done { t.Fatalf("done=%v err=%v, want done=true after transition", done, err) } }) t.Run("fail state reported immediately even without transition (H1 regression)", func(t *testing.T) { - done, _, err := decideServerState(components.ServerDataStatusFailedDeployment, want, fail, true, false) + done, _, err := decideServerState(ServerStatusFailedDeployment, want, fail, true, false) if done { t.Error("fail state must not be 'done'") } diff --git a/main.go b/main.go index f5506e7..4030f82 100644 --- a/main.go +++ b/main.go @@ -4,13 +4,15 @@ import ( "os" "github.com/latitudesh/lsh/cmd" + "github.com/latitudesh/lsh/internal/exitcode" ) func main() { // Propagate command failures as a non-zero exit code. Cobra already prints // the error (commands set SilenceUsage), so we only need to set the status - // — scripts, CI and AI agents rely on it. + // — scripts, CI and AI agents rely on it. Commands that attach an explicit + // exit code (see internal/exitcode) get it; everything else exits with 1. if _, err := cmd.Execute(); err != nil { - os.Exit(1) + os.Exit(exitcode.Of(err)) } }