Skip to content

Commit 0993c6d

Browse files
kucukaslankucukaslan
authored
command/storage: add versioning support (#475)
This commit adds versioning support to the s5cmd. Added --all-versions flag to ls, rm, du and select subcommands to apply operation on(/over) all versions of the objects. Added --version-id flag to cat, cp/mv, rm, du and select subcommands to apply operation on(/over) a specific versions of the object. Added bucket-version command to configure bucket versioning. Bucket name alone returns the bucket versioning status of the bucket. Bucket versioning can be configured with set flag which only accepts. Added --raw flag to cat and select subcommands. It disables the wildcard operations. Note: Google Cloud Storage uses a different approach for versioning. So with current implementation, s5cmd cannot use or retrieve generation numbers . However, bucket-version command and du command with all-versions flag works as expected since they do not use version ids. Fixes: #218 Fixes: #386 Fixes: #539
1 parent f0ce87d commit 0993c6d

27 files changed

Lines changed: 1348 additions & 152 deletions

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,12 @@
1111
- Added `--profile` flag to allow users to specify a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html). ([#353](https://github.com/peak/s5cmd/issues/353))
1212
- Added `--credentials-file` flag to allow users to specify path for the AWS credentials file instead of using the [default location](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-where).
1313
- Added `bench.py` script under new `benchmark` folder to compare performances of two different builds of s5cmd. ([#471](https://github.com/peak/s5cmd/pull/471))
14+
- Added `--all-versions` flag to `ls`, `rm`, `du` and `select` subcommands to apply operation on(/over) all versions of the objects. ([#475](https://github.com/peak/s5cmd/pull/475))
15+
- Added `--version-id` flag to `cat`, `cp`/`mv`, `rm`, `du` and `select` subcommands to apply operation on(/over) a specific versions of the object. ([#475](https://github.com/peak/s5cmd/pull/475))
16+
- Added `bucket-version` command to configure bucket versioning. Bucket name
17+
alone returns the bucket versioning status of the bucket. Bucket versioning can
18+
be configured with `set` flag. ([#475](https://github.com/peak/s5cmd/pull/475))
19+
- Added `--raw` flag to `cat` and `select` subcommands. It disables the wildcard operations. ([#475](https://github.com/peak/s5cmd/pull/475))
1420

1521
#### Improvements
1622
- Disable AWS SDK logger if log level is not `trace`. ([##460](https://github.com/peak/s5cmd/pull/460))

‎command/app.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ func Commands() []*cli.Command {
207207
NewRunCommand(),
208208
NewSyncCommand(),
209209
NewVersionCommand(),
210+
NewBucketVersionCommand(),
210211
}
211212
}
212213

‎command/bucket_version.go‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package command
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
8+
"github.com/urfave/cli/v2"
9+
10+
"github.com/peak/s5cmd/log"
11+
"github.com/peak/s5cmd/storage"
12+
"github.com/peak/s5cmd/storage/url"
13+
"github.com/peak/s5cmd/strutil"
14+
)
15+
16+
var bucketVersionHelpTemplate = `Name:
17+
{{.HelpName}} - {{.Usage}}
18+
19+
Usage:
20+
{{.HelpName}} [options] s3://bucketname
21+
22+
Options:
23+
{{range .VisibleFlags}}{{.}}
24+
{{end}}
25+
Examples:
26+
1. Get bucket versioning status of a bucket
27+
> s5cmd {{.HelpName}} s3://bucketname
28+
29+
2. Enable bucket versioning for the bucket
30+
> s5cmd {{.HelpName}} --set Enabled s3://bucketname
31+
32+
3. Suspend bucket versioning for the bucket
33+
> s5cmd {{.HelpName}} --set Suspended s3://bucketname
34+
`
35+
36+
func NewBucketVersionCommand() *cli.Command {
37+
cmd := &cli.Command{
38+
Name: "bucket-version",
39+
CustomHelpTemplate: bucketVersionHelpTemplate,
40+
HelpName: "bucket-version",
41+
Usage: "configure bucket versioning",
42+
Flags: []cli.Flag{
43+
&cli.GenericFlag{
44+
Name: "set",
45+
Value: &EnumValue{
46+
Enum: []string{"Suspended", "Enabled"},
47+
Default: "",
48+
ConditionFunction: strings.EqualFold,
49+
},
50+
Usage: "set versioning status of bucket: (Suspended, Enabled)",
51+
},
52+
},
53+
Before: func(ctx *cli.Context) error {
54+
if err := checkNumberOfArguments(ctx, 1, 1); err != nil {
55+
printError(commandFromContext(ctx), ctx.Command.Name, err)
56+
return err
57+
}
58+
return nil
59+
},
60+
Action: func(c *cli.Context) error {
61+
status := c.String("set")
62+
63+
fullCommand := commandFromContext(c)
64+
65+
bucket, err := url.New(c.Args().First())
66+
if err != nil {
67+
printError(fullCommand, c.Command.Name, err)
68+
return err
69+
}
70+
71+
return BucketVersion{
72+
src: bucket,
73+
op: c.Command.Name,
74+
fullCommand: fullCommand,
75+
76+
status: status,
77+
storageOpts: NewStorageOpts(c),
78+
}.Run(c.Context)
79+
},
80+
}
81+
cmd.BashComplete = getBashCompleteFn(cmd, true, true)
82+
return cmd
83+
}
84+
85+
type BucketVersion struct {
86+
src *url.URL
87+
op string
88+
fullCommand string
89+
90+
status string
91+
storageOpts storage.Options
92+
}
93+
94+
func (v BucketVersion) Run(ctx context.Context) error {
95+
client, err := storage.NewRemoteClient(ctx, &url.URL{}, v.storageOpts)
96+
if err != nil {
97+
printError(v.fullCommand, v.op, err)
98+
return err
99+
}
100+
101+
if v.status != "" {
102+
v.status = strutil.CapitalizeFirstRune(v.status)
103+
104+
err := client.SetBucketVersioning(ctx, v.status, v.src.Bucket)
105+
if err != nil {
106+
printError(v.fullCommand, v.op, err)
107+
return err
108+
}
109+
msg := BucketVersionMessage{
110+
Bucket: v.src.Bucket,
111+
Status: v.status,
112+
isSet: true,
113+
}
114+
log.Info(msg)
115+
return nil
116+
}
117+
118+
status, err := client.GetBucketVersioning(ctx, v.src.Bucket)
119+
if err != nil {
120+
printError(v.fullCommand, v.op, err)
121+
return err
122+
}
123+
124+
msg := BucketVersionMessage{
125+
Bucket: v.src.Bucket,
126+
Status: status,
127+
isSet: false,
128+
}
129+
log.Info(msg)
130+
return nil
131+
}
132+
133+
type BucketVersionMessage struct {
134+
Bucket string `json:"bucket"`
135+
Status string `json:"status"`
136+
isSet bool
137+
}
138+
139+
func (v BucketVersionMessage) String() string {
140+
if v.isSet {
141+
return fmt.Sprintf("Bucket versioning for %q is set to %q", v.Bucket, v.Status)
142+
}
143+
if v.Status != "" {
144+
return fmt.Sprintf("Bucket versioning for %q is %q", v.Bucket, v.Status)
145+
}
146+
return fmt.Sprintf("%q is an unversioned bucket", v.Bucket)
147+
}
148+
149+
func (v BucketVersionMessage) JSON() string {
150+
return strutil.JSON(v)
151+
}

‎command/cat.go‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,26 @@ Options:
2525
Examples:
2626
1. Print a remote object's content to stdout
2727
> s5cmd {{.HelpName}} s3://bucket/prefix/object
28+
29+
2. Print specific version of a remote object's content to stdout
30+
> s5cmd {{.HelpName}} --version-id VERSION_ID s3://bucket/prefix/object
2831
`
2932

3033
func NewCatCommand() *cli.Command {
3134
cmd := &cli.Command{
32-
Name: "cat",
33-
HelpName: "cat",
34-
Usage: "print remote object content",
35+
Name: "cat",
36+
HelpName: "cat",
37+
Usage: "print remote object content",
38+
Flags: []cli.Flag{
39+
&cli.BoolFlag{
40+
Name: "raw",
41+
Usage: "disable the wildcard operations, useful with filenames that contains glob characters",
42+
},
43+
&cli.StringFlag{
44+
Name: "version-id",
45+
Usage: "use the specified version of an object",
46+
},
47+
},
3548
CustomHelpTemplate: catHelpTemplate,
3649
Before: func(c *cli.Context) error {
3750
err := validateCatCommand(c)
@@ -43,9 +56,11 @@ func NewCatCommand() *cli.Command {
4356
Action: func(c *cli.Context) (err error) {
4457
defer stat.Collect(c.Command.FullName(), &err)()
4558

46-
src, err := url.New(c.Args().Get(0))
4759
op := c.Command.Name
4860
fullCommand := commandFromContext(c)
61+
62+
src, err := url.New(c.Args().Get(0), url.WithVersion(c.String("version-id")),
63+
url.WithRaw(c.Bool("raw")))
4964
if err != nil {
5065
printError(fullCommand, op, err)
5166
return err
@@ -102,8 +117,8 @@ func validateCatCommand(c *cli.Context) error {
102117
return fmt.Errorf("expected only one argument")
103118
}
104119

105-
src, err := url.New(c.Args().Get(0))
106-
120+
src, err := url.New(c.Args().Get(0), url.WithVersion(c.String("version-id")),
121+
url.WithRaw(c.Bool("raw")))
107122
if err != nil {
108123
return err
109124
}
@@ -119,5 +134,10 @@ func validateCatCommand(c *cli.Context) error {
119134
if src.IsWildcard() {
120135
return fmt.Errorf("remote source %q can not contain glob characters", src)
121136
}
137+
138+
if err := checkVersioningWithGoogleEndpoint(c); err != nil {
139+
return err
140+
}
141+
122142
return nil
123143
}

0 commit comments

Comments
 (0)