Skip to content

Commit

Permalink
OlderThan & NewerThan configured for Days, Hours and Minutes (#2614)
Browse files Browse the repository at this point in the history 
Previously OlderThan and NewerThan was configured for day
which has been changed to Days, Hours and minutes.

Also, changed the `flag` in `find` command to be `newer-than` and `older-than` to be consistent with other commands.
  • Loading branch information
@sinhaashish @kannappanr
sinhaashish authored and kannappanr committed Mar 4, 2019
1 parent c63c573 commit 1761b33
Show file tree
Hide file tree
Showing 10 changed files with 351 additions and 181 deletions.
32 changes: 16 additions & 16 deletions cmd/cp-main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@ var (
Name: "recursive, r",
Usage: "copy recursively",
},
cli.IntFlag{
cli.StringFlag{
Name: "older-than",
Usage: "copy object(s) older than N days",
Usage: "copy objects older than L days, M hours and N minutes",
},
cli.IntFlag{
cli.StringFlag{
Name: "newer-than",
Usage: "copy object(s) newer than N days",
Usage: "copy objects newer than L days, M hours and N minutes",
},
cli.StringFlag{
Name: "storage-class, sc",
Expand Down Expand Up @@ -93,11 +93,11 @@ EXAMPLES:
4. Copy a bucket recursively from aliased Amazon S3 cloud storage to local filesystem on Windows.
$ {{.HelpName}} --recursive s3\documents\2014\ C:\Backups\2014
5. Copy files older than 7 days from Minio cloud storage to Amazon S3 cloud storage.
$ {{.HelpName}} --older-than 7 play/mybucket/burningman2011/ s3/mybucket/
5. Copy files older than 7 days and 10 hours from Minio cloud storage to Amazon S3 cloud storage.
$ {{.HelpName}} --older-than 7d10h play/mybucket/burningman2011/ s3/mybucket/
6. Copy files newer than 7 days from Minio cloud storage to a local path.
$ {{.HelpName}} --newer-than 7 play/mybucket/burningman2011/ ~/latest/
6. Copy files newer than 7 days and 10 hours from Minio cloud storage to a local path.
$ {{.HelpName}} --newer-than 7d10h play/mybucket/burningman2011/ ~/latest/
7. Copy an object with name containing unicode characters to Amazon S3 cloud storage.
$ {{.HelpName}} 本語 s3/andoria/
Expand Down Expand Up @@ -217,8 +217,8 @@ func doPrepareCopyURLs(session *sessionV8, trapCh <-chan bool, cancelCopy contex
// Access recursive flag inside the session header.
isRecursive := session.Header.CommandBoolFlags["recursive"]

olderThan := session.Header.CommandIntFlags["older-than"]
newerThan := session.Header.CommandIntFlags["newer-than"]
olderThan := session.Header.CommandStringFlags["older-than"]
newerThan := session.Header.CommandStringFlags["newer-than"]
encryptKeys := session.Header.CommandStringFlags["encrypt-key"]
encrypt := session.Header.CommandStringFlags["encrypt"]
encKeyDB, err := parseAndValidateEncryptionKeys(encryptKeys, encrypt)
Expand Down Expand Up @@ -260,12 +260,12 @@ func doPrepareCopyURLs(session *sessionV8, trapCh <-chan bool, cancelCopy contex
}

// Skip objects older than --older-than parameter if specified
if olderThan > 0 && isOlder(cpURLs.SourceContent, olderThan) {
if olderThan != "" && isOlder(cpURLs.SourceContent.Time, olderThan) {
continue
}

// Skip objects newer than --newer-than parameter if specified
if newerThan > 0 && isNewer(cpURLs.SourceContent, newerThan) {
if newerThan != "" && isNewer(cpURLs.SourceContent.Time, newerThan) {
continue
}

Expand Down Expand Up @@ -452,8 +452,8 @@ func mainCopy(ctx *cli.Context) error {
console.SetColor("Copy", color.New(color.FgGreen, color.Bold))

recursive := ctx.Bool("recursive")
olderThan := ctx.Int("older-than")
newerThan := ctx.Int("newer-than")
olderThan := ctx.String("older-than")
newerThan := ctx.String("newer-than")
storageClass := ctx.String("storage-class")
sseKeys := os.Getenv("MC_ENCRYPT_KEY")
if key := ctx.String("encrypt-key"); key != "" {
Expand All @@ -464,8 +464,8 @@ func mainCopy(ctx *cli.Context) error {
session := newSessionV8()
session.Header.CommandType = "cp"
session.Header.CommandBoolFlags["recursive"] = recursive
session.Header.CommandIntFlags["older-than"] = olderThan
session.Header.CommandIntFlags["newer-than"] = newerThan
session.Header.CommandStringFlags["older-than"] = olderThan
session.Header.CommandStringFlags["newer-than"] = newerThan
session.Header.CommandStringFlags["storage-class"] = storageClass
session.Header.CommandStringFlags["encrypt-key"] = sseKeys
session.Header.CommandStringFlags["encrypt"] = sse
Expand Down
35 changes: 15 additions & 20 deletions cmd/find-main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package cmd

import (
"strings"
"time"

humanize "github.com/dustin/go-humanize"
"github.com/fatih/color"
Expand All @@ -43,12 +42,12 @@ var (
Usage: "find object names matching wildcard pattern",
},
cli.StringFlag{
Name: "newer",
Usage: "match all objects newer than specified time in units (see UNITS)",
Name: "newer-than",
Usage: "match all objects newer than L days, M hours and N minutes",
},
cli.StringFlag{
Name: "older",
Usage: "match all objects older than specified time in units (see UNITS)",
Name: "older-than",
Usage: "match all objects older than L days, M hours and N minutes",
},
cli.StringFlag{
Name: "path",
Expand Down Expand Up @@ -103,10 +102,8 @@ UNITS
units, so that "gi" refers to "gibibyte" or "GiB". A "b" at the end is
also accepted. Without suffixes the unit is bytes.
--older, --newer flags accept the suffixes "d", "w", "m" and "y" to refer
to units of days, weeks, months and years respectively. With the standard
rate of conversion being 7 days in 1 week, 30 days in 1 month, and 365
days in one year.
--older-than, --newer-than flags accept the string for days, hours and minutes
i.e. 1d2h30m states 1 day, 2 hours and 30 minutes.
FORMAT
Support string substitutions with special interpretations for following keywords.
Expand Down Expand Up @@ -147,9 +144,9 @@ EXAMPLES:
08. Find all objects created in the last week under "s3/bucket".
$ {{.HelpName}} s3/bucket --newer 1w
09. Find all objects which were created more than 6 months ago, and exclude the ones with ".jpg"
09. Find all objects which were created are older than 2 days, 5 hours and 10 minutes and exclude the ones with ".jpg"
extension under "s3".
$ {{.HelpName}} s3 --older 6m --ignore "*.jpg"
$ {{.HelpName}} s3 --older-than 2d5h10m --ignore "*.jpg"
10. List all objects up to 3 levels sub-directory deep under "s3/bucket".
$ {{.HelpName}} s3/bucket --maxdepth 3
Expand Down Expand Up @@ -197,8 +194,8 @@ type findContext struct {
regexPattern string
maxDepth uint
printFmt string
olderThan time.Time
newerThan time.Time
olderThan string
newerThan string
largerSize uint64
smallerSize uint64
watch bool
Expand Down Expand Up @@ -232,15 +229,13 @@ func mainFind(ctx *cli.Context) error {
clnt, err := newClient(args[0])
fatalIf(err.Trace(args...), "Unable to initialize `"+args[0]+"`.")

var olderThan, newerThan time.Time
var olderThan, newerThan string

if ctx.String("older") != "" {
olderThan, err = parseTime(ctx.String("older"))
fatalIf(err.Trace(ctx.String("older")), "Unable to parse input time.")
if ctx.String("older-than") != "" {
olderThan = ctx.String("older-than")
}
if ctx.String("newer") != "" {
newerThan, err = parseTime(ctx.String("newer"))
fatalIf(err.Trace(ctx.String("newer")), "Unable to parse input time.")
if ctx.String("newer-than") != "" {
newerThan = ctx.String("newer-than")
}

// Use 'e' to indicate Go error, this is a convention followed in `mc`. For probe.Error we call it
Expand Down
45 changes: 4 additions & 41 deletions cmd/find.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,11 +396,11 @@ func matchFind(ctx *findContext, fileContent contentMessage) (match bool) {
if match && ctx.regexPattern != "" {
match = regexMatch(ctx.regexPattern, path)
}
if match && !ctx.olderThan.IsZero() {
match = fileContent.Time.Before(ctx.olderThan)
if match && ctx.olderThan != "" {
match = !isOlder(fileContent.Time, ctx.olderThan)
}
if match && !ctx.newerThan.IsZero() {
match = fileContent.Time.After(ctx.newerThan) || fileContent.Time.Equal(ctx.newerThan)
if match && ctx.newerThan != "" {
match = !isNewer(fileContent.Time, ctx.newerThan)
}
if match && ctx.largerSize > 0 {
match = int64(ctx.largerSize) < fileContent.Size
Expand All @@ -411,43 +411,6 @@ func matchFind(ctx *findContext, fileContent contentMessage) (match bool) {
return match
}

// parseTime - parses input value into a corresponding time value in
// time.Time by adding the input time duration to local UTC time.Now().
func parseTime(duration string) (time.Time, *probe.Error) {
if duration == "" {
return time.Time{}, errInvalidArgument().Trace(duration)
}

conversion := map[string]int{
"d": 1,
"w": 7,
"m": 30,
"y": 365,
}

// Parse the incoming pattern if its exact number.
i, e := strconv.Atoi(duration)
if e != nil {
// If cant parse as regular string look for
// a conversion multiplier, either d,w,m,y.
p := duration[len(duration)-1:]
i, e = strconv.Atoi(duration[:len(duration)-1])
if e != nil {
// if we still cant parse, user input is invalid, return error.
return time.Time{}, probe.NewError(e)
}
i = i * conversion[strings.ToLower(p)]
}

now := UTCNow()

// Find all time in which the time in which the object was just created is after the current time
t := time.Date(now.Year(), now.Month(), now.Day()-i, now.Hour(), now.Minute(), 0, 0, time.UTC)

// if we reach this line, user has passed a valid alphanumeric string
return t, nil
}

// 7 days in seconds.
var defaultSevenDays = time.Duration(604800) * time.Second

Expand Down
55 changes: 2 additions & 53 deletions cmd/find_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,13 @@ func TestMatchFind(t *testing.T) {
clnt: &s3Client{
targetURL: &clientURL{},
},
olderThan: time.Unix(12000, 0).UTC(),
olderThan: "1d",
},
&findContext{
clnt: &s3Client{
targetURL: &clientURL{},
},
newerThan: time.Unix(12000, 0).UTC(),
newerThan: "32000d",
},
&findContext{
clnt: &s3Client{
Expand Down Expand Up @@ -337,57 +337,6 @@ func TestFindMatch(t *testing.T) {
}
}

// Tests for parsing time layout.
func TestParseTime(t *testing.T) {
testCases := []struct {
value string
success bool
}{
// Parses 1 day successfully.
{
value: "1d",
success: true,
},
// Parses 1 week successfully.
{
value: "1w",
success: true,
},
// Parses 1 year successfully.
{
value: "1y",
success: true,
},
// Parses 2 months successfully.
{
value: "2m",
success: true,
},
// Failure to parse "xd".
{
value: "xd",
success: false,
},
// Failure to parse empty string.
{
value: "",
success: false,
},
}
for i, testCase := range testCases {
pt, err := parseTime(testCase.value)
if err != nil && testCase.success {
t.Errorf("Test: %d, Expected to be successful, but found error %s, for time value %s", i+1, err, testCase.value)
}
if pt.IsZero() && testCase.success {
t.Errorf("Test: %d, Expected time to be non zero, but found zero time for time value %s", i+1, testCase.value)
}
if err == nil && !testCase.success {
t.Errorf("Test: %d, Expected error but found to be successful for time value %s", i+1, testCase.value)
}
}
}

// Tests string substitution function.
func TestStringReplace(t *testing.T) {
testCases := []struct {
Expand Down
32 changes: 14 additions & 18 deletions cmd/mirror-main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,13 +71,13 @@ var (
Name: "exclude",
Usage: "exclude object(s) that match specified object name pattern",
},
cli.IntFlag{
cli.StringFlag{
Name: "older-than",
Usage: "filter object(s) older than N days",
Usage: "filter object(s) older than L days, M hours and N minutes",
},
cli.IntFlag{
cli.StringFlag{
Name: "newer-than",
Usage: "filter object(s) newer than N days",
Usage: "filter object(s) newer than L days, M hours and N minutes",
},
cli.StringFlag{
Name: "storage-class, sc",
Expand Down Expand Up @@ -117,8 +117,8 @@ EXAMPLES:
2. Mirror a local folder recursively to Amazon S3 cloud storage.
$ {{.HelpName}} backup/ s3/archive
3. Only mirror files that are newer than 7 days to Amazon S3 cloud storage.
$ {{.HelpName}} --newer-than 7 backup/ s3/archive
3. Only mirror files that are newer than 7 days, 10 hours and 30 minutes to Amazon S3 cloud storage.
$ {{.HelpName}} --newer-than "7d10h30m" backup/ s3/archive
4. Mirror a bucket from aliased Amazon S3 cloud storage to a folder on Windows.
$ {{.HelpName}} s3\documents\2014\ C:\backup\2014
Expand All @@ -139,10 +139,10 @@ EXAMPLES:
$ {{.HelpName}} --exclude ".*" --exclude "*.temp" s3/test ~/test
9. Mirror objects newer than 10 days from bucket test to a local folder.
$ {{.HelpName}} --newer-than=10 s3/test ~/localfolder
$ {{.HelpName}} --newer-than 10d s3/test ~/localfolder
10. Mirror objects older than 30 days from Amazon S3 bucket test to a local folder.
$ {{.HelpName}} --older-than=30 s3/test ~/test
$ {{.HelpName}} --older-than 30d s3/test ~/test
11. Mirror server encrypted objects from Minio cloud storage to a bucket on Amazon S3 cloud storage
$ {{.HelpName}} --encrypt-key "minio/photos=32byteslongsecretkeymustbegiven1,s3/archive=32byteslongsecretkeymustbegiven2" minio/photos/ s3/archive/
Expand Down Expand Up @@ -185,7 +185,7 @@ type mirrorJob struct {
targetURL string

isFake, isRemove, isOverwrite, isWatch bool
olderThan, newerThan int
olderThan, newerThan string
storageClass string

excludeOptions []string
Expand Down Expand Up @@ -522,16 +522,12 @@ func (mj *mirrorJob) startMirror(ctx context.Context, cancelMirror context.Cance
}

if sURLs.SourceContent != nil {
// Skip objects older than --older-than parameter if specified
if mj.olderThan > 0 && isOlder(sURLs.SourceContent, mj.olderThan) {
if mj.olderThan != "" && isOlder(sURLs.SourceContent.Time, mj.olderThan) {
continue
}

// Skip objects newer than --newer-than parameter if specified
if mj.newerThan > 0 && isNewer(sURLs.SourceContent, mj.newerThan) {
if mj.newerThan != "" && isNewer(sURLs.SourceContent.Time, mj.newerThan) {
continue
}

// copy
totalBytes += sURLs.SourceContent.Size
}
Expand Down Expand Up @@ -614,7 +610,7 @@ func (mj *mirrorJob) mirror(ctx context.Context, cancelMirror context.CancelFunc
}
}

func newMirrorJob(srcURL, dstURL string, isFake, isRemove, isOverwrite, isWatch bool, excludeOptions []string, olderThan, newerThan int, storageClass string, encKeyDB map[string][]prefixSSEPair) *mirrorJob {
func newMirrorJob(srcURL, dstURL string, isFake, isRemove, isOverwrite, isWatch bool, excludeOptions []string, olderThan, newerThan string, storageClass string, encKeyDB map[string][]prefixSSEPair) *mirrorJob {
mj := mirrorJob{
trapCh: signalTrap(os.Interrupt, syscall.SIGTERM, syscall.SIGKILL),
m: new(sync.Mutex),
Expand Down Expand Up @@ -691,8 +687,8 @@ func runMirror(srcURL, dstURL string, ctx *cli.Context, encKeyDB map[string][]pr
isOverwrite,
ctx.Bool("watch"),
ctx.StringSlice("exclude"),
ctx.Int("older-than"),
ctx.Int("newer-than"),
ctx.String("older-than"),
ctx.String("newer-than"),
ctx.String("storage-class"),
encKeyDB)

Expand Down
Loading

0 comments on commit 1761b33

Please sign in to comment.