From cf50c16fe4652bdf5c8ee805f822f4ba0432b3ef Mon Sep 17 00:00:00 2001 From: Kenny Stuart Date: Thu, 5 Nov 2020 13:11:43 +0000 Subject: [PATCH 01/21] preliminary support for unixfs Mode and ModTime This commit introduces initial Mode and ModTime support for single filesystem files and webfiles. The ipfs add options --preserve-mode and --preserve-mtime are used to store the original mode and last modified time of the file being added, the options --mode, --mtime and --mtime-nsecs are used to store custom values. A custom value of 0 is a no-op. The preserve flags and custom options are mutually exclusive, if both are provided the custom options take precedence. --- core/commands/add.go | 29 +++++++++++++++++++++++++++++ core/commands/get.go | 33 +++++++++++++++++++++++++++------ core/coreapi/unixfs.go | 5 ++++- core/coreunix/add.go | 17 +++++++++++++++++ 4 files changed, 77 insertions(+), 7 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 1c4c8603004..905a5927af3 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -45,6 +45,12 @@ const ( hashOptionName = "hash" inlineOptionName = "inline" inlineLimitOptionName = "inline-limit" + + preserveModeOptionName = "preserve-mode" + preserveMtimeOptionName = "preserve-mtime" + modeOptionName = "mode" + mtimeOptionName = "mtime" + mtimeNsecsOptionName = "mtime-nsecs" ) const adderOutChanSize = 8 @@ -140,6 +146,11 @@ only-hash, and progress/status related flags) will change the final hash. cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"), cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"), cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32), + cmds.BoolOption(preserveModeOptionName, "Apply permissions to created UnixFS entries"), + cmds.BoolOption(preserveMtimeOptionName, "Apply modification time to created UnixFS entries"), + cmds.UintOption(modeOptionName, "File mode to apply to created UnixFS entries"), + cmds.Int64Option(mtimeOptionName, "Modification time in seconds before or after the Unix Epoch to apply to created UnixFS entries"), + cmds.UintOption(mtimeNsecsOptionName, "Modification time fraction in nanoseconds"), }, PreRun: func(req *cmds.Request, env cmds.Environment) error { quiet, _ := req.Options[quietOptionName].(bool) @@ -180,6 +191,11 @@ only-hash, and progress/status related flags) will change the final hash. hashFunStr, _ := req.Options[hashOptionName].(string) inline, _ := req.Options[inlineOptionName].(bool) inlineLimit, _ := req.Options[inlineLimitOptionName].(int) + preserveMode, _ := req.Options[preserveModeOptionName].(bool) + preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool) + mode, _ := req.Options[modeOptionName].(uint) + mtime, _ := req.Options[mtimeOptionName].(int64) + mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint) hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { @@ -213,6 +229,19 @@ only-hash, and progress/status related flags) will change the final hash. options.Unixfs.Progress(progress), options.Unixfs.Silent(silent), + + options.Unixfs.PreserveMode(preserveMode), + options.Unixfs.PreserveMtime(preserveMtime), + } + + if mode != 0 { + opts = append(opts, options.Unixfs.Mode(os.FileMode(mode))) + } + + if mtime != 0 { + opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs))) + } else if mtimeNsecs != 0 { + fmt.Printf("option %s ignored as no valid %s value provided\n", mtimeNsecsOptionName, mtimeOptionName) } if cidVerSet { diff --git a/core/commands/get.go b/core/commands/get.go index 7087c71ca3d..7d7b92f034a 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -262,7 +262,22 @@ func (i *identityWriteCloser) Close() error { return nil } +func updateFileMeta(fn files.Node, filename string) error { + if t := fn.ModTime(); !t.IsZero() { + if err := os.Chtimes(filename, t, t); err != nil { + return err + } + } + if mode := fn.Mode(); mode != 0 { + if err := os.Chmod(filename, mode); err != nil { + return err + } + } + return nil +} + func fileArchive(f files.Node, name string, archive bool, compression int) (io.Reader, error) { + var err error = nil cleaned := gopath.Clean(name) _, filename := gopath.Split(cleaned) @@ -285,14 +300,20 @@ func fileArchive(f files.Node, name string, archive bool, compression int) (io.R return nil, err } - closeGzwAndPipe := func() { + closeGzwAndPipe := func() error { if err := maybeGzw.Close(); checkErrAndClosePipe(err) { - return + return nil } if err := bufw.Flush(); checkErrAndClosePipe(err) { - return + return nil } pipew.Close() // everything seems to be ok. + if !archive { + if err := updateFileMeta(f, filename); err != nil { + return err + } + } + return nil } if !archive && compression != gzip.NoCompression { @@ -306,7 +327,7 @@ func fileArchive(f files.Node, name string, archive bool, compression int) (io.R if _, err := io.Copy(maybeGzw, r); checkErrAndClosePipe(err) { return } - closeGzwAndPipe() // everything seems to be ok + err = closeGzwAndPipe() // everything seems to be ok }() } else { // the case for 1. archive, and 2. not archived and not compressed, in which tar is used anyway as a transport format @@ -323,11 +344,11 @@ func fileArchive(f files.Node, name string, archive bool, compression int) (io.R return } w.Close() // close tar writer - closeGzwAndPipe() // everything seems to be ok + err = closeGzwAndPipe() // everything seems to be ok }() } - return piper, nil + return piper, err } func newMaybeGzWriter(w io.Writer, compression int) (io.WriteCloser, error) { diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 53d13a3a497..24337d64af7 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/ipfs/go-ipfs/core" - "github.com/ipfs/go-ipfs/core/coreunix" blockservice "github.com/ipfs/go-blockservice" @@ -134,6 +133,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options fileAdder.RawLeaves = settings.RawLeaves fileAdder.NoCopy = settings.NoCopy fileAdder.CidBuilder = prefix + fileAdder.PreserveMode = settings.PreserveMode + fileAdder.PreserveMtime = settings.PreserveMtime + fileAdder.FileMode = settings.Mode + fileAdder.FileMtime = settings.Mtime switch settings.Layout { case options.BalancedLayout: diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 68be08056ff..a3e3b603742 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "io" + "os" gopath "path" "strconv" + "time" "github.com/ipfs/go-cid" bstore "github.com/ipfs/go-ipfs-blockstore" @@ -79,6 +81,11 @@ type Adder struct { tempRoot cid.Cid CidBuilder cid.Builder liveNodes uint64 + + PreserveMode bool + PreserveMtime bool + FileMode os.FileMode + FileMtime time.Time } func (adder *Adder) mfsRoot() (*mfs.Root, error) { @@ -113,6 +120,8 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) { Maxlinks: ihelper.DefaultLinksPerBlock, NoCopy: adder.NoCopy, CidBuilder: adder.CidBuilder, + FileMode: adder.FileMode, + FileMtime: adder.FileMtime, } db, err := params.New(chnk) @@ -396,6 +405,14 @@ func (adder *Adder) addFile(path string, file files.File) error { } } + if adder.PreserveMtime { + adder.FileMtime = file.ModTime() + } + + if adder.PreserveMode { + adder.FileMode = file.Mode() + } + dagnode, err := adder.add(reader) if err != nil { return err From d93f1efa51d74d690aa68ee9795d5b4166ac088a Mon Sep 17 00:00:00 2001 From: Kenny Stuart Date: Tue, 10 Nov 2020 19:33:02 +0000 Subject: [PATCH 02/21] [ipfs get] pass the output path to FileArchive This is needed for when ipfs get is used with the --output option to ensure the target for applying metadata is known. --- core/commands/get.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/commands/get.go b/core/commands/get.go index 7d7b92f034a..37e3fb6fbed 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -85,7 +85,7 @@ may also specify the level of compression by specifying '-l=<1-9>'. res.SetLength(uint64(size)) archive, _ := req.Options[archiveOptionName].(bool) - reader, err := fileArchive(file, p.String(), archive, cmplvl) + reader, err := fileArchive(file, p.String(), getOutPath(req), archive, cmplvl) if err != nil { return err } @@ -276,7 +276,7 @@ func updateFileMeta(fn files.Node, filename string) error { return nil } -func fileArchive(f files.Node, name string, archive bool, compression int) (io.Reader, error) { +func fileArchive(f files.Node, name string, outpath string, archive bool, compression int) (io.Reader, error) { var err error = nil cleaned := gopath.Clean(name) _, filename := gopath.Split(cleaned) @@ -309,7 +309,7 @@ func fileArchive(f files.Node, name string, archive bool, compression int) (io.R } pipew.Close() // everything seems to be ok. if !archive { - if err := updateFileMeta(f, filename); err != nil { + if err := updateFileMeta(f, outpath); err != nil { return err } } @@ -343,7 +343,7 @@ func fileArchive(f files.Node, name string, archive bool, compression int) (io.R if err := w.WriteFile(f, filename); checkErrAndClosePipe(err) { return } - w.Close() // close tar writer + w.Close() // close tar writer err = closeGzwAndPipe() // everything seems to be ok }() } From 0d883e9ea4a188d9376e5d20ae45c6a1c798fc54 Mon Sep 17 00:00:00 2001 From: Kenny Stuart Date: Tue, 10 Nov 2020 19:51:07 +0000 Subject: [PATCH 03/21] [sharness] add tests for preserving/setting file mode and modification time --- test/sharness/t0047-add-mode-mtime.sh | 86 +++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100755 test/sharness/t0047-add-mode-mtime.sh diff --git a/test/sharness/t0047-add-mode-mtime.sh b/test/sharness/t0047-add-mode-mtime.sh new file mode 100755 index 00000000000..4621c9647a1 --- /dev/null +++ b/test/sharness/t0047-add-mode-mtime.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +test_description="Test storing and retrieving mode and mtime" + +. lib/test-lib.sh + +test_init_ipfs + +TESTFILE=mountdir/testfile.txt + +PRESERVE_MTIME=1604320482 +PRESERVE_MODE=0640 +HASH_PRESERVE_MODE=QmQLgxypSNGNFTuUPGCecq6dDEjb6hNB5xSyVmP3cEuNtq +HASH_PRESERVE_MTIME=QmbfPiiFH3mM9qCCsCL1BdDfDBxeCrVXfiiSEUYdjPzeQf +HASH_PRESERVE_MODE_AND_MTIME=QmT858Bjy215QhtJKBiEueLZq2VDoiiJ6dDP2RNAkPMT7f + +CUSTOM_MTIME=1603539720 +CUSTOM_MTIME_NSECS=54321 +CUSTOM_MODE=0764 +HASH_CUSTOM_MODE=QmchD3BN8TQ3RW6jPLxSaNkqvfuj7syKhzTRmL4EpyY1Nz +HASH_CUSTOM_MTIME=QmS7vmVxe9YgbWm2CjEKBTYnhhrUjy6cbnDTqe4tkPhu32 +HASH_CUSTOM_MTIME_NSECS=QmaKH8H5rXBUBCX4vdxi7ktGQEL7wejV7L9rX2qpZjwncz +HASH_CUSTOM_MODE_AND_MTIME=QmUkxrtBA8tPjwCYz1HrsoRfDz6NgKut3asVeHVQNH4C8L + +test_expect_success "can preserve file mode" ' + touch "$TESTFILE" && + chmod $PRESERVE_MODE "$TESTFILE" && + HASH=$(ipfs add -q --hash=sha2-256 --preserve-mode "$TESTFILE") && + test "$HASH_PRESERVE_MODE" = "$HASH" +' + +test_expect_success "can preserve file modification time" ' + touch -m -d @$PRESERVE_MTIME "$TESTFILE" && + HASH=$(ipfs add -q --hash=sha2-256 --preserve-mtime "$TESTFILE") && + test "$HASH_PRESERVE_MTIME" = "$HASH" +' + +test_expect_success "can set file mode" ' + touch "$TESTFILE" && + chmod 0600 "$TESTFILE" && + HASH=$(ipfs add -q --hash=sha2-256 --mode=$CUSTOM_MODE "$TESTFILE") && + test "$HASH_CUSTOM_MODE" = "$HASH" +' + +test_expect_success "can set file mtime" ' + touch -m -t 202011021234.42 "$TESTFILE" && + HASH=$(ipfs add -q --hash=sha2-256 --mtime=$CUSTOM_MTIME "$TESTFILE") && + test "$HASH_CUSTOM_MTIME" = "$HASH" +' + +test_expect_success "can set file mtime nanoseconds" ' + touch -m -t 202011021234.42 "$TESTFILE" && + HASH=$(ipfs add -q --hash=sha2-256 --mtime=$CUSTOM_MTIME --mtime-nsecs=$CUSTOM_MTIME_NSECS "$TESTFILE") && + test "$HASH_CUSTOM_MTIME_NSECS" = "$HASH" +' + +test_expect_success "can preserve file mode and modification time" ' + touch -m -d @$PRESERVE_MTIME "$TESTFILE" && + chmod $PRESERVE_MODE "$TESTFILE" && + HASH=$(ipfs add -q --hash=sha2-256 --preserve-mode --preserve-mtime "$TESTFILE") && + test "$HASH_PRESERVE_MODE_AND_MTIME" = "$HASH" +' + +test_expect_success "can set file mode and mtime" ' + touch -m -t 202011021234.42 "$TESTFILE" && + chmod 0600 "$TESTFILE" && + HASH=$(ipfs add -q --hash=sha2-256 --mode=$CUSTOM_MODE --mtime=$CUSTOM_MTIME --mtime-nsecs=$CUSTOM_MTIME_NSECS "$TESTFILE") && + test "$HASH_CUSTOM_MODE_AND_MTIME" = "$HASH" +' + +test_expect_success "can get preserved mode and mtime" ' + OUTFILE="mountdir/$HASH_PRESERVE_MODE_AND_MTIME" && + ipfs get -o "$OUTFILE" $HASH_PRESERVE_MODE_AND_MTIME && + test "$PRESERVE_MODE:$PRESERVE_MTIME" = "$(stat -c "0%a:%Y" "$OUTFILE")" +' + +test_expect_success "can get custom mode and mtime" ' + OUTFILE="mountdir/$HASH_CUSTOM_MODE_AND_MTIME" && + ipfs get -o "$OUTFILE" $HASH_CUSTOM_MODE_AND_MTIME && + TIMESTAMP=$(date +%s%N --date="$(stat -c "%y" $OUTFILE)") && + MODETIME=$(stat -c "0%a:$TIMESTAMP" "$OUTFILE") && + printf -v EXPECTED "$CUSTOM_MODE:$CUSTOM_MTIME%09d" $CUSTOM_MTIME_NSECS && + test "$EXPECTED" = "$MODETIME" +' + +test_done From 9d257fe2609577dbfc9945cf796d3b9a4e03d139 Mon Sep 17 00:00:00 2001 From: Kenny Stuart Date: Sat, 3 Apr 2021 15:12:08 +0100 Subject: [PATCH 04/21] add file mode and mtime support for file stat --- core/commands/files.go | 70 ++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/core/commands/files.go b/core/commands/files.go index db498d815ff..245f954d81c 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -4,15 +4,15 @@ import ( "context" "errors" "fmt" + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/core/commands/cmdenv" "io" "os" gopath "path" "sort" + "strconv" "strings" - "github.com/ipfs/go-ipfs/core" - "github.com/ipfs/go-ipfs/core/commands/cmdenv" - "github.com/dustin/go-humanize" bservice "github.com/ipfs/go-blockservice" cid "github.com/ipfs/go-cid" @@ -101,6 +101,9 @@ type statOutput struct { WithLocality bool `json:",omitempty"` Local bool `json:",omitempty"` SizeLocal uint64 `json:",omitempty"` + Mode string `json:",omitempty"` + Mtime int64 `json:",omitempty"` + MtimeNsecs int `json:",omitempty"` } const ( @@ -252,28 +255,7 @@ func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) { switch n := nd.(type) { case *dag.ProtoNode: - d, err := ft.FSNodeFromBytes(n.Data()) - if err != nil { - return nil, err - } - - var ndtype string - switch d.Type() { - case ft.TDirectory, ft.THAMTShard: - ndtype = "directory" - case ft.TFile, ft.TMetadata, ft.TRaw: - ndtype = "file" - default: - return nil, fmt.Errorf("unrecognized node type: %s", d.Type()) - } - - return &statOutput{ - Hash: enc.Encode(c), - Blocks: len(nd.Links()), - Size: d.FileSize(), - CumulativeSize: cumulsize, - Type: ndtype, - }, nil + return statProtoNode(n, enc, c, cumulsize) case *dag.RawNode: return &statOutput{ Hash: enc.Encode(c), @@ -287,6 +269,42 @@ func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) { } } +func statProtoNode(n *dag.ProtoNode, enc cidenc.Encoder, cid cid.Cid, cumulsize uint64) (*statOutput, error) { + d, err := ft.FSNodeFromBytes(n.Data()) + if err != nil { + return nil, err + } + + stat := statOutput{ + Hash: enc.Encode(cid), + Blocks: len(n.Links()), + Size: d.FileSize(), + CumulativeSize: cumulsize, + } + + switch d.Type() { + case ft.TDirectory, ft.THAMTShard: + stat.Type = "directory" + case ft.TFile, ft.TMetadata, ft.TRaw: + stat.Type = "file" + default: + return nil, fmt.Errorf("unrecognized node type: %s", d.Type()) + } + + if mode := d.Mode(); mode != 0 { + stat.Mode = "0" + strconv.FormatUint(uint64(mode), 8) + } + + if mt := d.ModTime(); !mt.IsZero() { + stat.Mtime = mt.Unix() + if ns := mt.Nanosecond(); ns > 0 { + stat.MtimeNsecs = ns + } + } + + return &stat, nil +} + func walkBlock(ctx context.Context, dagserv ipld.DAGService, nd ipld.Node) (bool, uint64, error) { // Start with the block data size sizeLocal := uint64(len(nd.RawData())) @@ -419,7 +437,7 @@ var filesLsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List directories in the local mutable namespace.", ShortDescription: ` -List directories in the local mutable namespace (works on both IPFS and MFS paths). +List directories in the local mutable namespace. Examples: From 659e7091fed167bbf1b3daf510dae41352994598 Mon Sep 17 00:00:00 2001 From: Kenny Stuart Date: Sat, 3 Apr 2021 15:19:23 +0100 Subject: [PATCH 05/21] [ipfs add] emit mtime and mode in AddEvent --- core/commands/add.go | 65 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 905a5927af3..f348378748d 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -6,7 +6,9 @@ import ( "io" "os" "path" + "strconv" "strings" + "time" "github.com/ipfs/go-ipfs/core/commands/cmdenv" @@ -21,11 +23,35 @@ import ( // ErrDepthLimitExceeded indicates that the max depth has been exceeded. var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded") +type TimeParts struct { + t *time.Time +} + +func (t TimeParts) MarshalJSON() ([]byte, error) { + return t.t.MarshalJSON() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +// The time is expected to be a quoted string in RFC 3339 format. +func (t *TimeParts) UnmarshalJSON(data []byte) (err error) { + // Fractional seconds are handled implicitly by Parse. + tt, err := time.Parse("\"2006-01-02T15:04:05Z\"", string(data)) + *t = TimeParts{&tt} + return +} + +func AsJSON(t *time.Time) string { + return fmt.Sprintf("{\"secs\":%v,\"nsecs\":%v}", t.Unix(), t.Nanosecond()) +} + type AddEvent struct { - Name string - Hash string `json:",omitempty"` - Bytes int64 `json:",omitempty"` - Size string `json:",omitempty"` + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` + Mode string `json:",omitempty"` + Mtime int64 `json:",omitempty"` + MtimeNsecs int `json:",omitempty"` } const ( @@ -290,12 +316,31 @@ only-hash, and progress/status related flags) will change the final hash. output.Name = path.Join(addit.Name(), output.Name) } - if err := res.Emit(&AddEvent{ - Name: output.Name, - Hash: h, - Bytes: output.Bytes, - Size: output.Size, - }); err != nil { + output.Mode = addit.Node().Mode() + output.Mtime = addit.Node().ModTime().Unix() + output.MtimeNsecs = addit.Node().ModTime().Nanosecond() + + addEvent := AddEvent{ + Name: output.Name, + Hash: h, + Bytes: output.Bytes, + Size: output.Size, + Mtime: output.Mtime, + MtimeNsecs: output.MtimeNsecs, + } + + if output.Mode != 0 { + addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8) + } + + if output.Mtime > 0 { + addEvent.Mtime = output.Mtime + if output.MtimeNsecs > 0 { + addEvent.MtimeNsecs = output.MtimeNsecs + } + } + + if err := res.Emit(&addEvent); err != nil { return err } } From e1d987911341cb9f88210155cc131c0accd8e3db Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Sat, 28 Aug 2021 11:32:29 +0100 Subject: [PATCH 06/21] add optional mode and mtime to ipfs files stat output --- core/commands/files.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/commands/files.go b/core/commands/files.go index 245f954d81c..c27cdf0ef38 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -216,6 +216,17 @@ var filesStatCmd = &cmds.Command{ ) } + if out.Mode != "" { + fmt.Fprintf(w, "Mode: %s\n", out.Mode) + } + + if out.Mtime > 0 { + fmt.Fprintf(w, "Mtime: %d\n", out.Mtime) + if out.MtimeNsecs > 0 { + fmt.Fprintf(w, "MtimeNsecs: %d\n", out.MtimeNsecs) + } + } + return nil }), }, From 16b57a66a365d12ded53c14acd6bde67f2a5b82f Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Thu, 2 Sep 2021 21:18:30 +0100 Subject: [PATCH 07/21] humanize mode and mtime in ipfs files stat output --- core/commands/files.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/core/commands/files.go b/core/commands/files.go index cffdeec8897..8a70a6a92c9 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -8,8 +8,8 @@ import ( "os" gopath "path" "sort" - "strconv" "strings" + "time" humanize "github.com/dustin/go-humanize" "github.com/ipfs/go-ipfs/core" @@ -103,7 +103,7 @@ type statOutput struct { WithLocality bool `json:",omitempty"` Local bool `json:",omitempty"` SizeLocal uint64 `json:",omitempty"` - Mode string `json:",omitempty"` + Mode uint32 `json:",omitempty"` Mtime int64 `json:",omitempty"` MtimeNsecs int `json:",omitempty"` } @@ -218,15 +218,13 @@ var filesStatCmd = &cmds.Command{ ) } - if out.Mode != "" { - fmt.Fprintf(w, "Mode: %s\n", out.Mode) + if out.Mode != 0 { + fmt.Fprintf(w, "Mode: %s\n", os.FileMode(out.Mode).String()) } if out.Mtime > 0 { - fmt.Fprintf(w, "Mtime: %d\n", out.Mtime) - if out.MtimeNsecs > 0 { - fmt.Fprintf(w, "MtimeNsecs: %d\n", out.MtimeNsecs) - } + fmt.Fprintf(w, "Mtime: %s\n", + time.Unix(out.Mtime, int64(out.MtimeNsecs)).Format("2 Jan 2006, 15:04:05 MST")) } return nil @@ -305,7 +303,7 @@ func statProtoNode(n *dag.ProtoNode, enc cidenc.Encoder, cid cid.Cid, cumulsize } if mode := d.Mode(); mode != 0 { - stat.Mode = "0" + strconv.FormatUint(uint64(mode), 8) + stat.Mode = uint32(mode) } if mt := d.ModTime(); !mt.IsZero() { From 45713c2c5a02db7d19878383c4a36bcd6a31ec20 Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Mon, 6 Sep 2021 17:48:07 +0100 Subject: [PATCH 08/21] don't set modification time when none provided --- core/commands/add.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 33fd51b41d3..39d0f2c6e29 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -317,8 +317,10 @@ only-hash, and progress/status related flags) will change the final hash. } output.Mode = addit.Node().Mode() - output.Mtime = addit.Node().ModTime().Unix() - output.MtimeNsecs = addit.Node().ModTime().Nanosecond() + if ts := addit.Node().ModTime(); !ts.IsZero() { + output.Mtime = addit.Node().ModTime().Unix() + output.MtimeNsecs = addit.Node().ModTime().Nanosecond() + } addEvent := AddEvent{ Name: output.Name, From 9172385e758f3c4ab8c33b59dadfe9fec88d4822 Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Mon, 6 Sep 2021 18:02:08 +0100 Subject: [PATCH 09/21] fix/enhance files stat template --- core/commands/files.go | 20 +++++++++++++++++++- test/sharness/t0250-files-api.sh | 4 ++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/commands/files.go b/core/commands/files.go index 8a70a6a92c9..97d995ab439 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -8,6 +8,7 @@ import ( "os" gopath "path" "sort" + "strconv" "strings" "time" @@ -113,7 +114,9 @@ const ( Size: CumulativeSize: ChildBlocks: -Type: ` +Type: +Mode: +Mtime: ` filesFormatOptionName = "format" filesSizeOptionName = "size" filesWithLocalOptionName = "with-local" @@ -202,11 +205,26 @@ var filesStatCmd = &cmds.Command{ Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *statOutput) error { s, _ := statGetFormatOptions(req) + + var mode os.FileMode + if out.Mode != 0 { + mode = os.FileMode(out.Mode) + } + var mtime string + if out.Mtime > 0 { + mtime = time.Unix(out.Mtime, int64(out.MtimeNsecs)).Format("2 Jan 2006, 15:04:05 MST") + } + s = strings.Replace(s, "", out.Hash, -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Size), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.CumulativeSize), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Blocks), -1) s = strings.Replace(s, "", out.Type, -1) + s = strings.Replace(s, "", mode.String(), -1) + s = strings.Replace(s, "", mtime, -1) + s = strings.Replace(s, "", strconv.FormatInt(out.Mtime, 10), -1) + s = strings.Replace(s, "", strconv.Itoa(out.MtimeNsecs), -1) + s = strings.Replace(s, "", strconv.FormatInt(int64(out.Mode), 8), -1) fmt.Fprintln(w, s) diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index e2162cdf79e..a457ea8dc31 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -230,6 +230,8 @@ test_files_api() { echo "Size: 4" >> file1stat_expect && echo "ChildBlocks: 0" >> file1stat_expect && echo "Type: file" >> file1stat_expect && + echo "Mode: ----------" >> file1stat_expect && + echo "Mtime: " >> file1stat_expect && test_cmp file1stat_expect file1stat_actual ' @@ -243,6 +245,8 @@ test_files_api() { echo "Size: 4" >> file1stat_expect && echo "ChildBlocks: 0" >> file1stat_expect && echo "Type: file" >> file1stat_expect && + echo "Mode: ----------" >> file1stat_expect && + echo "Mtime: " >> file1stat_expect && test_cmp file1stat_expect file1stat_actual ' From a6f5f09e7bf08d20d0f303f144fac3d8d5e6734e Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Mon, 6 Sep 2021 18:04:02 +0100 Subject: [PATCH 10/21] add ipfs files chmod and touch --- core/commands/commands_test.go | 2 + core/commands/files.go | 95 ++++++++++++++++++++++++--- test/sharness/t0047-add-mode-mtime.sh | 28 ++++++++ 3 files changed, 116 insertions(+), 9 deletions(-) diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 5c076307413..5c2fa90d066 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -139,6 +139,8 @@ func TestCommands(t *testing.T) { "/files/rm", "/files/stat", "/files/write", + "/files/chmod", + "/files/touch", "/filestore", "/filestore/dups", "/filestore/ls", diff --git a/core/commands/files.go b/core/commands/files.go index 97d995ab439..de60dbab056 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -82,6 +82,8 @@ operations. "rm": filesRmCmd, "flush": filesFlushCmd, "chcid": filesChcidCmd, + "touch": filesTouchCmd, + "chmod": filesChmodCmd, }, } @@ -236,15 +238,6 @@ var filesStatCmd = &cmds.Command{ ) } - if out.Mode != 0 { - fmt.Fprintf(w, "Mode: %s\n", os.FileMode(out.Mode).String()) - } - - if out.Mtime > 0 { - fmt.Fprintf(w, "Mtime: %s\n", - time.Unix(out.Mtime, int64(out.MtimeNsecs)).Format("2 Jan 2006, 15:04:05 MST")) - } - return nil }), }, @@ -1303,3 +1296,87 @@ func getParentDir(root *mfs.Root, dir string) (*mfs.Directory, error) { } return pdir, nil } + +var filesChmodCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Change mode permissions", + ShortDescription: ` +The mode argument must be specified in unix numeric notation. + + $ ipfs files chmod 0644 /foo + $ ipfs files stat /foo + ... + Type: file + Mode: -rw-r--r-- + ... +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("mode", true, false, "mode to apply to node"), + cmds.StringArg("path", true, false, "Path to target node"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + path, err := checkPath(req.Arguments[1]) + if err != nil { + return err + } + + mode, err := strconv.ParseInt(req.Arguments[0], 8, 32) + if err != nil { + return err + } + + return mfs.Chmod(nd.FilesRoot, path, os.FileMode(mode)) + }, +} + +var filesTouchCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Change file modification times.", + ShortDescription: ` +Examples: + + # set modification time to now. + $ ipfs files touch /foo + + # set a custom modification time. + $ ipfs files touch --mtime=1630937926 /foo + +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("path", true, false, "Path of target to update."), + }, + Options: []cmds.Option{ + cmds.Int64Option(mtimeOptionName, "Modification time in seconds before or since the Unix Epoch to apply to created UnixFS entries."), + cmds.UintOption(mtimeNsecsOptionName, "Modification time fraction in nanoseconds"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + path, err := checkPath(req.Arguments[0]) + if err != nil { + return err + } + + mtime, _ := req.Options[mtimeOptionName].(int64) + nsecs, _ := req.Options[mtimeNsecsOptionName].(uint) + + var ts time.Time + if mtime != 0 { + ts = time.Unix(int64(mtime), int64(nsecs)).UTC() + } else { + ts = time.Now().UTC() + } + + return mfs.Touch(nd.FilesRoot, path, ts) + }, +} diff --git a/test/sharness/t0047-add-mode-mtime.sh b/test/sharness/t0047-add-mode-mtime.sh index 4621c9647a1..c0652fd3395 100755 --- a/test/sharness/t0047-add-mode-mtime.sh +++ b/test/sharness/t0047-add-mode-mtime.sh @@ -83,4 +83,32 @@ test_expect_success "can get custom mode and mtime" ' test "$EXPECTED" = "$MODETIME" ' +test_expect_success "can change file mode" ' + HASH=$(echo testfile | ipfs add -q --mode=0600) && + ipfs files cp "/ipfs/$HASH" /chmod1 && + ipfs files chmod 444 /chmod1 && + HASH=$(ipfs files stat /chmod1|head -1) && + ipfs get -o mountdir/chmod1 $HASH && + test $(stat -c "%a" mountdir/chmod1) = 444 +' + +test_expect_success "can touch file modification time" ' + NOW=$(date +%s) && + HASH=$(echo testfile | ipfs add -q --mtime=$NOW) && + ipfs files cp "/ipfs/$HASH" /touch1 && + sleep 1 && + ipfs files touch /touch1 && + HASH=$(ipfs files stat /touch1|head -1) && + ipfs get -o mountdir/touch1 $HASH && + test $(stat -c "%Y" mountdir/touch1) -gt $NOW +' + +test_expect_success "can change file modification time and nanoseconds" ' + echo test|ipfs files write --create /touch2 && + EXPECTED=$(date --date="yesterday" +%s) && + ipfs files touch --mtime=$EXPECTED --mtime-nsecs=55567 /touch2 && + test $(ipfs files stat --format="" /touch2) -eq $EXPECTED && + test $(ipfs files stat --format="" /touch2) -eq 55567 +' + test_done From d7e61f0da4aea2621a236612f7c03928922b34d9 Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Thu, 9 Sep 2021 22:02:26 +0100 Subject: [PATCH 11/21] make files stat json output compatible with js-ipfs again --- core/commands/files.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/core/commands/files.go b/core/commands/files.go index de60dbab056..6026bb0e3d0 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -2,6 +2,7 @@ package commands import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -111,6 +112,40 @@ type statOutput struct { MtimeNsecs int `json:",omitempty"` } +func (s *statOutput) MarshalJSON() ([]byte, error) { + type so statOutput + out := &struct { + *so + Mode string `json:",omitempty"` + }{so: (*so)(s)} + + if s.Mode != 0 { + out.Mode = fmt.Sprintf("%04d", s.Mode) + } + return json.Marshal(out) +} + +func (s *statOutput) UnmarshalJSON(data []byte) error { + var err error + type so statOutput + tmp := &struct { + *so + Mode string `json:",omitempty"` + }{so: (*so)(s)} + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + if tmp.Mode != "" { + mode, err := strconv.ParseUint(tmp.Mode, 8, 32) + if err == nil { + s.Mode = uint32(mode) + } + } + return err +} + const ( defaultStatFormat = ` Size: From 46e06608772c9e361c9f6a063b2d3a2420485a1c Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Sat, 8 Jan 2022 22:48:52 +0000 Subject: [PATCH 12/21] fix bad format of JSON statOutput.Mode --- core/commands/files.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/files.go b/core/commands/files.go index 788486dc475..f2f45e18a16 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -120,7 +120,7 @@ func (s *statOutput) MarshalJSON() ([]byte, error) { }{so: (*so)(s)} if s.Mode != 0 { - out.Mode = fmt.Sprintf("%04d", s.Mode) + out.Mode = fmt.Sprintf("%04o", s.Mode) } return json.Marshal(out) } From 29860a1979048742a7267fb66103d90baeacaed8 Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Sun, 26 Jun 2022 17:36:00 +0100 Subject: [PATCH 13/21] remove leftover code --- core/commands/add.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index a712c1be20e..b59c9edd211 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -40,10 +40,6 @@ func (t *TimeParts) UnmarshalJSON(data []byte) (err error) { return } -func AsJSON(t *time.Time) string { - return fmt.Sprintf("{\"secs\":%v,\"nsecs\":%v}", t.Unix(), t.Nanosecond()) -} - type AddEvent struct { Name string Hash string `json:",omitempty"` From e40cf4111ee383ce99c8717017367bc45d7bb43f Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Sun, 26 Jun 2022 17:43:20 +0100 Subject: [PATCH 14/21] preserve mode/mtime on directories and symlinks --- core/commands/files.go | 2 +- core/commands/get.go | 29 +-- core/coreunix/add.go | 55 ++++-- test/sharness/t0047-add-mode-mtime.sh | 271 +++++++++++++++++--------- 4 files changed, 228 insertions(+), 129 deletions(-) diff --git a/core/commands/files.go b/core/commands/files.go index f2f45e18a16..1c21ef060d0 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -342,7 +342,7 @@ func statProtoNode(n *dag.ProtoNode, enc cidenc.Encoder, cid cid.Cid, cumulsize switch d.Type() { case ft.TDirectory, ft.THAMTShard: stat.Type = "directory" - case ft.TFile, ft.TMetadata, ft.TRaw: + case ft.TFile, ft.TSymlink, ft.TMetadata, ft.TRaw: stat.Type = "file" default: return nil, fmt.Errorf("unrecognized node type: %s", d.Type()) diff --git a/core/commands/get.go b/core/commands/get.go index 778b6e23262..d4a452beb69 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -1,10 +1,12 @@ package commands import ( + tar2 "archive/tar" "bufio" "compress/gzip" "errors" "fmt" + "github.com/ipfs/tar-utils" "io" "os" gopath "path" @@ -18,7 +20,6 @@ import ( cmds "github.com/ipfs/go-ipfs-cmds" files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/ipfs/tar-utils" ) var ErrInvalidCompressionLevel = errors.New("compression level must be between 1 and 9") @@ -262,20 +263,6 @@ func (i *identityWriteCloser) Close() error { return nil } -func updateFileMeta(fn files.Node, filename string) error { - if t := fn.ModTime(); !t.IsZero() { - if err := os.Chtimes(filename, t, t); err != nil { - return err - } - } - if mode := fn.Mode(); mode != 0 { - if err := os.Chmod(filename, mode); err != nil { - return err - } - } - return nil -} - func fileArchive(f files.Node, name string, outpath string, archive bool, compression int) (io.Reader, error) { var err error = nil cleaned := gopath.Clean(name) @@ -308,11 +295,6 @@ func fileArchive(f files.Node, name string, outpath string, archive bool, compre return nil } pipew.Close() // everything seems to be ok. - if !archive { - if err := updateFileMeta(f, outpath); err != nil { - return err - } - } return nil } @@ -338,9 +320,14 @@ func fileArchive(f files.Node, name string, outpath string, archive bool, compre return nil, err } + // if not creating an archive set the format to PAX in order to preserve nanoseconds + if !archive { + w.SetFormat(tar2.FormatPAX) + } + go func() { // write all the nodes recursively - if err := w.WriteFile(f, filename); checkErrAndClosePipe(err) { + if err := w.WriteNode(f, filename); checkErrAndClosePipe(err) { return } w.Close() // close tar writer diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 9241453ea85..b5bea59d073 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -115,13 +115,13 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) { } params := ihelper.DagBuilderParams{ - Dagserv: adder.bufferedDS, - RawLeaves: adder.RawLeaves, - Maxlinks: ihelper.DefaultLinksPerBlock, - NoCopy: adder.NoCopy, - CidBuilder: adder.CidBuilder, - FileMode: adder.FileMode, - FileMtime: adder.FileMtime, + Dagserv: adder.bufferedDS, + RawLeaves: adder.RawLeaves, + Maxlinks: ihelper.DefaultLinksPerBlock, + NoCopy: adder.NoCopy, + CidBuilder: adder.CidBuilder, + FileMode: adder.FileMode, + FileModTime: adder.FileMtime, } db, err := params.New(chnk) @@ -350,6 +350,14 @@ func (adder *Adder) addFileNode(ctx context.Context, path string, file files.Nod return err } + if adder.PreserveMtime { + adder.FileMtime = file.ModTime() + } + + if adder.PreserveMode { + adder.FileMode = file.Mode() + } + if adder.liveNodes >= liveCacheSize { // TODO: A smarter cache that uses some sort of lru cache with an eviction handler mr, err := adder.mfsRoot() @@ -382,6 +390,18 @@ func (adder *Adder) addSymlink(path string, l *files.Symlink) error { return err } + if adder.PreserveMtime { + fsn, err := unixfs.FSNodeFromBytes(sdata) + if err != nil { + return err + } + + fsn.SetModTime(l.ModTime()) + if sdata, err = fsn.GetBytes(); err != nil { + return err + } + } + dagnode := dag.NodeWithData(sdata) dagnode.SetCidBuilder(adder.CidBuilder) err = adder.dagService.Add(adder.ctx, dagnode) @@ -405,14 +425,6 @@ func (adder *Adder) addFile(path string, file files.File) error { } } - if adder.PreserveMtime { - adder.FileMtime = file.ModTime() - } - - if adder.PreserveMode { - adder.FileMode = file.Mode() - } - dagnode, err := adder.add(reader) if err != nil { return err @@ -425,6 +437,17 @@ func (adder *Adder) addFile(path string, file files.File) error { func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory, toplevel bool) error { log.Infof("adding directory: %s", path) + // if we need to store mode or modification time then create a new root which includes that data + if toplevel && (adder.PreserveMode || adder.PreserveMtime) { + nd := unixfs.EmptyDirNodeWithStat(adder.FileMode, adder.FileMtime) + nd.SetCidBuilder(adder.CidBuilder) + mr, err := mfs.NewRoot(ctx, adder.dagService, nd, nil) + if err != nil { + return err + } + adder.SetMfsRoot(mr) + } + if !(toplevel && path == "") { mr, err := adder.mfsRoot() if err != nil { @@ -434,6 +457,8 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory Mkparents: true, Flush: false, CidBuilder: adder.CidBuilder, + Mode: adder.FileMode, + ModTime: adder.FileMtime, }) if err != nil { return err diff --git a/test/sharness/t0047-add-mode-mtime.sh b/test/sharness/t0047-add-mode-mtime.sh index c0652fd3395..53aadcae327 100755 --- a/test/sharness/t0047-add-mode-mtime.sh +++ b/test/sharness/t0047-add-mode-mtime.sh @@ -6,109 +6,196 @@ test_description="Test storing and retrieving mode and mtime" test_init_ipfs -TESTFILE=mountdir/testfile.txt +HASH_NO_PRESERVE=QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH PRESERVE_MTIME=1604320482 PRESERVE_MODE=0640 HASH_PRESERVE_MODE=QmQLgxypSNGNFTuUPGCecq6dDEjb6hNB5xSyVmP3cEuNtq -HASH_PRESERVE_MTIME=QmbfPiiFH3mM9qCCsCL1BdDfDBxeCrVXfiiSEUYdjPzeQf -HASH_PRESERVE_MODE_AND_MTIME=QmT858Bjy215QhtJKBiEueLZq2VDoiiJ6dDP2RNAkPMT7f +HASH_PRESERVE_MTIME=QmQ6kErEW8kztQFV8vbwNU8E4dmtGsYpRiboiLxUEwibvj +HASH_PRESERVE_MODE_AND_MTIME=QmYkvboLsvLFcSYmqVJRxvBdYRQLroLv9kELf3LRiCqBri CUSTOM_MTIME=1603539720 CUSTOM_MTIME_NSECS=54321 CUSTOM_MODE=0764 HASH_CUSTOM_MODE=QmchD3BN8TQ3RW6jPLxSaNkqvfuj7syKhzTRmL4EpyY1Nz -HASH_CUSTOM_MTIME=QmS7vmVxe9YgbWm2CjEKBTYnhhrUjy6cbnDTqe4tkPhu32 +HASH_CUSTOM_MTIME=QmT3aY4avDcYXCWpU8CJzqUkW7YEuEsx36S8cTNoLcuK1B HASH_CUSTOM_MTIME_NSECS=QmaKH8H5rXBUBCX4vdxi7ktGQEL7wejV7L9rX2qpZjwncz HASH_CUSTOM_MODE_AND_MTIME=QmUkxrtBA8tPjwCYz1HrsoRfDz6NgKut3asVeHVQNH4C8L -test_expect_success "can preserve file mode" ' - touch "$TESTFILE" && - chmod $PRESERVE_MODE "$TESTFILE" && - HASH=$(ipfs add -q --hash=sha2-256 --preserve-mode "$TESTFILE") && - test "$HASH_PRESERVE_MODE" = "$HASH" -' - -test_expect_success "can preserve file modification time" ' - touch -m -d @$PRESERVE_MTIME "$TESTFILE" && - HASH=$(ipfs add -q --hash=sha2-256 --preserve-mtime "$TESTFILE") && - test "$HASH_PRESERVE_MTIME" = "$HASH" -' - -test_expect_success "can set file mode" ' - touch "$TESTFILE" && - chmod 0600 "$TESTFILE" && - HASH=$(ipfs add -q --hash=sha2-256 --mode=$CUSTOM_MODE "$TESTFILE") && - test "$HASH_CUSTOM_MODE" = "$HASH" -' - -test_expect_success "can set file mtime" ' - touch -m -t 202011021234.42 "$TESTFILE" && - HASH=$(ipfs add -q --hash=sha2-256 --mtime=$CUSTOM_MTIME "$TESTFILE") && - test "$HASH_CUSTOM_MTIME" = "$HASH" -' - -test_expect_success "can set file mtime nanoseconds" ' - touch -m -t 202011021234.42 "$TESTFILE" && - HASH=$(ipfs add -q --hash=sha2-256 --mtime=$CUSTOM_MTIME --mtime-nsecs=$CUSTOM_MTIME_NSECS "$TESTFILE") && - test "$HASH_CUSTOM_MTIME_NSECS" = "$HASH" -' - -test_expect_success "can preserve file mode and modification time" ' - touch -m -d @$PRESERVE_MTIME "$TESTFILE" && - chmod $PRESERVE_MODE "$TESTFILE" && - HASH=$(ipfs add -q --hash=sha2-256 --preserve-mode --preserve-mtime "$TESTFILE") && - test "$HASH_PRESERVE_MODE_AND_MTIME" = "$HASH" -' - -test_expect_success "can set file mode and mtime" ' - touch -m -t 202011021234.42 "$TESTFILE" && - chmod 0600 "$TESTFILE" && - HASH=$(ipfs add -q --hash=sha2-256 --mode=$CUSTOM_MODE --mtime=$CUSTOM_MTIME --mtime-nsecs=$CUSTOM_MTIME_NSECS "$TESTFILE") && - test "$HASH_CUSTOM_MODE_AND_MTIME" = "$HASH" -' - -test_expect_success "can get preserved mode and mtime" ' - OUTFILE="mountdir/$HASH_PRESERVE_MODE_AND_MTIME" && - ipfs get -o "$OUTFILE" $HASH_PRESERVE_MODE_AND_MTIME && - test "$PRESERVE_MODE:$PRESERVE_MTIME" = "$(stat -c "0%a:%Y" "$OUTFILE")" -' - -test_expect_success "can get custom mode and mtime" ' - OUTFILE="mountdir/$HASH_CUSTOM_MODE_AND_MTIME" && - ipfs get -o "$OUTFILE" $HASH_CUSTOM_MODE_AND_MTIME && - TIMESTAMP=$(date +%s%N --date="$(stat -c "%y" $OUTFILE)") && - MODETIME=$(stat -c "0%a:$TIMESTAMP" "$OUTFILE") && - printf -v EXPECTED "$CUSTOM_MODE:$CUSTOM_MTIME%09d" $CUSTOM_MTIME_NSECS && - test "$EXPECTED" = "$MODETIME" -' - -test_expect_success "can change file mode" ' - HASH=$(echo testfile | ipfs add -q --mode=0600) && - ipfs files cp "/ipfs/$HASH" /chmod1 && - ipfs files chmod 444 /chmod1 && - HASH=$(ipfs files stat /chmod1|head -1) && - ipfs get -o mountdir/chmod1 $HASH && - test $(stat -c "%a" mountdir/chmod1) = 444 -' - -test_expect_success "can touch file modification time" ' - NOW=$(date +%s) && - HASH=$(echo testfile | ipfs add -q --mtime=$NOW) && - ipfs files cp "/ipfs/$HASH" /touch1 && - sleep 1 && - ipfs files touch /touch1 && - HASH=$(ipfs files stat /touch1|head -1) && - ipfs get -o mountdir/touch1 $HASH && - test $(stat -c "%Y" mountdir/touch1) -gt $NOW -' - -test_expect_success "can change file modification time and nanoseconds" ' - echo test|ipfs files write --create /touch2 && - EXPECTED=$(date --date="yesterday" +%s) && - ipfs files touch --mtime=$EXPECTED --mtime-nsecs=55567 /touch2 && - test $(ipfs files stat --format="" /touch2) -eq $EXPECTED && - test $(ipfs files stat --format="" /touch2) -eq 55567 -' +mk_name() { + tr -dc '[:alnum:]' Date: Sun, 26 Jun 2022 17:44:05 +0100 Subject: [PATCH 15/21] correct comment --- core/node/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/node/storage.go b/core/node/storage.go index 3e3e90fa174..1ea02162eb7 100644 --- a/core/node/storage.go +++ b/core/node/storage.go @@ -58,7 +58,7 @@ func GcBlockstoreCtor(bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockst return } -// GcBlockstoreCtor wraps GcBlockstore and adds Filestore support +// FilestoreBlockstoreCtor wraps GcBlockstore and adds Filestore support func FilestoreBlockstoreCtor(repo repo.Repo, bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore, fstore *filestore.Filestore) { gclocker = blockstore.NewGCLocker() From ae46f6a1e412e95346f5a429b6890fe1b5842c88 Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Sun, 26 Jun 2022 20:10:04 +0100 Subject: [PATCH 16/21] remove unneeded parameter --- core/commands/get.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/commands/get.go b/core/commands/get.go index d4a452beb69..a0175aa4d82 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -86,7 +86,7 @@ may also specify the level of compression by specifying '-l=<1-9>'. res.SetLength(uint64(size)) archive, _ := req.Options[archiveOptionName].(bool) - reader, err := fileArchive(file, p.String(), getOutPath(req), archive, cmplvl) + reader, err := fileArchive(file, p.String(), archive, cmplvl) if err != nil { return err } @@ -263,7 +263,7 @@ func (i *identityWriteCloser) Close() error { return nil } -func fileArchive(f files.Node, name string, outpath string, archive bool, compression int) (io.Reader, error) { +func fileArchive(f files.Node, name string, archive bool, compression int) (io.Reader, error) { var err error = nil cleaned := gopath.Clean(name) _, filename := gopath.Split(cleaned) From fb8d100a33036a68d59d3660b39760d6140c1751 Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Tue, 25 Oct 2022 16:56:59 +0100 Subject: [PATCH 17/21] remove unneeded cast --- core/commands/files.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/files.go b/core/commands/files.go index 1c21ef060d0..a760bcf2810 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -1432,7 +1432,7 @@ Examples: var ts time.Time if mtime != 0 { - ts = time.Unix(int64(mtime), int64(nsecs)).UTC() + ts = time.Unix(mtime, int64(nsecs)).UTC() } else { ts = time.Now().UTC() } From c000653f008f7fca2a29d1ff2c0525f1ecb6aa5d Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Mon, 31 Oct 2022 13:44:05 +0000 Subject: [PATCH 18/21] finalize files stat format for mode and mtime --- core/commands/files.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/commands/files.go b/core/commands/files.go index 669053fca42..093d7ac0526 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -249,7 +249,7 @@ var filesStatCmd = &cmds.Command{ } var mtime string if out.Mtime > 0 { - mtime = time.Unix(out.Mtime, int64(out.MtimeNsecs)).Format("2 Jan 2006, 15:04:05 MST") + mtime = time.Unix(out.Mtime, int64(out.MtimeNsecs)).UTC().Format("2 Jan 2006, 15:04:05 MST") } s = strings.Replace(s, "", out.Hash, -1) @@ -257,11 +257,11 @@ var filesStatCmd = &cmds.Command{ s = strings.Replace(s, "", fmt.Sprintf("%d", out.CumulativeSize), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Blocks), -1) s = strings.Replace(s, "", out.Type, -1) - s = strings.Replace(s, "", mode.String(), -1) + s = strings.Replace(s, "", strings.ToLower(mode.String()), -1) s = strings.Replace(s, "", mtime, -1) s = strings.Replace(s, "", strconv.FormatInt(out.Mtime, 10), -1) s = strings.Replace(s, "", strconv.Itoa(out.MtimeNsecs), -1) - s = strings.Replace(s, "", strconv.FormatInt(int64(out.Mode), 8), -1) + s = strings.Replace(s, "", "0"+strconv.FormatInt(int64(out.Mode&0x1FF), 8), -1) fmt.Fprintln(w, s) @@ -350,6 +350,8 @@ func statProtoNode(n *dag.ProtoNode, enc cidenc.Encoder, cid cid.Cid, cumulsize if mode := d.Mode(); mode != 0 { stat.Mode = uint32(mode) + } else if d.Type() == ft.TSymlink { + stat.Mode = uint32(os.ModeSymlink | 0x1FF) } if mt := d.ModTime(); !mt.IsZero() { From 03a37d6f8cfdafab0b6b1930157533940c8bd66f Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Mon, 31 Oct 2022 13:48:01 +0000 Subject: [PATCH 19/21] fix storing mode and mtime on symlinks and directories --- core/coreunix/add.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 8f2a34df136..25d933e0d8e 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -400,13 +400,13 @@ func (adder *Adder) addSymlink(path string, l *files.Symlink) error { return err } - if adder.PreserveMtime { + if !adder.FileMtime.IsZero() { fsn, err := unixfs.FSNodeFromBytes(sdata) if err != nil { return err } - fsn.SetModTime(l.ModTime()) + fsn.SetModTime(adder.FileMtime) if sdata, err = fsn.GetBytes(); err != nil { return err } @@ -448,7 +448,7 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory log.Infof("adding directory: %s", path) // if we need to store mode or modification time then create a new root which includes that data - if toplevel && (adder.PreserveMode || adder.PreserveMtime) { + if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) { nd := unixfs.EmptyDirNodeWithStat(adder.FileMode, adder.FileMtime) nd.SetCidBuilder(adder.CidBuilder) mr, err := mfs.NewRoot(ctx, adder.dagService, nd, nil) From ed2b8420ac5b16b35430bda57bd1f97d68893a8b Mon Sep 17 00:00:00 2001 From: Kenneth Stuart Date: Mon, 31 Oct 2022 13:49:22 +0000 Subject: [PATCH 20/21] complete sharness tests for mode and modification time --- test/sharness/t0047-add-mode-mtime.sh | 317 ++++++++++++++++++++++---- 1 file changed, 268 insertions(+), 49 deletions(-) diff --git a/test/sharness/t0047-add-mode-mtime.sh b/test/sharness/t0047-add-mode-mtime.sh index 53aadcae327..1f2cf748d53 100755 --- a/test/sharness/t0047-add-mode-mtime.sh +++ b/test/sharness/t0047-add-mode-mtime.sh @@ -9,28 +9,35 @@ test_init_ipfs HASH_NO_PRESERVE=QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH PRESERVE_MTIME=1604320482 -PRESERVE_MODE=0640 +PRESERVE_MODE="0640" HASH_PRESERVE_MODE=QmQLgxypSNGNFTuUPGCecq6dDEjb6hNB5xSyVmP3cEuNtq HASH_PRESERVE_MTIME=QmQ6kErEW8kztQFV8vbwNU8E4dmtGsYpRiboiLxUEwibvj +HASH_PRESERVE_LINK_MTIME=QmbJwotgtr84JxcnjpwJ86uZiyMoxbZuNH4YrdJMypkYaB HASH_PRESERVE_MODE_AND_MTIME=QmYkvboLsvLFcSYmqVJRxvBdYRQLroLv9kELf3LRiCqBri CUSTOM_MTIME=1603539720 CUSTOM_MTIME_NSECS=54321 -CUSTOM_MODE=0764 +CUSTOM_MODE="0764" HASH_CUSTOM_MODE=QmchD3BN8TQ3RW6jPLxSaNkqvfuj7syKhzTRmL4EpyY1Nz HASH_CUSTOM_MTIME=QmT3aY4avDcYXCWpU8CJzqUkW7YEuEsx36S8cTNoLcuK1B HASH_CUSTOM_MTIME_NSECS=QmaKH8H5rXBUBCX4vdxi7ktGQEL7wejV7L9rX2qpZjwncz HASH_CUSTOM_MODE_AND_MTIME=QmUkxrtBA8tPjwCYz1HrsoRfDz6NgKut3asVeHVQNH4C8L +HASH_CUSTOM_LINK_MTIME=QmV1Uot2gy4bhY9yvYiZxhhchhyYC6MKKoGV1XtWNmpCLe +HASH_CUSTOM_LINK_MTIME_NSECS=QmPHYCxYvvHj6VxiPNJ3kXxcPsnJLDYUJqsDJWjvytmrmY mk_name() { tr -dc '[:alnum:]' Date: Mon, 31 Oct 2022 14:04:02 +0000 Subject: [PATCH 21/21] add missing test and minor fixes --- test/sharness/t0047-add-mode-mtime.sh | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/test/sharness/t0047-add-mode-mtime.sh b/test/sharness/t0047-add-mode-mtime.sh index 1f2cf748d53..e0e3e670f1c 100755 --- a/test/sharness/t0047-add-mode-mtime.sh +++ b/test/sharness/t0047-add-mode-mtime.sh @@ -101,6 +101,9 @@ test_file() { ' test_expect_success "cannot set mode on symbolic link" ' + HASH=$(ipfs add -q --hash=sha2-256 --mtime=$CUSTOM_MTIME --mode=$CUSTOM_MODE "$TESTLINK") && + ACTUAL=$(ipfs files stat --format="" /ipfs/$HASH) && + test "$ACTUAL" = "lrwxrwxrwx" ' @@ -293,7 +296,6 @@ test_directory() { test_expect_success "can recursively set directory mode [$1]" ' HASHES=($(ipfs add -qr --hash=sha2-256 --mode=0753 "$TESTDIR"|sort)) && - echo "${HASHES[*]}" && test "${HASHES[*]}" = "${HASH_DIR_CUSTOM_MODE[*]}" ' @@ -344,35 +346,35 @@ test_stat_template() { test_expect_success "can stat $2 string mode [$1]" ' touch "$STAT_TARGET" && HASH=$(ipfs add -qr --hash=sha2-256 --mode="$STAT_MODE_OCTAL" "$STAT_TARGET") && - EXPECTED=$(ipfs files stat --format="" /ipfs/$HASH) && - test "$EXPECTED" = "$STAT_MODE_STRING" + ACTUAL=$(ipfs files stat --format="" /ipfs/$HASH) && + test "$ACTUAL" = "$STAT_MODE_STRING" ' test_expect_success "can stat $2 octal mode [$1]" ' touch "$STAT_TARGET" && HASH=$(ipfs add -qr --hash=sha2-256 --mode="$STAT_MODE_OCTAL" "$STAT_TARGET") && - EXPECTED=$(ipfs files stat --format="" /ipfs/$HASH) && - test "$EXPECTED" = "$STAT_MODE_OCTAL" + ACTUAL=$(ipfs files stat --format="" /ipfs/$HASH) && + test "$ACTUAL" = "$STAT_MODE_OCTAL" ' test_expect_success "can stat $2 modification time string [$1]" ' touch "$STAT_TARGET" && HASH=$(ipfs add -qr --hash=sha2-256 --mtime=$CUSTOM_MTIME "$STAT_TARGET") && - EXPECTED=$(ipfs files stat --format="" /ipfs/$HASH) && - test "$EXPECTED" = "24 Oct 2020, 11:42:00 UTC" + ACTUAL=$(ipfs files stat --format="" /ipfs/$HASH) && + test "$ACTUAL" = "24 Oct 2020, 11:42:00 UTC" ' test_expect_success "can stat $2 modification time seconds [$1]" ' touch "$STAT_TARGET" && HASH=$(ipfs add -qr --hash=sha2-256 --mtime=$CUSTOM_MTIME "$STAT_TARGET") && - EXPECTED=$(ipfs files stat --format="" /ipfs/$HASH) && - test $EXPECTED -eq $CUSTOM_MTIME + ACTUAL=$(ipfs files stat --format="" /ipfs/$HASH) && + test $ACTUAL -eq $CUSTOM_MTIME ' test_expect_success "can stat $2 modification time nanoseconds [$1]" ' touch "$STAT_TARGET" && HASH=$(ipfs add -qr --hash=sha2-256 --mtime=$CUSTOM_MTIME --mtime-nsecs=$CUSTOM_MTIME_NSECS "$STAT_TARGET") && - EXPECTED=$(ipfs files stat --format="" /ipfs/$HASH) && - test $EXPECTED -eq $CUSTOM_MTIME_NSECS + ACTUAL=$(ipfs files stat --format="" /ipfs/$HASH) && + test $ACTUAL -eq $CUSTOM_MTIME_NSECS ' } @@ -398,8 +400,8 @@ test_stat() { STAT_TARGET="mountdir/statfile$1" test_expect_success "can chain stat template [$1]" ' HASH=$(ipfs add -q --hash=sha2-256 --mode=0644 --mtime=$CUSTOM_MTIME --mtime-nsecs=$CUSTOM_MTIME_NSECS "$STAT_TARGET") && - EXPECTED=$(ipfs files stat --format=" " /ipfs/$HASH) && - test "$EXPECTED" = "24 Oct 2020, 11:42:00 UTC 1603539720 54321 -rw-r--r-- 0644" + ACTUAL=$(ipfs files stat --format=" " /ipfs/$HASH) && + test "$ACTUAL" = "24 Oct 2020, 11:42:00 UTC 1603539720 54321 -rw-r--r-- 0644" ' }