diff --git a/core/commands/add.go b/core/commands/add.go index 12a38aaebe7..212c706f8b9 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -6,7 +6,9 @@ import ( "io" "os" "path" + "strconv" "strings" + "time" "github.com/ipfs/kubo/core/commands/cmdenv" @@ -23,11 +25,31 @@ import ( // ErrDepthLimitExceeded indicates that the max depth has been exceeded. var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded") +type TimeParts struct { + t *time.Time +} + +func (t TimeParts) MarshalJSON() ([]byte, error) { + return t.t.MarshalJSON() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +// The time is expected to be a quoted string in RFC 3339 format. +func (t *TimeParts) UnmarshalJSON(data []byte) (err error) { + // Fractional seconds are handled implicitly by Parse. + tt, err := time.Parse("\"2006-01-02T15:04:05Z\"", string(data)) + *t = TimeParts{&tt} + return +} + type AddEvent struct { - Name string - Hash string `json:",omitempty"` - Bytes int64 `json:",omitempty"` - Size string `json:",omitempty"` + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` + Mode string `json:",omitempty"` + Mtime int64 `json:",omitempty"` + MtimeNsecs int `json:",omitempty"` } const ( @@ -48,6 +70,12 @@ const ( inlineOptionName = "inline" inlineLimitOptionName = "inline-limit" toFilesOptionName = "to-files" + + preserveModeOptionName = "preserve-mode" + preserveMtimeOptionName = "preserve-mtime" + modeOptionName = "mode" + mtimeOptionName = "mtime" + mtimeNsecsOptionName = "mtime-nsecs" ) const adderOutChanSize = 8 @@ -164,6 +192,11 @@ See 'dag export' and 'dag import' for more information. cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32), cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true), cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."), + cmds.BoolOption(preserveModeOptionName, "Apply permissions to created UnixFS entries"), + cmds.BoolOption(preserveMtimeOptionName, "Apply modification time to created UnixFS entries"), + cmds.UintOption(modeOptionName, "File mode to apply to created UnixFS entries"), + cmds.Int64Option(mtimeOptionName, "Modification time in seconds before or after the Unix Epoch to apply to created UnixFS entries"), + cmds.UintOption(mtimeNsecsOptionName, "Modification time fraction in nanoseconds"), }, PreRun: func(req *cmds.Request, env cmds.Environment) error { quiet, _ := req.Options[quietOptionName].(bool) @@ -205,6 +238,11 @@ See 'dag export' and 'dag import' for more information. inline, _ := req.Options[inlineOptionName].(bool) inlineLimit, _ := req.Options[inlineLimitOptionName].(int) toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string) + preserveMode, _ := req.Options[preserveModeOptionName].(bool) + preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool) + mode, _ := req.Options[modeOptionName].(uint) + mtime, _ := req.Options[mtimeOptionName].(int64) + mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint) hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { @@ -241,6 +279,19 @@ See 'dag export' and 'dag import' for more information. options.Unixfs.Progress(progress), options.Unixfs.Silent(silent), + + options.Unixfs.PreserveMode(preserveMode), + options.Unixfs.PreserveMtime(preserveMtime), + } + + if mode != 0 { + opts = append(opts, options.Unixfs.Mode(os.FileMode(mode))) + } + + if mtime != 0 { + opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs))) + } else if mtimeNsecs != 0 { + fmt.Printf("option %s ignored as no valid %s value provided\n", mtimeNsecsOptionName, mtimeOptionName) } if cidVerSet { @@ -352,12 +403,33 @@ See 'dag export' and 'dag import' for more information. output.Name = path.Join(addit.Name(), output.Name) } - if err := res.Emit(&AddEvent{ - Name: output.Name, - Hash: h, - Bytes: output.Bytes, - Size: output.Size, - }); err != nil { + output.Mode = addit.Node().Mode() + if ts := addit.Node().ModTime(); !ts.IsZero() { + output.Mtime = addit.Node().ModTime().Unix() + output.MtimeNsecs = addit.Node().ModTime().Nanosecond() + } + + addEvent := AddEvent{ + Name: output.Name, + Hash: h, + Bytes: output.Bytes, + Size: output.Size, + Mtime: output.Mtime, + MtimeNsecs: output.MtimeNsecs, + } + + if output.Mode != 0 { + addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8) + } + + if output.Mtime > 0 { + addEvent.Mtime = output.Mtime + if output.MtimeNsecs > 0 { + addEvent.MtimeNsecs = output.MtimeNsecs + } + } + + if err := res.Emit(&addEvent); err != nil { return err } } diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index e40a146d665..1fbc81e2cf1 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -145,6 +145,8 @@ func TestCommands(t *testing.T) { "/files/rm", "/files/stat", "/files/write", + "/files/chmod", + "/files/touch", "/filestore", "/filestore/dups", "/filestore/ls", diff --git a/core/commands/files.go b/core/commands/files.go index e48abba2815..093d7ac0526 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -2,13 +2,16 @@ package commands import ( "context" + "encoding/json" "errors" "fmt" "io" "os" gopath "path" "sort" + "strconv" "strings" + "time" humanize "github.com/dustin/go-humanize" "github.com/ipfs/kubo/core" @@ -80,6 +83,8 @@ operations. "rm": filesRmCmd, "flush": filesFlushCmd, "chcid": filesChcidCmd, + "touch": filesTouchCmd, + "chmod": filesChmodCmd, }, } @@ -102,6 +107,43 @@ type statOutput struct { WithLocality bool `json:",omitempty"` Local bool `json:",omitempty"` SizeLocal uint64 `json:",omitempty"` + Mode uint32 `json:",omitempty"` + Mtime int64 `json:",omitempty"` + MtimeNsecs int `json:",omitempty"` +} + +func (s *statOutput) MarshalJSON() ([]byte, error) { + type so statOutput + out := &struct { + *so + Mode string `json:",omitempty"` + }{so: (*so)(s)} + + if s.Mode != 0 { + out.Mode = fmt.Sprintf("%04o", s.Mode) + } + return json.Marshal(out) +} + +func (s *statOutput) UnmarshalJSON(data []byte) error { + var err error + type so statOutput + tmp := &struct { + *so + Mode string `json:",omitempty"` + }{so: (*so)(s)} + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + if tmp.Mode != "" { + mode, err := strconv.ParseUint(tmp.Mode, 8, 32) + if err == nil { + s.Mode = uint32(mode) + } + } + return err } const ( @@ -109,7 +151,9 @@ const ( Size: CumulativeSize: ChildBlocks: -Type: ` +Type: +Mode: +Mtime: ` filesFormatOptionName = "format" filesSizeOptionName = "size" filesWithLocalOptionName = "with-local" @@ -198,11 +242,26 @@ var filesStatCmd = &cmds.Command{ Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *statOutput) error { s, _ := statGetFormatOptions(req) + + var mode os.FileMode + if out.Mode != 0 { + mode = os.FileMode(out.Mode) + } + var mtime string + if out.Mtime > 0 { + mtime = time.Unix(out.Mtime, int64(out.MtimeNsecs)).UTC().Format("2 Jan 2006, 15:04:05 MST") + } + s = strings.Replace(s, "", out.Hash, -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Size), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.CumulativeSize), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Blocks), -1) s = strings.Replace(s, "", out.Type, -1) + s = strings.Replace(s, "", strings.ToLower(mode.String()), -1) + s = strings.Replace(s, "", mtime, -1) + s = strings.Replace(s, "", strconv.FormatInt(out.Mtime, 10), -1) + s = strings.Replace(s, "", strconv.Itoa(out.MtimeNsecs), -1) + s = strings.Replace(s, "", "0"+strconv.FormatInt(int64(out.Mode&0x1FF), 8), -1) fmt.Fprintln(w, s) @@ -253,28 +312,7 @@ func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) { switch n := nd.(type) { case *dag.ProtoNode: - d, err := ft.FSNodeFromBytes(n.Data()) - if err != nil { - return nil, err - } - - var ndtype string - switch d.Type() { - case ft.TDirectory, ft.THAMTShard: - ndtype = "directory" - case ft.TFile, ft.TMetadata, ft.TRaw: - ndtype = "file" - default: - return nil, fmt.Errorf("unrecognized node type: %s", d.Type()) - } - - return &statOutput{ - Hash: enc.Encode(c), - Blocks: len(nd.Links()), - Size: d.FileSize(), - CumulativeSize: cumulsize, - Type: ndtype, - }, nil + return statProtoNode(n, enc, c, cumulsize) case *dag.RawNode: return &statOutput{ Hash: enc.Encode(c), @@ -288,6 +326,44 @@ func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) { } } +func statProtoNode(n *dag.ProtoNode, enc cidenc.Encoder, cid cid.Cid, cumulsize uint64) (*statOutput, error) { + d, err := ft.FSNodeFromBytes(n.Data()) + if err != nil { + return nil, err + } + + stat := statOutput{ + Hash: enc.Encode(cid), + Blocks: len(n.Links()), + Size: d.FileSize(), + CumulativeSize: cumulsize, + } + + switch d.Type() { + case ft.TDirectory, ft.THAMTShard: + stat.Type = "directory" + case ft.TFile, ft.TSymlink, ft.TMetadata, ft.TRaw: + stat.Type = "file" + default: + return nil, fmt.Errorf("unrecognized node type: %s", d.Type()) + } + + if mode := d.Mode(); mode != 0 { + stat.Mode = uint32(mode) + } else if d.Type() == ft.TSymlink { + stat.Mode = uint32(os.ModeSymlink | 0x1FF) + } + + if mt := d.ModTime(); !mt.IsZero() { + stat.Mtime = mt.Unix() + if ns := mt.Nanosecond(); ns > 0 { + stat.MtimeNsecs = ns + } + } + + return &stat, nil +} + func walkBlock(ctx context.Context, dagserv ipld.DAGService, nd ipld.Node) (bool, uint64, error) { // Start with the block data size sizeLocal := uint64(len(nd.RawData())) @@ -341,7 +417,7 @@ $ ipfs add --quieter --pin=false $ ipfs files cp /ipfs/ /your/desired/mfs/path If you wish to fully copy content from a different IPFS peer into MFS, do not -forget to force IPFS to fetch to full DAG after doing the "cp" operation. i.e: +forget to force IPFS to fetch the full DAG after doing the "cp" operation. i.e: $ ipfs files cp /ipfs/ /your/desired/mfs/path $ ipfs pin add @@ -1295,3 +1371,87 @@ func getParentDir(root *mfs.Root, dir string) (*mfs.Directory, error) { } return pdir, nil } + +var filesChmodCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Change mode permissions", + ShortDescription: ` +The mode argument must be specified in unix numeric notation. + + $ ipfs files chmod 0644 /foo + $ ipfs files stat /foo + ... + Type: file + Mode: -rw-r--r-- + ... +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("mode", true, false, "mode to apply to node"), + cmds.StringArg("path", true, false, "Path to target node"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + path, err := checkPath(req.Arguments[1]) + if err != nil { + return err + } + + mode, err := strconv.ParseInt(req.Arguments[0], 8, 32) + if err != nil { + return err + } + + return mfs.Chmod(nd.FilesRoot, path, os.FileMode(mode)) + }, +} + +var filesTouchCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Change file modification times.", + ShortDescription: ` +Examples: + + # set modification time to now. + $ ipfs files touch /foo + + # set a custom modification time. + $ ipfs files touch --mtime=1630937926 /foo + +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("path", true, false, "Path of target to update."), + }, + Options: []cmds.Option{ + cmds.Int64Option(mtimeOptionName, "Modification time in seconds before or since the Unix Epoch to apply to created UnixFS entries."), + cmds.UintOption(mtimeNsecsOptionName, "Modification time fraction in nanoseconds"), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + path, err := checkPath(req.Arguments[0]) + if err != nil { + return err + } + + mtime, _ := req.Options[mtimeOptionName].(int64) + nsecs, _ := req.Options[mtimeNsecsOptionName].(uint) + + var ts time.Time + if mtime != 0 { + ts = time.Unix(mtime, int64(nsecs)).UTC() + } else { + ts = time.Now().UTC() + } + + return mfs.Touch(nd.FilesRoot, path, ts) + }, +} diff --git a/core/commands/get.go b/core/commands/get.go index 85bb43a57ec..8c9aeabace6 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -1,6 +1,7 @@ package commands import ( + tar2 "archive/tar" "bufio" "compress/gzip" "errors" @@ -336,9 +337,14 @@ func fileArchive(f files.Node, name string, archive bool, compression int) (io.R return nil, err } + // if not creating an archive set the format to PAX in order to preserve nanoseconds + if !archive { + w.SetFormat(tar2.FormatPAX) + } + go func() { // write all the nodes recursively - if err := w.WriteFile(f, filename); checkErrAndClosePipe(err) { + if err := w.WriteNode(f, filename); checkErrAndClosePipe(err) { return } w.Close() // close tar writer diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index d43bfe3132b..d805f1e642b 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -155,6 +155,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options fileAdder.RawLeaves = settings.RawLeaves fileAdder.NoCopy = settings.NoCopy fileAdder.CidBuilder = prefix + fileAdder.PreserveMode = settings.PreserveMode + fileAdder.PreserveMtime = settings.PreserveMtime + fileAdder.FileMode = settings.Mode + fileAdder.FileMtime = settings.Mtime switch settings.Layout { case options.BalancedLayout: diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 54e5873a9d4..25d933e0d8e 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "io" + "os" gopath "path" "strconv" + "time" "github.com/ipfs/go-cid" bstore "github.com/ipfs/go-ipfs-blockstore" @@ -80,6 +82,11 @@ type Adder struct { tempRoot cid.Cid CidBuilder cid.Builder liveNodes uint64 + + PreserveMode bool + PreserveMtime bool + FileMode os.FileMode + FileMtime time.Time } func (adder *Adder) mfsRoot() (*mfs.Root, error) { @@ -109,11 +116,13 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) { } params := ihelper.DagBuilderParams{ - Dagserv: adder.bufferedDS, - RawLeaves: adder.RawLeaves, - Maxlinks: ihelper.DefaultLinksPerBlock, - NoCopy: adder.NoCopy, - CidBuilder: adder.CidBuilder, + Dagserv: adder.bufferedDS, + RawLeaves: adder.RawLeaves, + Maxlinks: ihelper.DefaultLinksPerBlock, + NoCopy: adder.NoCopy, + CidBuilder: adder.CidBuilder, + FileMode: adder.FileMode, + FileModTime: adder.FileMtime, } db, err := params.New(chnk) @@ -351,6 +360,14 @@ func (adder *Adder) addFileNode(ctx context.Context, path string, file files.Nod return err } + if adder.PreserveMtime { + adder.FileMtime = file.ModTime() + } + + if adder.PreserveMode { + adder.FileMode = file.Mode() + } + if adder.liveNodes >= liveCacheSize { // TODO: A smarter cache that uses some sort of lru cache with an eviction handler mr, err := adder.mfsRoot() @@ -383,6 +400,18 @@ func (adder *Adder) addSymlink(path string, l *files.Symlink) error { return err } + if !adder.FileMtime.IsZero() { + fsn, err := unixfs.FSNodeFromBytes(sdata) + if err != nil { + return err + } + + fsn.SetModTime(adder.FileMtime) + if sdata, err = fsn.GetBytes(); err != nil { + return err + } + } + dagnode := dag.NodeWithData(sdata) dagnode.SetCidBuilder(adder.CidBuilder) err = adder.dagService.Add(adder.ctx, dagnode) @@ -418,6 +447,17 @@ func (adder *Adder) addFile(path string, file files.File) error { func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory, toplevel bool) error { log.Infof("adding directory: %s", path) + // if we need to store mode or modification time then create a new root which includes that data + if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) { + nd := unixfs.EmptyDirNodeWithStat(adder.FileMode, adder.FileMtime) + nd.SetCidBuilder(adder.CidBuilder) + mr, err := mfs.NewRoot(ctx, adder.dagService, nd, nil) + if err != nil { + return err + } + adder.SetMfsRoot(mr) + } + if !(toplevel && path == "") { mr, err := adder.mfsRoot() if err != nil { @@ -427,6 +467,8 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory Mkparents: true, Flush: false, CidBuilder: adder.CidBuilder, + Mode: adder.FileMode, + ModTime: adder.FileMtime, }) if err != nil { return err diff --git a/core/node/storage.go b/core/node/storage.go index 2e831fae22e..c3918188aeb 100644 --- a/core/node/storage.go +++ b/core/node/storage.go @@ -58,7 +58,7 @@ func GcBlockstoreCtor(bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockst return } -// GcBlockstoreCtor wraps GcBlockstore and adds Filestore support +// FilestoreBlockstoreCtor wraps GcBlockstore and adds Filestore support func FilestoreBlockstoreCtor(repo repo.Repo, bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore, fstore *filestore.Filestore) { gclocker = blockstore.NewGCLocker() diff --git a/test/sharness/t0047-add-mode-mtime.sh b/test/sharness/t0047-add-mode-mtime.sh new file mode 100755 index 00000000000..e0e3e670f1c --- /dev/null +++ b/test/sharness/t0047-add-mode-mtime.sh @@ -0,0 +1,422 @@ +#!/usr/bin/env bash + +test_description="Test storing and retrieving mode and mtime" + +. lib/test-lib.sh + +test_init_ipfs + +HASH_NO_PRESERVE=QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH + +PRESERVE_MTIME=1604320482 +PRESERVE_MODE="0640" +HASH_PRESERVE_MODE=QmQLgxypSNGNFTuUPGCecq6dDEjb6hNB5xSyVmP3cEuNtq +HASH_PRESERVE_MTIME=QmQ6kErEW8kztQFV8vbwNU8E4dmtGsYpRiboiLxUEwibvj +HASH_PRESERVE_LINK_MTIME=QmbJwotgtr84JxcnjpwJ86uZiyMoxbZuNH4YrdJMypkYaB +HASH_PRESERVE_MODE_AND_MTIME=QmYkvboLsvLFcSYmqVJRxvBdYRQLroLv9kELf3LRiCqBri + +CUSTOM_MTIME=1603539720 +CUSTOM_MTIME_NSECS=54321 +CUSTOM_MODE="0764" +HASH_CUSTOM_MODE=QmchD3BN8TQ3RW6jPLxSaNkqvfuj7syKhzTRmL4EpyY1Nz +HASH_CUSTOM_MTIME=QmT3aY4avDcYXCWpU8CJzqUkW7YEuEsx36S8cTNoLcuK1B +HASH_CUSTOM_MTIME_NSECS=QmaKH8H5rXBUBCX4vdxi7ktGQEL7wejV7L9rX2qpZjwncz +HASH_CUSTOM_MODE_AND_MTIME=QmUkxrtBA8tPjwCYz1HrsoRfDz6NgKut3asVeHVQNH4C8L +HASH_CUSTOM_LINK_MTIME=QmV1Uot2gy4bhY9yvYiZxhhchhyYC6MKKoGV1XtWNmpCLe +HASH_CUSTOM_LINK_MTIME_NSECS=QmPHYCxYvvHj6VxiPNJ3kXxcPsnJLDYUJqsDJWjvytmrmY + +mk_name() { + tr -dc '[:alnum:]'> file1stat_expect && echo "ChildBlocks: 0" >> file1stat_expect && echo "Type: file" >> file1stat_expect && + echo "Mode: ----------" >> file1stat_expect && + echo "Mtime: " >> file1stat_expect && test_cmp file1stat_expect file1stat_actual ' @@ -243,6 +245,8 @@ test_files_api() { echo "Size: 4" >> file1stat_expect && echo "ChildBlocks: 0" >> file1stat_expect && echo "Type: file" >> file1stat_expect && + echo "Mode: ----------" >> file1stat_expect && + echo "Mtime: " >> file1stat_expect && test_cmp file1stat_expect file1stat_actual '