feat: Support storing UnixFS 1.5 Mode and ModTime

client/rpc/apifile.go

@@ -5,6 +5,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
+	"strconv"
+	"time"
 
 	"github.com/ipfs/boxo/files"
 	unixfs "github.com/ipfs/boxo/ipld/unixfs"
@@ -24,9 +27,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error)
 	}
 
 	var stat struct {
-		Hash string
-		Type string
-		Size int64 // unixfs size
+		Hash       string
+		Type       string
+		Size       int64 // unixfs size
+		Mode       string
+		Mtime      int64
+		MtimeNsecs int
 	}
 	err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat)
 	if err != nil {
@@ -35,9 +41,9 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error)
 
 	switch stat.Type {
 	case "file":
-		return api.getFile(ctx, p, stat.Size)
+		return api.getFile(ctx, p, stat.Size, stat.Mode, stat.Mtime, stat.MtimeNsecs)
 	case "directory":
-		return api.getDir(ctx, p, stat.Size)
+		return api.getDir(ctx, p, stat.Size, stat.Mode, stat.Mtime, stat.MtimeNsecs)
 	default:
 		return nil, fmt.Errorf("unsupported file type '%s'", stat.Type)
 	}
@@ -49,6 +55,9 @@ type apiFile struct {
 	size int64
 	path path.Path
 
+	mode  os.FileMode
+	mtime time.Time
+
 	r  *Response
 	at int64
 }
@@ -128,16 +137,44 @@ func (f *apiFile) Close() error {
 	return nil
 }
 
+func (f *apiFile) Mode() os.FileMode {
+	return f.mode
+}
+
+func (f *apiFile) ModTime() time.Time {
+	return f.mtime
+}
+
 func (f *apiFile) Size() (int64, error) {
 	return f.size, nil
 }
 
-func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64) (files.Node, error) {
+func stringToFileMode(mode string) (os.FileMode, error) {
+	if mode == "" {
+		return 0, nil
+	}
+	mode64, err := strconv.ParseUint(mode, 8, 32)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse mode %s: %s", mode, err)
+	}
+	return os.FileMode(uint32(mode64)), nil
+}
+
+func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64, mode string, mtime int64, mtimeNsecs int) (files.Node, error) {
+	fm, err := stringToFileMode(mode)
+	if err != nil {
+		return nil, err
+	}
+
 	f := &apiFile{
 		ctx:  ctx,
 		core: api.core(),
 		size: size,
 		path: p,
+		mode: fm,
+	}
+	if mtime != 0 {
+		f.mtime = time.Unix(mtime, int64(mtimeNsecs))
 	}
 
 	return f, f.reset()
@@ -195,13 +232,13 @@ func (it *apiIter) Next() bool {
 
 	switch it.cur.Type {
 	case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory:
-		it.curFile, err = it.core.getDir(it.ctx, path.FromCid(c), int64(it.cur.Size))
+		it.curFile, err = it.core.getDir(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.Mtime, it.cur.MtimeNsecs)
 		if err != nil {
 			it.err = err
 			return false
 		}
 	case unixfs.TFile:
-		it.curFile, err = it.core.getFile(it.ctx, path.FromCid(c), int64(it.cur.Size))
+		it.curFile, err = it.core.getFile(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.Mtime, it.cur.MtimeNsecs)
 		if err != nil {
 			it.err = err
 			return false
@@ -223,13 +260,24 @@ type apiDir struct {
 	size int64
 	path path.Path
 
+	mode  os.FileMode
+	mtime time.Time
+
 	dec *json.Decoder
 }
 
 func (d *apiDir) Close() error {
 	return nil
 }
 
+func (d *apiDir) Mode() os.FileMode {
+	return d.mode
+}
+
+func (d *apiDir) ModTime() time.Time {
+	return d.mtime
+}
+
 func (d *apiDir) Size() (int64, error) {
 	return d.size, nil
 }
@@ -242,7 +290,7 @@ func (d *apiDir) Entries() files.DirIterator {
 	}
 }
 
-func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (files.Node, error) {
+func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64, mode string, mtime int64, mtimeNsecs int) (files.Node, error) {
 	resp, err := api.core().Request("ls", p.String()).
 		Option("resolve-size", true).
 		Option("stream", true).Send(ctx)
@@ -253,15 +301,25 @@ func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (file
 		return nil, resp.Error
 	}
 
+	fm, err := stringToFileMode(mode)
+	if err != nil {
+		return nil, err
+	}
+
 	d := &apiDir{
 		ctx:  ctx,
 		core: api,
 		size: size,
 		path: p,
+		mode: fm,
 
 		dec: json.NewDecoder(resp.Output),
 	}
 
+	if mtime != 0 {
+		d.mtime = time.Unix(mtime, int64(mtimeNsecs)).UTC()
+	}
+
 	return d, nil
 }
 

client/rpc/unixfs.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 
 	"github.com/ipfs/boxo/files"
 	unixfs "github.com/ipfs/boxo/ipld/unixfs"
@@ -22,6 +23,10 @@ type addEvent struct {
 	Hash  string `json:",omitempty"`
 	Bytes int64  `json:",omitempty"`
 	Size  string `json:",omitempty"`
+
+	Mode       os.FileMode `json:",omitempty"`
+	Mtime      int64       `json:",omitempty"`
+	MtimeNsecs int         `json:",omitempty"`
 }
 
 type UnixfsAPI HttpApi
@@ -80,23 +85,25 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
 	}
 	defer resp.Output.Close()
 	dec := json.NewDecoder(resp.Output)
-loop:
+
 	for {
 		var evt addEvent
-		switch err := dec.Decode(&evt); err {
-		case nil:
-		case io.EOF:
-			break loop
-		default:
+		if err := dec.Decode(&evt); err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
 			return path.ImmutablePath{}, err
 		}
 		out = evt
 
 		if options.Events != nil {
 			ifevt := &iface.AddEvent{
-				Name:  out.Name,
-				Size:  out.Size,
-				Bytes: out.Bytes,
+				Name:       out.Name,
+				Size:       out.Size,
+				Bytes:      out.Bytes,
+				Mode:       out.Mode,
+				Mtime:      out.Mtime,
+				MtimeNsecs: out.MtimeNsecs,
 			}
 
 			if out.Hash != "" {
@@ -129,6 +136,10 @@ type lsLink struct {
 	Size       uint64
 	Type       unixfs_pb.Data_DataType
 	Target     string
+
+	Mode       string
+	Mtime      int64
+	MtimeNsecs int
 }
 
 type lsObject struct {

core/commands/add.go

@@ -6,7 +6,9 @@ import (
 	"io"
 	"os"
 	gopath "path"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
@@ -25,11 +27,31 @@ import (
 // ErrDepthLimitExceeded indicates that the max depth has been exceeded.
 var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")
 
+type TimeParts struct {
+	t *time.Time
+}
+
+func (t TimeParts) MarshalJSON() ([]byte, error) {
+	return t.t.MarshalJSON()
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+// The time is expected to be a quoted string in RFC 3339 format.
+func (t *TimeParts) UnmarshalJSON(data []byte) (err error) {
+	// Fractional seconds are handled implicitly by Parse.
+	tt, err := time.Parse("\"2006-01-02T15:04:05Z\"", string(data))
+	*t = TimeParts{&tt}
+	return
+}
+
 type AddEvent struct {
-	Name  string
-	Hash  string `json:",omitempty"`
-	Bytes int64  `json:",omitempty"`
-	Size  string `json:",omitempty"`
+	Name       string
+	Hash       string `json:",omitempty"`
+	Bytes      int64  `json:",omitempty"`
+	Size       string `json:",omitempty"`
+	Mode       string `json:",omitempty"`
+	Mtime      int64  `json:",omitempty"`
+	MtimeNsecs int    `json:",omitempty"`
 }
 
 const (
@@ -50,6 +72,12 @@ const (
 	inlineOptionName      = "inline"
 	inlineLimitOptionName = "inline-limit"
 	toFilesOptionName     = "to-files"
+
+	preserveModeOptionName  = "preserve-mode"
+	preserveMtimeOptionName = "preserve-mtime"
+	modeOptionName          = "mode"
+	mtimeOptionName         = "mtime"
+	mtimeNsecsOptionName    = "mtime-nsecs"
 )
 
 const adderOutChanSize = 8
@@ -166,6 +194,12 @@ See 'dag export' and 'dag import' for more information.
 		cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
 		cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
 		cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."),
+
+		cmds.BoolOption(preserveModeOptionName, "Apply existing POSIX permissions to created UnixFS entries"),
+		cmds.BoolOption(preserveMtimeOptionName, "Apply existing POSIX modification time to created UnixFS entries"),
+		cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries"),
+		cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch)"),
+		cmds.UintOption(mtimeNsecsOptionName, "Custom POSIX modification time (optional time fraction in nanoseconds)"),
 	},
 	PreRun: func(req *cmds.Request, env cmds.Environment) error {
 		quiet, _ := req.Options[quietOptionName].(bool)
@@ -217,6 +251,11 @@ See 'dag export' and 'dag import' for more information.
 		inline, _ := req.Options[inlineOptionName].(bool)
 		inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
 		toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)
+		preserveMode, _ := req.Options[preserveModeOptionName].(bool)
+		preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool)
+		mode, _ := req.Options[modeOptionName].(uint)
+		mtime, _ := req.Options[mtimeOptionName].(int64)
+		mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint)
 
 		if chunker == "" {
 			chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
@@ -272,6 +311,19 @@ See 'dag export' and 'dag import' for more information.
 
 			options.Unixfs.Progress(progress),
 			options.Unixfs.Silent(silent),
+
+			options.Unixfs.PreserveMode(preserveMode),
+			options.Unixfs.PreserveMtime(preserveMtime),
+		}
+
+		if mode != 0 {
+			opts = append(opts, options.Unixfs.Mode(os.FileMode(mode)))
+		}
+
+		if mtime != 0 {
+			opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs)))
+		} else if mtimeNsecs != 0 {
+			return fmt.Errorf("option %q requires %q to be provided as well", mtimeNsecsOptionName, mtimeOptionName)
 		}
 
 		if cidVerSet {
@@ -383,12 +435,33 @@ See 'dag export' and 'dag import' for more information.
 					output.Name = gopath.Join(addit.Name(), output.Name)
 				}
 
-				if err := res.Emit(&AddEvent{
-					Name:  output.Name,
-					Hash:  h,
-					Bytes: output.Bytes,
-					Size:  output.Size,
-				}); err != nil {
+				output.Mode = addit.Node().Mode()
+				if ts := addit.Node().ModTime(); !ts.IsZero() {
+					output.Mtime = addit.Node().ModTime().Unix()
+					output.MtimeNsecs = addit.Node().ModTime().Nanosecond()
+				}
+
+				addEvent := AddEvent{
+					Name:       output.Name,
+					Hash:       h,
+					Bytes:      output.Bytes,
+					Size:       output.Size,
+					Mtime:      output.Mtime,
+					MtimeNsecs: output.MtimeNsecs,
+				}
+
+				if output.Mode != 0 {
+					addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8)
+				}
+
+				if output.Mtime > 0 {
+					addEvent.Mtime = output.Mtime
+					if output.MtimeNsecs > 0 {
+						addEvent.MtimeNsecs = output.MtimeNsecs
+					}
+				}
+
+				if err := res.Emit(&addEvent); err != nil {
 					return err
 				}
 			}

core/commands/commands_test.go

@@ -89,6 +89,8 @@ func TestCommands(t *testing.T) {
 		"/files/rm",
 		"/files/stat",
 		"/files/write",
+		"/files/chmod",
+		"/files/touch",
 		"/filestore",
 		"/filestore/dups",
 		"/filestore/ls",