add juicefs.skip-dir-mtime for sdk (#4651)

juicedata · Apr 7, 2024 · b5efe8a · b5efe8a
1 parent 0681c7b
commit b5efe8a
Show file tree

Hide file tree

Showing 11 changed files with 74 additions and 71 deletions.
diff --git a/cmd/flags.go b/cmd/flags.go
@@ -18,14 +18,10 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/urfave/cli/v2"
 	"os"
 	"path"
 	"runtime"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/urfave/cli/v2"
 )
 
 func globalFlags() []cli.Flag {
@@ -350,29 +346,3 @@ func expandFlags(compoundFlags ...[]cli.Flag) []cli.Flag {
 	}
 	return flags
 }
-
-func duration(s string) time.Duration {
-	if s == "" {
-		return 0
-	}
-	v, err := strconv.ParseFloat(s, 64)
-	if err == nil {
-		return time.Microsecond * time.Duration(v*1e6)
-	}
-
-	err = nil
-	var d time.Duration
-	p := strings.Index(s, "d")
-	if p >= 0 {
-		v, err = strconv.ParseFloat(s[:p], 64)
-	}
-	if err == nil && s[p+1:] != "" {
-		d, err = time.ParseDuration(s[p+1:])
-	}
-
-	if err != nil {
-		logger.Warnf("Invalid duration value: %s, setting it to 0", s)
-		return 0
-	}
-	return d + time.Hour*time.Duration(v*24)
-}
diff --git a/cmd/flags_test.go b/cmd/flags_test.go
@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"github.com/juicedata/juicefs/pkg/utils"
 	"testing"
 	"time"
 
@@ -49,7 +50,7 @@ func Test_duration(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			assert.Equalf(t, tt.want, duration(tt.args.s), "duration(%v)", tt.args.s)
+			assert.Equalf(t, tt.want, utils.Duration(tt.args.s), "duration(%v)", tt.args.s)
 		})
 	}
 }
diff --git a/cmd/gateway.go b/cmd/gateway.go
@@ -22,6 +22,7 @@ package cmd
 import (
 	"context"
 	"errors"
+	"github.com/juicedata/juicefs/pkg/utils"
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
@@ -249,9 +250,9 @@ func initForSvc(c *cli.Context, mp string, metaUrl string) (*vfs.Config, *fs.Fil
 	}()
 	vfsConf := getVfsConf(c, metaConf, format, chunkConf)
 	vfsConf.AccessLog = c.String("access-log")
-	vfsConf.AttrTimeout = duration(c.String("attr-cache"))
-	vfsConf.EntryTimeout = duration(c.String("entry-cache"))
-	vfsConf.DirEntryTimeout = duration(c.String("dir-entry-cache"))
+	vfsConf.AttrTimeout = utils.Duration(c.String("attr-cache"))
+	vfsConf.EntryTimeout = utils.Duration(c.String("entry-cache"))
+	vfsConf.DirEntryTimeout = utils.Duration(c.String("dir-entry-cache"))
 
 	initBackgroundTasks(c, vfsConf, metaConf, metaCli, blob, registerer, registry)
 	jfs, err := fs.NewFileSystem(vfsConf, metaCli, store)

diff --git a/cmd/mdtest.go b/cmd/mdtest.go
@@ -217,9 +217,9 @@ func initForMdtest(c *cli.Context, mp string, metaUrl string) *fs.FileSystem {
 
 	conf := getVfsConf(c, metaConf, format, chunkConf)
 	conf.AccessLog = c.String("access-log")
-	conf.AttrTimeout = duration(c.String("attr-cache"))
-	conf.EntryTimeout = duration(c.String("entry-cache"))
-	conf.DirEntryTimeout = duration(c.String("dir-entry-cache"))
+	conf.AttrTimeout = utils.Duration(c.String("attr-cache"))
+	conf.EntryTimeout = utils.Duration(c.String("entry-cache"))
+	conf.DirEntryTimeout = utils.Duration(c.String("dir-entry-cache"))
 
 	metricsAddr := exposeMetrics(c, registerer, registry)
 	m.InitMetrics(registerer)

diff --git a/cmd/mount.go b/cmd/mount.go
@@ -254,7 +254,7 @@ func getVfsConf(c *cli.Context, metaConf *meta.Config, format *meta.Format, chun
 		Format:          *format,
 		Version:         version.Version(),
 		Chunk:           chunkConf,
-		BackupMeta:      duration(c.String("backup-meta")),
+		BackupMeta:      utils.Duration(c.String("backup-meta")),
 		BackupSkipTrash: c.Bool("backup-skip-trash"),
 		Port:            &vfs.Port{DebugAgent: debugAgent, PyroscopeAddr: c.String("pyroscope")},
 		PrefixInternal:  c.Bool("prefix-internal"),
@@ -294,10 +294,10 @@ func getMetaConf(c *cli.Context, mp string, readOnly bool) *meta.Config {
 	conf.NoBGJob = c.Bool("no-bgjob")
 	conf.OpenCache = duration(c.String("open-cache"))
 	conf.OpenCacheLimit = c.Uint64("open-cache-limit")
-	conf.Heartbeat = duration(c.String("heartbeat"))
+	conf.Heartbeat = utils.Duration(c.String("heartbeat"))
 	conf.MountPoint = mp
 	conf.Subdir = c.String("subdir")
-	conf.SkipDirMtime = duration(c.String("skip-dir-mtime"))
+	conf.SkipDirMtime = utils.Duration(c.String("skip-dir-mtime"))
 	conf.Sid, _ = strconv.ParseUint(os.Getenv("_JFS_META_SID"), 10, 64)
 
 	atimeMode := c.String("atime-mode")
@@ -320,16 +320,16 @@ func getChunkConf(c *cli.Context, format *meta.Format) *chunk.Config {
 		Compress:   format.Compression,
 		HashPrefix: format.HashPrefix,
 
-		GetTimeout:    duration(c.String("get-timeout")),
-		PutTimeout:    duration(c.String("put-timeout")),
+		GetTimeout:    utils.Duration(c.String("get-timeout")),
+		PutTimeout:    utils.Duration(c.String("put-timeout")),
 		MaxUpload:     c.Int("max-uploads"),
 		MaxRetries:    c.Int("io-retries"),
 		Writeback:     c.Bool("writeback"),
 		Prefetch:      c.Int("prefetch"),
 		BufferSize:    utils.ParseBytes(c, "buffer-size", 'M'),
 		UploadLimit:   utils.ParseMbps(c, "upload-limit") * 1e6 / 8,
 		DownloadLimit: utils.ParseMbps(c, "download-limit") * 1e6 / 8,
-		UploadDelay:   duration(c.String("upload-delay")),
+		UploadDelay:   utils.Duration(c.String("upload-delay")),
 		UploadHours:   c.String("upload-hours"),
 
 		CacheDir:          c.String("cache-dir"),
@@ -339,8 +339,8 @@ func getChunkConf(c *cli.Context, format *meta.Format) *chunk.Config {
 		CacheFullBlock:    !c.Bool("cache-partial-only"),
 		CacheChecksum:     c.String("verify-cache-checksum"),
 		CacheEviction:     c.String("cache-eviction"),
-		CacheScanInterval: duration(c.String("cache-scan-interval")),
-		CacheExpire:       duration(c.String("cache-expire")),
+		CacheScanInterval: utils.Duration(c.String("cache-scan-interval")),
+		CacheExpire:       utils.Duration(c.String("cache-expire")),
 		AutoCreate:        true,
 	}
 	if chunkConf.UploadLimit == 0 {

diff --git a/cmd/mount_unix.go b/cmd/mount_unix.go
@@ -724,9 +724,9 @@ func mountMain(v *vfs.VFS, c *cli.Context) {
 		disableUpdatedb()
 	}
 	conf := v.Conf
-	conf.AttrTimeout = duration(c.String("attr-cache"))
-	conf.EntryTimeout = duration(c.String("entry-cache"))
-	conf.DirEntryTimeout = duration(c.String("dir-entry-cache"))
+	conf.AttrTimeout = utils.Duration(c.String("attr-cache"))
+	conf.EntryTimeout = utils.Duration(c.String("entry-cache"))
+	conf.DirEntryTimeout = utils.Duration(c.String("dir-entry-cache"))
 	conf.NonDefaultPermission = c.Bool("non-default-permission")
 	rootSquash := c.String("root-squash")
 	if rootSquash != "" {

diff --git a/docs/en/deployment/hadoop_java_sdk.md b/docs/en/deployment/hadoop_java_sdk.md
@@ -190,27 +190,28 @@ Please refer to the following table to set the relevant parameters of the JuiceF
 
 #### Other Configurations
 
-| Configuration             | Default Value | Description                                                                                                                                                                 |
-|---------------------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `juicefs.bucket`          |               | Specify a different endpoint for object storage                                                                                                                             |
-| `juicefs.debug`           | `false`       | Whether enable debug log                                                                                                                                                    |
-| `juicefs.access-log`      |               | Access log path. Ensure Hadoop application has write permission, e.g. `/tmp/juicefs.access.log`. The log file will rotate  automatically to keep at most 7 files.           |
-| `juicefs.superuser`       | `hdfs`        | The super user                                                                                                                                                              |
-| `juicefs.supergroup`      | `supergroup`  | The super user group                                                                                                                                                        |
-| `juicefs.users`           | `null`        | The path of username and UID list file, e.g. `jfs://name/etc/users`. The file format is `<username>:<UID>`, one user per line.                                              |
-| `juicefs.groups`          | `null`        | The path of group name, GID and group members list file, e.g. `jfs://name/etc/groups`. The file format is `<group-name>:<GID>:<username1>,<username2>`, one group per line. |
-| `juicefs.umask`           | `null`        | The umask used when creating files and directories (e.g. `0022`), default value is `fs.permissions.umask-mode`.                                                             |
-| `juicefs.push-gateway`    |               | [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) address, format is `<host>:<port>`.                                                                     |
-| `juicefs.push-auth`       |               | [Prometheus basic auth](https://prometheus.io/docs/guides/basic-auth) information, format is `<username>:<password>`.                                                       |
-| `juicefs.push-graphite`   |               | [Graphite](https://graphiteapp.org) address, format is `<host>:<port>`.                                                                                                     |
-| `juicefs.push-interval`   | 10            | Metric push interval (in seconds)                                                                                                                                           |
-| `juicefs.push-labels`     |               | Metric labels, format is `key1:value1;key2:value2`.                                    |
-| `juicefs.fast-resolve`    | `true`        | Whether enable faster metadata lookup using Redis Lua script                                                                                                                |
+| Configuration           | Default Value | Description                                                                                                                                                                 |
+|-------------------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `juicefs.bucket`        |               | Specify a different endpoint for object storage                                                                                                                             |
+| `juicefs.debug`         | `false`       | Whether enable debug log                                                                                                                                                    |
+| `juicefs.access-log`    |               | Access log path. Ensure Hadoop application has write permission, e.g. `/tmp/juicefs.access.log`. The log file will rotate  automatically to keep at most 7 files.           |
+| `juicefs.superuser`     | `hdfs`        | The super user                                                                                                                                                              |
+| `juicefs.supergroup`    | `supergroup`  | The super user group                                                                                                                                                        |
+| `juicefs.users`         | `null`        | The path of username and UID list file, e.g. `jfs://name/etc/users`. The file format is `<username>:<UID>`, one user per line.                                              |
+| `juicefs.groups`        | `null`        | The path of group name, GID and group members list file, e.g. `jfs://name/etc/groups`. The file format is `<group-name>:<GID>:<username1>,<username2>`, one group per line. |
+| `juicefs.umask`         | `null`        | The umask used when creating files and directories (e.g. `0022`), default value is `fs.permissions.umask-mode`.                                                             |
+| `juicefs.push-gateway`  |               | [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) address, format is `<host>:<port>`.                                                                     |
+| `juicefs.push-auth`     |               | [Prometheus basic auth](https://prometheus.io/docs/guides/basic-auth) information, format is `<username>:<password>`.                                                       |
+| `juicefs.push-graphite` |               | [Graphite](https://graphiteapp.org) address, format is `<host>:<port>`.                                                                                                     |
+| `juicefs.push-interval` | 10            | Metric push interval (in seconds)                                                                                                                                           |
+| `juicefs.push-labels`   |               | Metric labels, format is `key1:value1;key2:value2`.                                                                                                                         |
+| `juicefs.fast-resolve`  | `true`        | Whether enable faster metadata lookup using Redis Lua script                                                                                                                |
 | `juicefs.no-usage-report` | `false`       | Whether disable usage reporting. JuiceFS only collects anonymous usage data (e.g. version number), no user or any sensitive data will be collected.                         |
-| `juicefs.no-bgjob`        | `false`       | Disable background jobs (clean-up, backup, etc.)                                                                                                                            |
-| `juicefs.backup-meta`     | 3600          | Interval (in seconds) to automatically backup metadata in the object storage (0 means disable backup)                                                                       |
-|`juicefs.backup-skip-trash`| `false`       | Skip files and directories in trash when backup metadata.                                                                  |
-| `juicefs.heartbeat`       | 12            | Heartbeat interval (in seconds) between client and metadata engine. It's recommended that all clients use the same value.                                                   |
+| `juicefs.no-bgjob`      | `false`       | Disable background jobs (clean-up, backup, etc.)                                                                                                                            |
+| `juicefs.backup-meta`   | 3600          | Interval (in seconds) to automatically backup metadata in the object storage (0 means disable backup)                                                                       |
+| `juicefs.backup-skip-trash` | `false`       | Skip files and directories in trash when backup metadata.                                                                                                                   |
+| `juicefs.heartbeat`     | 12            | Heartbeat interval (in seconds) between client and metadata engine. It's recommended that all clients use the same value.                                                   |
+| `juicefs.skip-dir-mtime`              | 100ms         | Minimal duration to modify parent dir mtime.                                                                                                                                |
 
 #### Multiple file systems configuration
 

diff --git a/docs/zh_cn/deployment/hadoop_java_sdk.md b/docs/zh_cn/deployment/hadoop_java_sdk.md
@@ -204,15 +204,16 @@ make win
 | `juicefs.push-auth`       |              | [Prometheus 基本认证](https://prometheus.io/docs/guides/basic-auth)信息，格式为 `<username>:<password>`。              |
 | `juicefs.push-graphite`   |              | [Graphite](https://graphiteapp.org) 地址，格式为 `<host>:<port>`。                                                 |
 | `juicefs.push-interval`   | 10           | 指标推送的时间间隔，单位为秒。                                                                                             |
-| `juicefs.push-labels`     |              | 指标额外标签，格式为 `key1:value1;key2:value2`。                                                                |
+| `juicefs.push-labels`     |              | 指标额外标签，格式为 `key1:value1;key2:value2`。                                                                       |
 | `juicefs.fast-resolve`    | `true`       | 是否开启快速元数据查找（通过 Redis Lua 脚本实现）                                                                              |
 | `juicefs.no-usage-report` | `false`      | 是否上报数据。仅上版本号等使用量数据，不包含任何用户信息。                                                                               |
 | `juicefs.block.size`      | `134217728`  | 单位为字节，同 HDFS 的 `dfs.blocksize`，默认 128 MB                                                                    |
 | `juicefs.file.checksum`   | `false`      | DistCp 使用 `-update` 参数时，是否计算文件 Checksum                                                                     |
 | `juicefs.no-bgjob`        | `false`      | 是否关闭后台任务（清理、备份等）                                                                                            |
 | `juicefs.backup-meta`     | 3600         | 自动将 JuiceFS 元数据备份到对象存储间隔（单位：秒），设置为 0 关闭自动备份                                                                 |
-|`juicefs.backup-skip-trash`| `false`      | 备份元数据时忽略回收站中的文件和目录。                                                                                    |
+|`juicefs.backup-skip-trash`| `false`      | 备份元数据时忽略回收站中的文件和目录。                                                                                         |
 | `juicefs.heartbeat`       | 12           | 客户端和元数据引擎之间的心跳间隔（单位：秒），建议所有客户端都设置一样                                                                         |
+| `juicefs.skip-dir-mtime`  | 100ms        | 修改父目录 mtime 间隔。                                                                                             |
 
 #### 多文件系统配置
 

diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
@@ -247,3 +247,29 @@ func LookupGroup(name string) int {
 	groups[name] = gid
 	return gid
 }
+
+func Duration(s string) time.Duration {
+	if s == "" {
+		return 0
+	}
+	v, err := strconv.ParseFloat(s, 64)
+	if err == nil {
+		return time.Microsecond * time.Duration(v*1e6)
+	}
+
+	err = nil
+	var d time.Duration
+	p := strings.Index(s, "d")
+	if p >= 0 {
+		v, err = strconv.ParseFloat(s[:p], 64)
+	}
+	if err == nil && s[p+1:] != "" {
+		d, err = time.ParseDuration(s[p+1:])
+	}
+
+	if err != nil {
+		logger.Warnf("Invalid duration value: %s, setting it to 0", s)
+		return 0
+	}
+	return d + time.Hour*time.Duration(v*24)
+}
diff --git a/sdk/java/libjfs/main.go b/sdk/java/libjfs/main.go
@@ -300,6 +300,7 @@ type javaConf struct {
 	MaxUploads        int     `json:"maxUploads"`
 	MaxDeletes        int     `json:"maxDeletes"`
 	SkipDirNlink      int     `json:"skipDirNlink"`
+	SkipDirMtime      string  `json:"skipDirMtime"`
 	IORetries         int     `json:"ioRetries"`
 	GetTimeout        int     `json:"getTimeout"`
 	PutTimeout        int     `json:"putTimeout"`
@@ -441,6 +442,7 @@ func jfs_init(cname, jsonConf, user, group, superuser, supergroup *C.char) int64
 		metaConf.Retries = jConf.IORetries
 		metaConf.MaxDeletes = jConf.MaxDeletes
 		metaConf.SkipDirNlink = jConf.SkipDirNlink
+		metaConf.SkipDirMtime = utils.Duration(jConf.SkipDirMtime)
 		metaConf.ReadOnly = jConf.ReadOnly
 		metaConf.NoBGJob = jConf.NoBGJob || jConf.NoSession
 		metaConf.OpenCache = time.Duration(jConf.OpenCache * 1e9)

diff --git a/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java b/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
@@ -390,6 +390,7 @@ public void initialize(URI uri, Configuration conf) throws IOException {
     obj.put("maxUploads", Integer.valueOf(getConf(conf, "max-uploads", "20")));
     obj.put("maxDeletes", Integer.valueOf(getConf(conf, "max-deletes", "10")));
     obj.put("skipDirNlink", Integer.valueOf(getConf(conf, "skip-dir-nlink", "20")));
+    obj.put("skipDirMtime", getConf(conf, "skip-dir-mtime", "100ms"));
     obj.put("uploadLimit", Integer.valueOf(getConf(conf, "upload-limit", "0")));
     obj.put("downloadLimit", Integer.valueOf(getConf(conf, "download-limit", "0")));
     obj.put("ioRetries", Integer.valueOf(getConf(conf, "io-retries", "10")));