Skip to content

Commit

Permalink
for virtual path, we should use path.
Browse files Browse the repository at this point in the history 
for local path, we should use filepath.
  • Loading branch information
@davies
davies committed Nov 7, 2023
1 parent c682d71 commit f9cbb0d
Show file tree
Hide file tree
Showing 9 changed files with 58 additions and 63 deletions.
6 changes: 3 additions & 3 deletions cmd/bench.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func randRead(buf []byte) {

func (bc *benchCase) writeFiles(index int) {
for i := 0; i < bc.fcount; i++ {
fname := fmt.Sprintf("%s/%s.%d.%d", bc.bm.tmpdir, bc.name, index, i)
fname := filepath.Join(bc.bm.tmpdir, fmt.Sprintf("%s.%d.%d", bc.name, index, i))
fp, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
logger.Fatalf("Failed to open file %s: %s", fname, err)
Expand All @@ -140,7 +140,7 @@ func (bc *benchCase) writeFiles(index int) {

func (bc *benchCase) readFiles(index int) {
for i := 0; i < bc.fcount; i++ {
fname := fmt.Sprintf("%s/%s.%d.%d", bc.bm.tmpdir, bc.name, index, i)
fname := filepath.Join(bc.bm.tmpdir, fmt.Sprintf("%s.%d.%d", bc.name, index, i))
fp, err := os.Open(fname)
if err != nil {
logger.Fatalf("Failed to open file %s: %s", fname, err)
Expand All @@ -158,7 +158,7 @@ func (bc *benchCase) readFiles(index int) {

func (bc *benchCase) statFiles(index int) {
for i := 0; i < bc.fcount; i++ {
fname := fmt.Sprintf("%s/%s.%d.%d", bc.bm.tmpdir, bc.name, index, i)
fname := filepath.Join(bc.bm.tmpdir, fmt.Sprintf("%s.%d.%d", bc.name, index, i))
if _, err := os.Stat(fname); err != nil {
logger.Fatalf("Failed to stat file %s: %s", fname, err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/clone.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func clone(ctx *cli.Context) error {
return fmt.Errorf("lookup inode for %s: %s", srcPath, err)
}
dst := ctx.Args().Get(1)
if strings.HasSuffix(dst, "/") {
if strings.HasSuffix(dst, string(filepath.Separator)) {
dst = filepath.Join(dst, filepath.Base(srcPath))
}
if _, err := os.Stat(dst); err == nil {
Expand Down
12 changes: 6 additions & 6 deletions cmd/mdtest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"math/rand"
_ "net/http/pprof"
"os"
"path/filepath"
"path"
"sync"
"time"

Expand All @@ -44,7 +44,7 @@ func createDir(jfs *fs.FileSystem, root string, d int, width int) error {
}
if d > 0 {
for i := 0; i < width; i++ {
dn := filepath.Join(root, fmt.Sprintf("mdtest_tree.%d", i))
dn := path.Join(root, fmt.Sprintf("mdtest_tree.%d", i))
if err := createDir(jfs, dn, d-1, width); err != nil {
return err
}
Expand All @@ -56,7 +56,7 @@ func createDir(jfs *fs.FileSystem, root string, d int, width int) error {
func createFile(jfs *fs.FileSystem, bar *utils.Bar, np int, root string, d int, width, files, bytes int) error {
m := jfs.Meta()
for i := 0; i < files; i++ {
fn := filepath.Join(root, fmt.Sprintf("file.mdtest.%d.%d", np, i))
fn := path.Join(root, fmt.Sprintf("file.mdtest.%d.%d", np, i))
f, err := jfs.Create(ctx, fn, 0666, umask)
if err != 0 {
return fmt.Errorf("create %s: %s", fn, err)
Expand Down Expand Up @@ -88,7 +88,7 @@ func createFile(jfs *fs.FileSystem, bar *utils.Bar, np int, root string, d int,
dirs[i], dirs[j] = dirs[j], dirs[i]
})
for i := range dirs {
dn := filepath.Join(root, fmt.Sprintf("mdtest_tree.%d", dirs[i]))
dn := path.Join(root, fmt.Sprintf("mdtest_tree.%d", dirs[i]))
if err := createFile(jfs, bar, np, dn, d-1, width, files, bytes); err != nil {
return err
}
Expand All @@ -115,11 +115,11 @@ func runTest(jfs *fs.FileSystem, rootDir string, np, width, depth, files, bytes
if err := jfs.Mkdir(ctx, rootDir, 0777, umask); err != 0 {
logger.Errorf("mkdir %s: %s", rootDir, err)
}
root := filepath.Join(rootDir, "test-dir.0-0")
root := path.Join(rootDir, "test-dir.0-0")
if err := jfs.Mkdir(ctx, root, 0777, umask); err != 0 {
logger.Fatalf("Mkdir %s: %s", root, err)
}
root = filepath.Join(root, "mdtest_tree.0")
root = path.Join(root, "mdtest_tree.0")
if err := createDir(jfs, root, depth, width); err != nil {
logger.Fatalf("initialize: %s", err)
}
Expand Down
11 changes: 5 additions & 6 deletions cmd/object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"math/rand"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -128,16 +127,16 @@ func (j *juiceFS) Put(key string, in io.Reader) (err error) {
if object.PutInplace {
tmp = p
} else {
tmp = filepath.Join(filepath.Dir(p), "."+filepath.Base(p)+".tmp"+strconv.Itoa(rand.Int()))
tmp = path.Join(path.Dir(p), "."+path.Base(p)+".tmp"+strconv.Itoa(rand.Int()))
}
f, eno := j.jfs.Create(ctx, tmp, 0666, j.umask)
if eno == syscall.ENOENT {
_ = j.jfs.MkdirAll(ctx, filepath.Dir(tmp), 0777, j.umask)
_ = j.jfs.MkdirAll(ctx, path.Dir(tmp), 0777, j.umask)
f, eno = j.jfs.Create(ctx, tmp, 0666, j.umask)
}

if eno == syscall.EEXIST {
_ = j.jfs.Delete(ctx, filepath.Dir(tmp))
_ = j.jfs.Delete(ctx, path.Dir(tmp))
f, eno = j.jfs.Create(ctx, tmp, 0666, j.umask)
}

Expand Down Expand Up @@ -280,7 +279,7 @@ func (j *juiceFS) readDirSorted(dirname string, followLink bool) ([]*mEntry, sys
if fi.IsDir() {
mEntries[i] = &mEntry{fi, string(e.Name) + dirSuffix, false}
} else if fi.IsSymlink() && followLink {
fi2, err := j.jfs.Stat(ctx, filepath.Join(dirname, string(e.Name)))
fi2, err := j.jfs.Stat(ctx, path.Join(dirname, string(e.Name)))
if err != 0 {
mEntries[i] = &mEntry{fi, string(e.Name), true}
continue
Expand Down Expand Up @@ -331,7 +330,7 @@ func (j *juiceFS) Symlink(oldName, newName string) error {
p := j.path(newName)
err := j.jfs.Symlink(ctx, oldName, p)
if err == syscall.ENOENT {
_ = j.jfs.MkdirAll(ctx, filepath.Dir(p), 0777, j.umask)
_ = j.jfs.MkdirAll(ctx, path.Dir(p), 0777, j.umask)
err = j.jfs.Symlink(ctx, oldName, p)
}
return toError(err)
Expand Down
11 changes: 5 additions & 6 deletions pkg/meta/base.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"fmt"
"os"
"path"
"path/filepath"
"reflect"
"runtime"
"sort"
Expand Down Expand Up @@ -1907,21 +1906,21 @@ func (m *baseMeta) countDirNlink(ctx Context, inode Ino) (uint32, syscall.Errno)
return dirCounter, 0
}

type metaWalkFunc func(ctx Context, inode Ino, path string, attr *Attr)
type metaWalkFunc func(ctx Context, inode Ino, p string, attr *Attr)

func (m *baseMeta) walk(ctx Context, inode Ino, path string, attr *Attr, walkFn metaWalkFunc) syscall.Errno {
walkFn(ctx, inode, path, attr)
func (m *baseMeta) walk(ctx Context, inode Ino, p string, attr *Attr, walkFn metaWalkFunc) syscall.Errno {
walkFn(ctx, inode, p, attr)
var entries []*Entry
st := m.en.doReaddir(ctx, inode, 1, &entries, -1)
if st != 0 && st != syscall.ENOENT {
logger.Errorf("list %s: %s", path, st)
logger.Errorf("list %s: %s", p, st)
return st
}
for _, entry := range entries {
if !entry.Attr.Full {
entry.Attr.Parent = inode
}
if st := m.walk(ctx, entry.Inode, filepath.Join(path, string(entry.Name)), entry.Attr, walkFn); st != 0 {
if st := m.walk(ctx, entry.Inode, path.Join(p, string(entry.Name)), entry.Attr, walkFn); st != 0 {
return st
}
}
Expand Down
30 changes: 15 additions & 15 deletions pkg/object/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func (d *filestore) path(key string) string {
if strings.HasSuffix(d.root, dirSuffix) {
return filepath.Join(d.root, key)
}
return d.root + key
return filepath.Clean(d.root + key)
}

func (d *filestore) Head(key string) (Object, error) {
Expand Down Expand Up @@ -232,10 +232,10 @@ func (m *mEntry) IsDir() bool {
return m.FileInfo.IsDir()
}

// readDirSorted reads the directory named by dirname and returns
// readDirSorted reads the directory named by dir and returns
// a sorted list of directory entries.
func readDirSorted(dirname string, followLink bool) ([]*mEntry, error) {
f, err := os.Open(dirname)
func readDirSorted(dir string, followLink bool) ([]*mEntry, error) {
f, err := os.Open(dir)
if err != nil {
return nil, err
}
Expand All @@ -250,7 +250,7 @@ func readDirSorted(dirname string, followLink bool) ([]*mEntry, error) {
if e.IsDir() {
mEntries[i] = &mEntry{e, e.Name() + dirSuffix, nil, false}
} else if !e.Mode().IsRegular() && followLink {
fi, err := os.Stat(filepath.Join(dirname, e.Name()))
fi, err := os.Stat(filepath.Join(dir, e.Name()))
if err != nil {
mEntries[i] = &mEntry{e, e.Name(), nil, true}
continue
Expand Down Expand Up @@ -303,7 +303,7 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLi
for _, e := range entries {
p := path.Join(dir, e.Name())
if e.IsDir() {
p = filepath.ToSlash(p + "/")
p = p + "/"
}
if !strings.HasPrefix(p, d.root) {
continue
Expand All @@ -322,35 +322,35 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLi
return objs, nil
}

func (d *filestore) Chtimes(path string, mtime time.Time) error {
p := d.path(path)
func (d *filestore) Chtimes(key string, mtime time.Time) error {
p := d.path(key)
return os.Chtimes(p, mtime, mtime)
}

func (d *filestore) Chmod(path string, mode os.FileMode) error {
p := d.path(path)
func (d *filestore) Chmod(key string, mode os.FileMode) error {
p := d.path(key)
return os.Chmod(p, mode)
}

func (d *filestore) Chown(path string, owner, group string) error {
p := d.path(path)
func (d *filestore) Chown(key string, owner, group string) error {
p := d.path(key)
uid := utils.LookupUser(owner)
gid := utils.LookupGroup(group)
return os.Chown(p, uid, gid)
}

func newDisk(root, accesskey, secretkey, token string) (ObjectStorage, error) {
// For Windows, the path looks like /C:/a/b/c/
if runtime.GOOS == "windows" && strings.HasPrefix(root, "/") {
root = root[1:]
if runtime.GOOS == "windows" {
root = strings.TrimPrefix(root, "/")
}
if strings.HasSuffix(root, dirSuffix) {
logger.Debugf("Ensure directory %s", root)
if err := os.MkdirAll(root, 0777); err != nil {
return nil, fmt.Errorf("Creating directory %s failed: %q", root, err)
}
} else {
dir := path.Dir(root)
dir := filepath.Dir(root)
logger.Debugf("Ensure directory %s", dir)
if err := os.MkdirAll(dir, 0777); err != nil {
return nil, fmt.Errorf("Creating directory %s failed: %q", dir, err)
Expand Down
18 changes: 9 additions & 9 deletions pkg/object/hdfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (
"math/rand"
"os"
"os/user"
"path/filepath"
"path"
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -123,23 +123,23 @@ func (h *hdfsclient) Get(key string, off, limit int64) (io.ReadCloser, error) {
}

func (h *hdfsclient) Put(key string, in io.Reader) error {
path := h.path(key)
if strings.HasSuffix(path, dirSuffix) {
return h.c.MkdirAll(path, 0777&^h.umask)
p := h.path(key)
if strings.HasSuffix(p, dirSuffix) {
return h.c.MkdirAll(p, 0777&^h.umask)
}
var tmp string
if PutInplace {
tmp = path
tmp = p
} else {
tmp = filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s.tmp.%d", filepath.Base(path), rand.Int()))
tmp = path.Join(path.Dir(p), fmt.Sprintf(".%s.tmp.%d", path.Base(p), rand.Int()))
}
f, err := h.c.CreateFile(tmp, h.dfsReplication, 128<<20, 0666&^h.umask)
if !PutInplace {
defer func() { _ = h.c.Remove(tmp) }()
}
if err != nil {
if pe, ok := err.(*os.PathError); ok && pe.Err == os.ErrNotExist {
_ = h.c.MkdirAll(filepath.Dir(path), 0777&^h.umask)
_ = h.c.MkdirAll(path.Dir(p), 0777&^h.umask)
f, err = h.c.CreateFile(tmp, h.dfsReplication, 128<<20, 0666&^h.umask)
}
if pe, ok := err.(*os.PathError); ok && errors.Is(pe.Err, os.ErrExist) {
Expand All @@ -162,7 +162,7 @@ func (h *hdfsclient) Put(key string, in io.Reader) error {
return err
}
if !PutInplace {
err = h.c.Rename(tmp, path)
err = h.c.Rename(tmp, p)
}
return err
}
Expand All @@ -187,7 +187,7 @@ func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64, followL
dir := h.path(prefix)
var objs []Object
if !strings.HasSuffix(dir, dirSuffix) {
dir = filepath.Dir(dir)
dir = path.Dir(dir)
if !strings.HasSuffix(dir, dirSuffix) {
dir += dirSuffix
}
Expand Down
25 changes: 12 additions & 13 deletions pkg/object/nfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"os"
"os/user"
"path"
"path/filepath"
"sort"
"strings"
"syscall"
Expand Down Expand Up @@ -116,12 +115,12 @@ func (n *nfsStore) Get(key string, off, limit int64) (io.ReadCloser, error) {
return ff, err
}

func (n *nfsStore) mkdirAll(path string, perm fs.FileMode) error {
path = strings.TrimSuffix(path, "/")
fi, _, err := n.target.Lookup(path)
func (n *nfsStore) mkdirAll(p string, perm fs.FileMode) error {
p = strings.TrimSuffix(p, "/")
fi, _, err := n.target.Lookup(p)
if err == nil {
if fi.IsDir() {
logger.Tracef("nfs mkdir: path %s already exists", path)
logger.Tracef("nfs mkdir: path %s already exists", p)
return nil
} else {
return syscall.ENOTDIR
Expand All @@ -130,13 +129,13 @@ func (n *nfsStore) mkdirAll(path string, perm fs.FileMode) error {
return err
}

dir, _ := filepath.Split(path)
dir, _ := path.Split(p)
if dir != "." {
if err = n.mkdirAll(dir, perm); err != nil {
return err
}
}
_, err = n.target.Mkdir(path, perm)
_, err = n.target.Mkdir(p, perm)
return err
}

Expand All @@ -149,11 +148,11 @@ func (n *nfsStore) Put(key string, in io.Reader) error {
if PutInplace {
tmp = p
} else {
tmp = filepath.Join(filepath.Dir(p), "."+filepath.Base(p)+".tmp")
tmp = path.Join(path.Dir(p), "."+path.Base(p)+".tmp")
}
_, err := n.target.Create(tmp, 0777)
if os.IsNotExist(err) {
_ = n.mkdirAll(filepath.Dir(p), 0777)
_ = n.mkdirAll(path.Dir(p), 0777)
_, err = n.target.Create(tmp, 0777)
}
if os.IsExist(err) {
Expand Down Expand Up @@ -228,7 +227,7 @@ func (n *nfsStore) readDirSorted(dirname string, followLink bool) ([]*nfsEntry,
nfsEntries[i] = &nfsEntry{e, e.Name() + dirSuffix, nil, false}
} else if e.Attr.Attr.Type == nfs.NF3Lnk && followLink {
// follow symlink
fi, _, err := n.target.Lookup(filepath.Join(dirname, e.Name()))
fi, _, err := n.target.Lookup(path.Join(dirname, e.Name()))
if err != nil {
nfsEntries[i] = &nfsEntry{e, e.Name(), nil, true}
continue
Expand Down Expand Up @@ -279,9 +278,9 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64, followLin
return nil, err
}
for _, e := range entries {
p := filepath.Join(dir, e.Name())
p := path.Join(dir, e.Name())
if e.IsDir() && !e.isSymlink {
p = filepath.ToSlash(p + "/")
p = p + "/"
}
if !strings.HasPrefix(p, prefix) || (marker != "" && p <= marker) {
continue
Expand Down Expand Up @@ -351,7 +350,7 @@ func (n *nfsStore) Chown(path string, owner, group string) error {
func (n *nfsStore) Symlink(oldName, newName string) error {
newName = strings.TrimRight(newName, "/")
p := n.path(newName)
dir := filepath.Dir(p)
dir := path.Dir(p)
if _, _, err := n.target.Lookup(dir); err != nil && os.IsNotExist(err) {
if _, err := n.target.Mkdir(dir, os.FileMode(0777)); err != nil && !os.IsExist(err) {
return errors.Wrapf(err, "mkdir %s", dir)
Expand Down
Loading

0 comments on commit f9cbb0d

Please sign in to comment.