diff --git a/file.go b/file.go new file mode 100644 index 0000000..882064d --- /dev/null +++ b/file.go @@ -0,0 +1,86 @@ +// Copyright (c) Jeevanandam M. (https://github.com/jeevatkm) +// aahframework.org/vfs source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package vfs + +import ( + "errors" + "fmt" + "io" + "os" +) + +var _ File = (*file)(nil) +var _ Gziper = (*file)(nil) + +// File struct represents the virtual file or directory. +// +// Implements interface `vfs.File`. +type file struct { + *node + rs io.ReadSeeker + pos int +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// File and Directory operations +//______________________________________________________________________________ + +func (f *file) Read(b []byte) (int, error) { + return f.rs.Read(b) +} + +func (f *file) Seek(offset int64, whence int) (int64, error) { + return f.rs.Seek(offset, whence) +} + +func (f *file) Readdir(count int) ([]os.FileInfo, error) { + if !f.IsDir() { + return []os.FileInfo{}, &os.PathError{Op: "read", Path: f.node.name, Err: errors.New("vfs: cannot find the specified path")} + } + + if f.pos >= len(f.node.childInfos) && count > 0 { + return nil, io.EOF + } + + if count <= 0 || count > len(f.node.childInfos)-f.pos { + count = len(f.node.childInfos) - f.pos + } + + ci := f.node.childInfos[f.pos : f.pos+count] + f.pos += count + + return ci, nil +} + +func (f *file) Readdirnames(count int) (names []string, err error) { + var list []string + infos, err := f.Readdir(count) + if err != nil { + return list, err + } + + for _, v := range infos { + list = append(list, v.Name()) + } + + return list, nil +} + +func (f *file) Stat() (os.FileInfo, error) { + return f, nil +} + +func (f *file) Close() error { + if f.IsGzip() { + return f.rs.(io.Closer).Close() + } + return nil +} + +// String method Stringer interface. +func (f file) String() string { + return fmt.Sprintf(`file(name=%s dir=%v gzip=%v)`, + f.node.name, f.IsDir(), f.IsGzip()) +} diff --git a/fs.go b/fs.go new file mode 100644 index 0000000..bbf4efa --- /dev/null +++ b/fs.go @@ -0,0 +1,141 @@ +// Copyright (c) Jeevanandam M. (https://github.com/jeevatkm) +// aahframework.org/vfs source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package vfs + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" +) + +var ( + _ FileSystem = (*VFS)(nil) + + // ErrNotExists = errors.New("file or directory does not exist") +) + +// VFS represents Virtual File System (VFS), it operates in-memory. +// if file/directory doesn't exists on in-memory then it tries physical file system. +// +// VFS implements `vfs.FileSystem`, its a combination of package `os` and `ioutil` +// focused on Read-Only operations. +// +// Single point of access for all mounted virtual directories in aah application. +type VFS struct { + mounts map[string]*Mount +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// VFS FileSystem interface methods +//______________________________________________________________________________ + +// Open method behaviour is same as `os.Open`. +func (v *VFS) Open(name string) (File, error) { + m, err := v.FindMount(name) + if err != nil { + return nil, err + } + return m.Open(name) +} + +// Lstat method behaviour is same as `os.Lstat`. +func (v *VFS) Lstat(name string) (os.FileInfo, error) { + m, err := v.FindMount(name) + if err != nil { + return nil, err + } + return m.Lstat(name) +} + +// Stat method behaviour is same as `os.Stat` +func (v *VFS) Stat(name string) (os.FileInfo, error) { + m, err := v.FindMount(name) + if err != nil { + return nil, err + } + return m.Stat(name) +} + +// ReadFile method behaviour is same as `ioutil.ReadFile`. +func (v *VFS) ReadFile(filename string) ([]byte, error) { + m, err := v.FindMount(filename) + if err != nil { + return nil, err + } + return m.ReadFile(filename) +} + +// ReadDir method behaviour is same as `ioutil.ReadDir`. +func (v *VFS) ReadDir(dirname string) ([]os.FileInfo, error) { + m, err := v.FindMount(dirname) + if err != nil { + return nil, err + } + return m.ReadDir(dirname) +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// VFS methods +//______________________________________________________________________________ + +// FindMount method finds the mounted virtual directory by mount path. +// if found then returns `Mount` instance otherwise nil and error. +// +// Mount implements `vfs.FileSystem`, its a combination of package `os` and `ioutil` +// focused on Read-Only operations. +func (v *VFS) FindMount(name string) (*Mount, error) { + name = path.Clean(name) + for _, m := range v.mounts { + if m.vroot == name || strings.HasPrefix(name, m.tree.name+"/") { + return m, nil + } + } + return nil, &os.PathError{Op: "read", Path: name, Err: fmt.Errorf("mount not exist")} +} + +// AddMount method used to mount physical directory as a virtual mounted directory. +// +// Basically aah scans and application source files and builds each file from +// mounted source dierctory into binary for single binary build. +func (v *VFS) AddMount(mountPath, physicalPath string) error { + pp, err := filepath.Abs(filepath.Clean(physicalPath)) + if err != nil { + return err + } + + fi, err := os.Lstat(pp) + if err != nil { + return err + } + + if !fi.IsDir() { + return &os.PathError{Op: "addmount", Path: pp, Err: errors.New("is a file")} + } + + mp := filepath.ToSlash(path.Clean(mountPath)) + if mp == "" { + mp = path.Base(pp) + } + mp = path.Clean("/" + mp) + + if v.mounts == nil { + v.mounts = make(map[string]*Mount) + } + + if _, found := v.mounts[mp]; found { + return &os.PathError{Op: "addmount", Path: mp, Err: errors.New("already exists")} + } + + v.mounts[mp] = &Mount{ + vroot: mp, + proot: pp, + tree: newNode(mp, fi), + } + + return nil +} diff --git a/mount.go b/mount.go new file mode 100644 index 0000000..4fcb1a5 --- /dev/null +++ b/mount.go @@ -0,0 +1,179 @@ +// Copyright (c) Jeevanandam M. (https://github.com/jeevatkm) +// aahframework.org/vfs source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package vfs + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "sort" + "strings" +) + +var _ FileSystem = (*Mount)(nil) + +// Gzip Member header +// RFC 1952 section 2.3 and 2.3.1 +var gzipMemberHeader = []byte("\x1F\x8B\x08") + +// Mount struct represents mount of single physical directory into virtual directory. +// +// Mount implements `vfs.FileSystem`, its a combination of package `os` and `ioutil` +// focused on Read-Only operations. +type Mount struct { + vroot string + proot string + tree *node +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Mount's FileSystem interface +//______________________________________________________________________________ + +// Open method behaviour is same as `os.Open`. +func (m Mount) Open(name string) (File, error) { + f, err := m.open(name) + if os.IsNotExist(err) { + return m.openPhysical(name) + } + return f, err +} + +// Lstat method behaviour is same as `os.Lstat`. +func (m Mount) Lstat(name string) (os.FileInfo, error) { + f, err := m.open(name) + if os.IsNotExist(err) { + return os.Lstat(m.namePhysical(name)) + } + return f, err +} + +// Stat method behaviour is same as `os.Stat` +func (m Mount) Stat(name string) (os.FileInfo, error) { + f, err := m.open(name) + if os.IsNotExist(err) { + return os.Stat(m.namePhysical(name)) + } + return f, err +} + +// ReadFile method behaviour is same as `ioutil.ReadFile`. +func (m Mount) ReadFile(name string) ([]byte, error) { + f, err := m.Open(name) + if os.IsNotExist(err) { + f, err = m.openPhysical(name) + } + + if err != nil { + return nil, err + } + + fi, err := f.Stat() + if err != nil { + return nil, err + } + + if fi.IsDir() { + return nil, &os.PathError{Op: "read", Path: name, Err: errors.New("is a directory")} + } + + return ioutil.ReadAll(f) +} + +// ReadDir method behaviour is same as `ioutil.ReadDir`. +func (m Mount) ReadDir(dirname string) ([]os.FileInfo, error) { + f, err := m.open(dirname) + if os.IsNotExist(err) { + return ioutil.ReadDir(m.namePhysical(dirname)) + } + + if !f.IsDir() { + return nil, &os.PathError{Op: "read", Path: dirname, Err: errors.New("is a file")} + } + + list := append([]os.FileInfo{}, f.node.childInfos...) + sort.Sort(byName(list)) + + return list, nil +} + +// String method Stringer interface. +func (m Mount) String() string { + return fmt.Sprintf("mount(%s => %s)", m.vroot, m.proot) +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Mount adding file and directory +//______________________________________________________________________________ + +// Name method returns mounted path. +func (m *Mount) Name() string { + return m.vroot +} + +// AddDir method is to add directory node into VFS from mounted source directory. +func (m *Mount) AddDir(mountPath string, fi os.FileInfo) error { + n, err := m.tree.findNode(m.cleanDir(mountPath)) + switch { + case err != nil: + return err + case n == nil: + return nil + } + + n.addChild(newNode(mountPath, fi)) + return nil +} + +// AddFile method is to add file node into VFS from mounted source directory. +func (m *Mount) AddFile(mountPath string, fi os.FileInfo, data []byte) error { + n, err := m.tree.findNode(m.cleanDir(mountPath)) + if err != nil { + return err + } + + f := newNode(mountPath, fi) + f.data = data + n.addChild(f) + + return nil +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Mount unexported methods +//______________________________________________________________________________ + +func (m Mount) cleanDir(p string) string { + dp := strings.TrimPrefix(p, m.vroot) + return path.Dir(dp) +} + +func (m Mount) open(name string) (*file, error) { + if m.tree == nil { + return nil, os.ErrInvalid + } + + name = path.Clean(name) + if m.vroot == name { // extact match, root dir + return newFile(m.tree), nil + } + + return m.tree.find(strings.TrimPrefix(name, m.vroot)) +} + +func (m Mount) openPhysical(name string) (File, error) { + pname := m.namePhysical(name) + if _, err := os.Lstat(pname); os.IsNotExist(err) { + return nil, err + } + return os.Open(pname) +} + +func (m Mount) namePhysical(name string) string { + return filepath.Clean(filepath.FromSlash(filepath.Join(m.proot, name[len(m.vroot):]))) +} diff --git a/node.go b/node.go new file mode 100644 index 0000000..3c152e0 --- /dev/null +++ b/node.go @@ -0,0 +1,205 @@ +// Copyright (c) Jeevanandam M. (https://github.com/jeevatkm) +// aahframework.org/vfs source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package vfs + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "strings" + "time" +) + +var _ os.FileInfo = (*node)(nil) + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Node and its methods +//______________________________________________________________________________ + +// Node represents the virtual Node of file/directory on mounted VFS. +// +// Implements interfaces `os.FileInfo` and `vfs.Gziper`. +type node struct { + dir bool + size int64 + name string + modTime time.Time + data []byte + childInfos []os.FileInfo + childs map[string]*node +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// os.FileInfo interface +//______________________________________________________________________________ + +func (n node) Name() string { + return path.Base(n.name) +} + +func (n node) Size() int64 { + if n.IsDir() { + return 0 + } + return n.size +} + +func (n node) Mode() os.FileMode { + if n.IsDir() { + return 0755 | os.ModeDir // drwxr-xr-x + } + return 0444 // -r--r--r-- +} + +func (n node) ModTime() time.Time { + return n.modTime +} + +func (n node) IsDir() bool { + return n.dir +} + +func (n node) Sys() interface{} { + return nil +} + +// String method Stringer interface. +func (n node) String() string { + return fmt.Sprintf(`node(name=%s dir=%v gzip=%v)`, n.name, n.IsDir(), n.IsGzip()) +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Gziper interface methods +//______________________________________________________________________________ + +// IsGzip method returns true if its statisfies Gzip Member header +// RFC 1952 section 2.3 and 2.3.1 otherwise false. +func (n node) IsGzip() bool { + return bytes.HasPrefix(n.data, gzipMemberHeader) +} + +func (n node) RawBytes() []byte { + return n.data +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Node unexported methods +//______________________________________________________________________________ + +func (n *node) find(name string) (*file, error) { + tn, err := n.findNode(name) + if err != nil { + return nil, err + } + + if tn.match(name) { + return newFile(tn), nil + } + + return nil, os.ErrNotExist +} + +func (n *node) findNode(name string) (*node, error) { + switch name { + case ".": + return nil, nil + case "/": + return n, nil + } + + search := strings.Split(strings.TrimLeft(name, "/"), "/") + if len(search) == 0 { + return nil, os.ErrNotExist + } + + tn := n + for _, s := range search { + if t, found := tn.childs[s]; found { + tn = t + } else { + break + } + } + + return tn, nil +} + +func (n *node) match(name string) bool { + return strings.EqualFold(n.Name(), path.Base(name)) +} + +func (n *node) addChild(child *node) { + n.childInfos = append(n.childInfos, child) + n.childs[child.Name()] = child +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// GzipData type and methods +//______________________________________________________________________________ + +var _ ReadSeekCloser = (*gzipData)(nil) + +// GzipData my goal is to expose transparent behavior for regular and gzip +// data bytes. So I have designed gzip data handing. +type gzipData struct { + n *node + r *gzip.Reader + rpos int64 + spos int64 +} + +// aah vfs exposes transparent interaction for caller regardless of data bytes +// be it regular or gzip. +// +// Imitating `Read` same as `os.File.Read` for Gzip data; logic is from +// https://github.com/shurcooL/vfsgen +func (g *gzipData) Read(b []byte) (int, error) { + if g.rpos > g.spos { // to the beginning + if err := g.r.Reset(bytes.NewReader(g.n.data)); err != nil { + return 0, err + } + g.rpos = 0 + } + + if g.rpos < g.spos { // move forward + if _, err := io.CopyN(ioutil.Discard, g.r, g.spos-g.rpos); err != nil { + return 0, err + } + g.rpos = g.spos + } + + size, err := g.r.Read(b) + g.rpos += int64(size) + g.spos = g.rpos + + return size, err +} + +// aah vfs exposes transparent interaction for caller regardless of data bytes +// be it regular or gzip. +// +// Imitating `Seek` same as `os.File.Seek` for Gzip data; logic is from +// https://github.com/shurcooL/vfsgen +func (g *gzipData) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + g.spos = 0 + offset + case io.SeekCurrent: + g.spos += offset + case io.SeekEnd: + g.spos = g.n.size + offset + default: + return 0, fmt.Errorf("invalid whence: %v", whence) + } + return g.spos, nil +} + +func (g *gzipData) Close() error { + return g.r.Close() +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..47ee0dc --- /dev/null +++ b/util.go @@ -0,0 +1,80 @@ +// Copyright (c) Jeevanandam M. (https://github.com/jeevatkm) +// aahframework.org/vfs source code and usage is governed by a MIT style +// license that can be found in the LICENSE file. + +package vfs + +import ( + "bytes" + "compress/gzip" + "fmt" + "os" + "unicode/utf8" +) + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Package methods +//______________________________________________________________________________ + +// Bytes2QuotedStr method converts byte slice into string take care of +// valid UTF-8 string preparation. +func Bytes2QuotedStr(b []byte) string { + if len(b) == 0 { + return "" + } + + if utf8.Valid(b) { + b = sanitize(b) + } + + return fmt.Sprintf("%+q", b) +} + +//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +// Package unexported methods +//______________________________________________________________________________ + +func newNode(name string, fi os.FileInfo) *node { + return &node{ + dir: fi.IsDir(), + name: name, + modTime: fi.ModTime(), + childInfos: make([]os.FileInfo, 0), + childs: make(map[string]*node), + } +} + +func newFile(n *node) *file { + f := &file{node: n} + + if !f.IsDir() { + // transparent reading for caller regardless of data bytes. + f.rs = bytes.NewReader(f.node.data) + if f.IsGzip() { + r, _ := gzip.NewReader(f.rs) + f.rs = &gzipData{n: n, r: r} + } + } + + return f +} + +// https://github.com/golang/tools/blob/master/godoc/static/gen.go +// sanitize prepares a valid UTF-8 string as a raw string constant. +func sanitize(b []byte) []byte { + // Replace ` with `+"`"+` + b = bytes.Replace(b, []byte("`"), []byte("`+\"`\"+`"), -1) + + // Replace BOM with `+"\xEF\xBB\xBF"+` + // (A BOM is valid UTF-8 but not permitted in Go source files. + // I wouldn't bother handling this, but for some insane reason + // jquery.js has a BOM somewhere in the middle.) + return bytes.Replace(b, []byte("\xEF\xBB\xBF"), []byte("`+\"\\xEF\\xBB\\xBF\"+`"), -1) +} + +// byName implements sort.Interface +type byName []os.FileInfo + +func (f byName) Len() int { return len(f) } +func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() } +func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] } diff --git a/version.go b/version.go index 0fddf19..c8a9492 100644 --- a/version.go +++ b/version.go @@ -4,5 +4,5 @@ package vfs -// Version no. of VFS (Virtual File System) library by aah framework +// Version no. of VFS (Virtual FileSystem) library by aah framework const Version = "0.1.0" diff --git a/vfs.go b/vfs.go index 920d045..5f52cbe 100644 --- a/vfs.go +++ b/vfs.go @@ -2,6 +2,13 @@ // aahframework.org/vfs source code and usage is governed by a MIT style // license that can be found in the LICENSE file. +// Package vfs provides Virtual FileSystem (VFS) capability. Typically it reflects +// OS FileSystem behavior in-memory. +// +// aah vfs is Read-Only, even though vfs design nature could support Write +// operations. I have limited it. +// +// The methods should behave the same as those on an *os.File for Read-Only. package vfs import (