From 70f351e16c6ce21f121ec1c5ea26fad2fc68bf2c Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 4 May 2022 12:42:15 -0400 Subject: [PATCH 01/19] feat: get fs file info via http head request --- server/http.go | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/server/http.go b/server/http.go index da4bc9ae..62765397 100644 --- a/server/http.go +++ b/server/http.go @@ -93,6 +93,7 @@ func NewHTTPServer(cfg *Config) (*HTTPServer, error) { mux.HandleFunc(pat.Get("/v1/bio/:name"), s.handleGetUser) mux.HandleFunc(pat.Post("/v1/bio"), s.handlePostUser) mux.HandleFunc(pat.Post("/v1/encrypt-key"), s.handlePostEncryptKey) + mux.HandleFunc(pat.Head("/v1/fs/*"), s.handleHeadFile) mux.HandleFunc(pat.Get("/v1/fs/*"), s.handleGetFile) mux.HandleFunc(pat.Post("/v1/fs/*"), s.handlePostFile) mux.HandleFunc(pat.Delete("/v1/fs/*"), s.handleDeleteFile) @@ -314,6 +315,32 @@ func (s *HTTPServer) handlePostFile(w http.ResponseWriter, r *http.Request) { s.cfg.Stats.FSFileWritten(u.CharmID, fh.Size) } +func (s *HTTPServer) writeFileHeaders(w http.ResponseWriter, f fs.FileInfo) { + w.Header().Set("X-Name", f.Name()) + w.Header().Set("X-File-Mode", fmt.Sprintf("%d", f.Mode())) + w.Header().Set("X-Is-Dir", fmt.Sprintf("%t", f.IsDir())) + w.Header().Set("X-Last-Modified", f.ModTime().Format(http.TimeFormat)) + w.Header().Set("X-Size", fmt.Sprintf("%d", f.Size())) + // Backwards compatibility with old clients + w.Header().Set("Last-Modified", f.ModTime().Format(http.TimeFormat)) +} + +func (s *HTTPServer) handleHeadFile(w http.ResponseWriter, r *http.Request) { + u := s.charmUserFromRequest(w, r) + path := pattern.Path(r.Context()) + f, err := s.cfg.FileStore.Stat(u.CharmID, path) + if errors.Is(err, fs.ErrNotExist) { + s.renderCustomError(w, "file not found", http.StatusNotFound) + return + } + if err != nil { + log.Printf("cannot get file: %s", err) + s.renderError(w) + return + } + s.writeFileHeaders(w, f) +} + func (s *HTTPServer) handleGetFile(w http.ResponseWriter, r *http.Request) { u := s.charmUserFromRequest(w, r) path := filepath.Clean(pattern.Path(r.Context())) @@ -343,7 +370,7 @@ func (s *HTTPServer) handleGetFile(w http.ResponseWriter, r *http.Request) { w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat)) s.cfg.Stats.FSFileRead(u.CharmID, fi.Size()) } - w.Header().Set("X-File-Mode", fmt.Sprintf("%d", fi.Mode())) + s.writeFileHeaders(w, fi) _, err = io.Copy(w, f) if err != nil { log.Printf("cannot copy file: %s", err) From 66648d20f325fdb6cdb07ae381c7c2ef1a226f1b Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 4 May 2022 12:50:49 -0400 Subject: [PATCH 02/19] chore(fs): simplify WriteFile() signature --- cmd/fs.go | 6 ++++- fs/fs.go | 9 +++---- kv/client.go | 66 ++------------------------------------------------ server/http.go | 7 +++++- 4 files changed, 16 insertions(+), 72 deletions(-) diff --git a/cmd/fs.go b/cmd/fs.go index 88fee7d9..83d9f603 100644 --- a/cmd/fs.go +++ b/cmd/fs.go @@ -177,7 +177,11 @@ func (lrfs *localRemoteFS) write(name string, src fs.File) error { } case remotePath: if !stat.IsDir() { - return lrfs.cfs.WriteFile(p.path, src) + buf, err := io.ReadAll(src) + if err != nil { + return err + } + return lrfs.cfs.WriteFile(p.path, buf, stat.Mode()) } default: return fmt.Errorf("invalid path type") diff --git a/fs/fs.go b/fs/fs.go index faba2de2..0436e722 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -181,11 +181,8 @@ func (cfs *FS) ReadFile(name string) ([]byte, error) { // configured Charm Cloud server. The fs.FileMode is retained. If the file is // in a directory that doesn't exist, it and any needed subdirectories are // created. -func (cfs *FS) WriteFile(name string, src fs.File) error { - info, err := src.Stat() - if err != nil { - return err - } +func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { + src := bytes.NewBuffer(data) ebuf := bytes.NewBuffer(nil) eb, err := cfs.crypt.NewEncryptedWriter(ebuf) if err != nil { @@ -257,7 +254,7 @@ func (cfs *FS) WriteFile(name string, src fs.File) error { if err != nil { return err } - path := fmt.Sprintf("/v1/fs/%s?mode=%d", ep, info.Mode()) + path := fmt.Sprintf("/v1/fs/%s?mode=%d", ep, perm) headers := http.Header{ "Content-Type": {w.FormDataContentType()}, "Content-Length": {fmt.Sprintf("%d", contentLength)}, diff --git a/kv/client.go b/kv/client.go index 4a72752f..da0c62a2 100644 --- a/kv/client.go +++ b/kv/client.go @@ -7,64 +7,12 @@ import ( "math" "strconv" "strings" - "time" "github.com/charmbracelet/charm/client" charm "github.com/charmbracelet/charm/proto" badger "github.com/dgraph-io/badger/v3" ) -type kvFile struct { - data *bytes.Buffer - info *kvFileInfo -} - -type kvFileInfo struct { - name string - size int64 - mode fs.FileMode - modTime time.Time -} - -func (f *kvFileInfo) Name() string { - return f.name -} - -func (f *kvFileInfo) Size() int64 { - return f.size -} - -func (f *kvFileInfo) Mode() fs.FileMode { - return f.mode -} - -func (f *kvFileInfo) ModTime() time.Time { - return f.modTime -} - -func (f *kvFileInfo) IsDir() bool { - return f.mode&fs.ModeDir != 0 -} - -func (f *kvFileInfo) Sys() interface{} { - return nil -} - -func (f *kvFile) Stat() (fs.FileInfo, error) { - if f.info == nil { - return nil, fmt.Errorf("file info not set") - } - return f.info, nil -} - -func (f *kvFile) Close() error { - return nil -} - -func (f *kvFile) Read(p []byte) (n int, err error) { - return f.data.Read(p) -} - func (kv *KV) seqStorageKey(seq uint64) string { return strings.Join([]string{kv.name, fmt.Sprintf("%d", seq)}, "/") } @@ -72,21 +20,11 @@ func (kv *KV) seqStorageKey(seq uint64) string { func (kv *KV) backupSeq(from uint64, at uint64) error { buf := bytes.NewBuffer(nil) s := kv.DB.NewStreamAt(math.MaxUint64) - size, err := s.Backup(buf, from) - if err != nil { + if _, err := s.Backup(buf, from); err != nil { return err } name := kv.seqStorageKey(at) - src := &kvFile{ - data: buf, - info: &kvFileInfo{ - name: name, - size: int64(size), - mode: fs.FileMode(0o660), - modTime: time.Now(), - }, - } - return kv.fs.WriteFile(name, src) + return kv.fs.WriteFile(name, buf.Bytes(), fs.FileMode(0o660)) } func (kv *KV) restoreSeq(seq uint64) error { diff --git a/server/http.go b/server/http.go index 62765397..aa198c86 100644 --- a/server/http.go +++ b/server/http.go @@ -281,7 +281,12 @@ func (s *HTTPServer) handlePostSeq(w http.ResponseWriter, r *http.Request) { func (s *HTTPServer) handlePostFile(w http.ResponseWriter, r *http.Request) { u := s.charmUserFromRequest(w, r) path := filepath.Clean(pattern.Path(r.Context())) - ms := r.URL.Query().Get("mode") + ms := r.Header.Get("X-File-Mode") + if ms == "" { + // Deprecated: remove in next release + ms = r.URL.Query().Get("mode") + log.Printf("deprecated: use X-File-Mode header instead of mode query param. Please update your client.") + } m, err := strconv.ParseUint(ms, 10, 32) if err != nil { log.Printf("file mode not a number: %s", err) From d7a6dba1a862e7f9feada8d94cba9f44d63cf93a Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 4 May 2022 14:33:27 -0400 Subject: [PATCH 03/19] fix(fs): download data on demand and improve code implement fs.StatFS and make use of sysFuture --- fs/fs.go | 268 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 174 insertions(+), 94 deletions(-) diff --git a/fs/fs.go b/fs/fs.go index 0436e722..4ab99e6c 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -4,6 +4,7 @@ package fs import ( "bytes" "encoding/json" + "errors" "fmt" "io" "io/fs" @@ -20,6 +21,13 @@ import ( charm "github.com/charmbracelet/charm/proto" ) +var ( + // ErrIsDir is returned when trying to read a directory. + ErrIsDir = errors.New("is a directory") + // ErrNotDir is returned when trying to read a file that is not a directory. + ErrNotDir = errors.New("not a directory") +) + // FS is an implementation of fs.FS, fs.ReadFileFS and fs.ReadDirFS with // additional write methods. Data is stored across the network on a Charm Cloud // server, with encryption and decryption happening client-side. @@ -40,8 +48,13 @@ type FileInfo struct { sys interface{} } +type readDirFileFS interface { + fs.ReadDirFS + fs.ReadFileFS +} + type sysFuture struct { - fs fs.FS + fs readDirFileFS path string } @@ -87,100 +100,159 @@ func NewFSWithClient(cc *client.Client) (*FS, error) { return &FS{cc: cc, crypt: crypt}, nil } +// Stat returns an fs.FileInfo that describes the file. Implements fs.StatFS. +func (cfs *FS) Stat(name string) (fs.FileInfo, error) { + info := &FileInfo{ + sys: &sysFuture{ + fs: cfs, + path: name, + }, + } + ep, err := cfs.EncryptPath(name) + if err != nil { + return nil, pathError("stat", name, err) + } + p := fmt.Sprintf("/v1/fs/%s", ep) + resp, err := cfs.cc.AuthedRawRequest("HEAD", p) + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, fs.ErrNotExist + } else if err != nil { + return nil, pathError("stat", name, err) + } + defer resp.Body.Close() // nolint:errcheck + fileName := resp.Header.Get("X-Name") + if fileName == "" { + fileName = name + } + fileMode := resp.Header.Get("X-File-Mode") + mode, _ := strconv.ParseInt(fileMode, 10, 64) + isDir := resp.Header.Get("X-Is-Dir") + lastModified := resp.Header.Get("X-Last-Modified") + if lastModified == "" { + lastModified = resp.Header.Get("Last-Modified") + } + modTime, err := time.Parse(http.TimeFormat, lastModified) + if err != nil { + return nil, pathError("stat", name, err) + } + fileSize := resp.Header.Get("X-Size") + size, _ := strconv.ParseInt(fileSize, 10, 64) + info.FileInfo = charm.FileInfo{ + Name: path.Base(fileName), + Mode: fs.FileMode(mode), + IsDir: isDir == "true", + ModTime: modTime, + Size: size, + } + return info, nil +} + // Open implements Open for fs.FS. func (cfs *FS) Open(name string) (fs.File, error) { - f := &File{ - info: &FileInfo{}, + f := &File{} + info, err := cfs.Stat(name) + if err != nil { + return nil, err + } + f.info = info.(*FileInfo) + return f, nil +} + +// ReadFile implements fs.ReadFileFS. +func (cfs *FS) ReadFile(name string) ([]byte, error) { + info, err := cfs.Stat(name) + if err != nil { + return nil, err + } + if info.IsDir() { + return nil, pathError("open", name, ErrIsDir) } ep, err := cfs.EncryptPath(name) if err != nil { - return nil, pathError(name, err) + return nil, pathError("open", name, err) } p := fmt.Sprintf("/v1/fs/%s", ep) resp, err := cfs.cc.AuthedRawRequest("GET", p) if resp != nil && resp.StatusCode == http.StatusNotFound { return nil, fs.ErrNotExist } else if err != nil { - return nil, pathError(name, err) + return nil, pathError("open", name, err) } defer resp.Body.Close() // nolint:errcheck - switch resp.Header.Get("Content-Type") { - case "application/json": - dir := &charm.FileInfo{} - dec := json.NewDecoder(resp.Body) - err = dec.Decode(&dir) - if err != nil { - return nil, pathError(name, err) - } - f.info.FileInfo = *dir - var des []fs.DirEntry - for _, de := range dir.Files { - p := fmt.Sprintf("%s/%s", strings.Trim(ep, "/"), de.Name) - sf := sysFuture{ - fs: cfs, - path: p, - } - dn, err := cfs.crypt.DecryptLookupField(de.Name) - if err != nil { - return nil, pathError(name, err) - } - dei := FileInfo{ - FileInfo: de, - sys: sf, - } - dei.FileInfo.Name = dn - des = append(des, &dei) - } - f.info.sys = des case "application/octet-stream": - f.info.FileInfo.Name = path.Base(name) - m, err := strconv.ParseUint(resp.Header.Get("X-File-Mode"), 10, 32) - if err != nil { - return nil, pathError(name, err) - } - f.info.FileInfo.Mode = fs.FileMode(m) b := bytes.NewBuffer(nil) dec, err := cfs.crypt.NewDecryptedReader(resp.Body) if err != nil { - return nil, pathError(name, err) + return nil, pathError("open", name, err) } _, err = io.Copy(b, dec) if err != nil { return nil, err } - modTime, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) - if err != nil { - return nil, pathError(name, err) - } - f.data = io.NopCloser(b) - f.info.FileInfo.Size = int64(b.Len()) - f.info.FileInfo.ModTime = modTime - f.info.FileInfo.IsDir = false + return b.Bytes(), nil default: - return nil, pathError(name, fmt.Errorf("invalid content-type returned from server")) + return nil, pathError("open", name, fmt.Errorf("invalid content-type returned from server")) } - return f, nil } -// ReadFile implements fs.ReadFileFS. -func (cfs *FS) ReadFile(name string) ([]byte, error) { - buf := bytes.NewBuffer(nil) +// ReadDir reads the named directory and returns a list of directory entries. +func (cfs *FS) ReadDir(name string) ([]fs.DirEntry, error) { f, err := cfs.Open(name) if err != nil { return nil, err } - _, err = io.Copy(buf, f) + info, err := f.Stat() if err != nil { - return nil, err + return nil, pathError("open", name, err) + } + if !info.IsDir() { + return nil, pathError("open", name, ErrNotDir) + } + ep, err := cfs.EncryptPath(name) + if err != nil { + return nil, pathError("open", name, err) + } + p := fmt.Sprintf("/v1/fs/%s", ep) + resp, err := cfs.cc.AuthedRawRequest("GET", p) + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, fs.ErrNotExist + } else if err != nil { + return nil, pathError("open", name, err) + } + defer resp.Body.Close() // nolint:errcheck + switch resp.Header.Get("Content-Type") { + case "application/json": + dirs := []fs.DirEntry{} + dir := &charm.FileInfo{} + dec := json.NewDecoder(resp.Body) + err = dec.Decode(&dir) + if err != nil { + return nil, pathError("open", name, err) + } + for _, e := range dir.Files { + n, err := cfs.crypt.DecryptLookupField(e.Name) + if err != nil { + return nil, pathError("open", name, err) + } + e.Name = n + dirs = append(dirs, &FileInfo{ + FileInfo: e, + sys: sysFuture{ + fs: cfs, + path: path.Join(name, e.Name), + }, + }) + } + return dirs, nil + default: + return nil, pathError("open", name, fmt.Errorf("invalid content-type returned from server")) } - return buf.Bytes(), nil } -// WriteFile encrypts data from the src io.Reader and stores it on the -// configured Charm Cloud server. The fs.FileMode is retained. If the file is -// in a directory that doesn't exist, it and any needed subdirectories are -// created. +// WriteFile encrypts data from data and stores it on the configured Charm Cloud +// server. The fs.FileMode is retained. If the file is in a directory that +// doesn't exist, it and any needed subdirectories are created. func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { src := bytes.NewBuffer(data) ebuf := bytes.NewBuffer(nil) @@ -254,12 +326,19 @@ func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { if err != nil { return err } - path := fmt.Sprintf("/v1/fs/%s?mode=%d", ep, perm) + // Deprecated: remove mode from request query in favor of X-File-Mode + // header. + rp := fmt.Sprintf("/v1/fs/%s?mode=%d", ep, perm) headers := http.Header{ "Content-Type": {w.FormDataContentType()}, "Content-Length": {fmt.Sprintf("%d", contentLength)}, + "X-File-Mode": {fmt.Sprintf("%d", perm)}, + // These headers are ignored by the server but implemented for consistency purposes. + "X-Is-Dir": {"false"}, + "X-Last-Modified": {time.Now().Format(http.TimeFormat)}, + "X-Size": {fmt.Sprintf("%d", ebuf.Len())}, } - resp, err := cfs.cc.AuthedRequest("POST", path, headers, rr) + resp, err := cfs.cc.AuthedRequest("POST", rp, headers, rr) if err != nil { return err } @@ -280,19 +359,6 @@ func (cfs *FS) Remove(name string) error { return resp.Body.Close() } -// ReadDir reads the named directory and returns a list of directory entries. -func (cfs *FS) ReadDir(name string) ([]fs.DirEntry, error) { - f, err := cfs.Open(name) - if err == fs.ErrNotExist { - return []fs.DirEntry{}, nil - } - if err != nil { - return nil, err - } - defer f.Close() // nolint:errcheck - return f.(*File).ReadDir(0) -} - // Client returns the underlying *client.Client. func (cfs *FS) Client() *client.Client { return cfs.cc @@ -306,46 +372,60 @@ func (f *File) Stat() (fs.FileInfo, error) { // Read reads bytes from the file returning number of bytes read or an error. // The error io.EOF will be returned when there is nothing else to read. func (f *File) Read(b []byte) (int, error) { + if f.data == nil { + sys := f.info.Sys().(*sysFuture) + b, err := sys.fs.ReadFile(sys.path) + if err != nil { + return 0, err + } + f.data = io.NopCloser(bytes.NewBuffer(b)) + } return f.data.Read(b) } // ReadDir returns the directory entries for the directory file. If needed, the // directory listing will be resolved from the Charm Cloud server. func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { + var dirs []fs.DirEntry fi, err := f.Stat() if err != nil { return nil, err } if !fi.IsDir() { - return nil, fmt.Errorf("file is not a directory") - } - sys := fi.Sys() - if sys == nil { - return nil, fmt.Errorf("missing underlying directory data") + return nil, ErrNotDir } - var des []fs.DirEntry - switch v := sys.(type) { - case sysFuture: - des, err = v.resolve() + if f.data == nil { + sys := f.info.Sys().(*sysFuture) + dirs, err = sys.fs.ReadDir(sys.path) + if err != nil { + return nil, err + } + data := bytes.NewBuffer(nil) + err = json.NewEncoder(data).Encode(dirs) + if err != nil { + return nil, err + } + f.data = io.NopCloser(data) + } else { + err = json.NewDecoder(f.data).Decode(&dirs) if err != nil { return nil, err } - f.info.sys = des - case []fs.DirEntry: - des = v - default: - return nil, fmt.Errorf("invalid FileInfo sys type") } - if n > 0 && n < len(des) { - return des[:n], nil + if n > 0 && n < len(dirs) { + return dirs[:n], nil } - return des, nil + return dirs, nil } // Close closes the underlying file datasource. func (f *File) Close() error { +<<<<<<< Updated upstream // directories won't have data if f.data == nil { +======= + if f.Data == nil { +>>>>>>> Stashed changes return nil } return f.data.Close() @@ -437,9 +517,9 @@ func (sf sysFuture) resolve() ([]fs.DirEntry, error) { return sys.([]fs.DirEntry), nil } -func pathError(path string, err error) *fs.PathError { +func pathError(op, path string, err error) *fs.PathError { return &fs.PathError{ - Op: "open", + Op: op, Path: path, Err: err, } From b80f8fa487c99d18553628de9f262c5fc4871f35 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 4 May 2022 14:48:55 -0400 Subject: [PATCH 04/19] chore: clean --- fs/fs.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/fs/fs.go b/fs/fs.go index 4ab99e6c..0e0248e4 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -500,23 +500,6 @@ func (cfs *FS) DecryptPath(path string) (string, error) { return strings.Join(dps, "/"), nil } -func (sf sysFuture) resolve() ([]fs.DirEntry, error) { - f, err := sf.fs.Open(sf.path) - if err != nil { - return nil, err - } - defer f.Close() // nolint:errcheck - fi, err := f.Stat() - if err != nil { - return nil, err - } - sys := fi.Sys() - if sys == nil { - return nil, fmt.Errorf("missing dir entry results") - } - return sys.([]fs.DirEntry), nil -} - func pathError(op, path string, err error) *fs.PathError { return &fs.PathError{ Op: op, From 641721b8c5411691ba34394c1b9d28d6580ae191 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 4 May 2022 14:55:35 -0400 Subject: [PATCH 05/19] fix(fs): use concrete type for sys in FileInfo --- fs/fs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/fs.go b/fs/fs.go index 0e0248e4..6e4bbbd1 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -45,7 +45,7 @@ type File struct { // FileInfo implements the fs.FileInfo interface. type FileInfo struct { charm.FileInfo - sys interface{} + sys *sysFuture } type readDirFileFS interface { @@ -238,7 +238,7 @@ func (cfs *FS) ReadDir(name string) ([]fs.DirEntry, error) { e.Name = n dirs = append(dirs, &FileInfo{ FileInfo: e, - sys: sysFuture{ + sys: &sysFuture{ fs: cfs, path: path.Join(name, e.Name), }, From 54ddb103806c66b195b70ecf089c0bb3b98334b6 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 4 May 2022 15:41:16 -0400 Subject: [PATCH 06/19] fix(fs): simplify directory listing --- fs/fs.go | 59 ++++++------------------- server/http.go | 76 ++++++++++++++------------------- server/storage/local/storage.go | 10 ++--- 3 files changed, 50 insertions(+), 95 deletions(-) diff --git a/fs/fs.go b/fs/fs.go index 6e4bbbd1..dbc88a35 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -38,8 +38,8 @@ type FS struct { // File implements the fs.File interface. type File struct { - data io.ReadCloser - info *FileInfo + Data io.ReadCloser + Info fs.FileInfo } // FileInfo implements the fs.FileInfo interface. @@ -58,30 +58,6 @@ type sysFuture struct { path string } -// DirFile is a fs.File that represents a directory entry. -type DirFile struct { - Buffer *bytes.Buffer - FileInfo fs.FileInfo -} - -// Stat returns a fs.FileInfo. -func (df *DirFile) Stat() (fs.FileInfo, error) { - if df.FileInfo == nil { - return nil, fmt.Errorf("missing file info") - } - return df.FileInfo, nil -} - -// Read reads from the DirFile and satisfies fs.FS. -func (df *DirFile) Read(buf []byte) (int, error) { - return df.Buffer.Read(buf) -} - -// Close is a no-op but satisfies fs.FS. -func (df *DirFile) Close() error { - return nil -} - // NewFS returns an FS with the default configuration. func NewFS() (*FS, error) { cc, err := client.NewClientWithDefaults() @@ -154,7 +130,7 @@ func (cfs *FS) Open(name string) (fs.File, error) { if err != nil { return nil, err } - f.info = info.(*FileInfo) + f.Info = info.(*FileInfo) return f, nil } @@ -225,9 +201,7 @@ func (cfs *FS) ReadDir(name string) ([]fs.DirEntry, error) { case "application/json": dirs := []fs.DirEntry{} dir := &charm.FileInfo{} - dec := json.NewDecoder(resp.Body) - err = dec.Decode(&dir) - if err != nil { + if err := json.NewDecoder(resp.Body).Decode(&dir); err != nil { return nil, pathError("open", name, err) } for _, e := range dir.Files { @@ -366,21 +340,21 @@ func (cfs *FS) Client() *client.Client { // Stat returns an fs.FileInfo that describes the file. func (f *File) Stat() (fs.FileInfo, error) { - return f.info, nil + return f.Info, nil } // Read reads bytes from the file returning number of bytes read or an error. // The error io.EOF will be returned when there is nothing else to read. func (f *File) Read(b []byte) (int, error) { - if f.data == nil { - sys := f.info.Sys().(*sysFuture) + if f.Data == nil { + sys := f.Info.Sys().(*sysFuture) b, err := sys.fs.ReadFile(sys.path) if err != nil { return 0, err } - f.data = io.NopCloser(bytes.NewBuffer(b)) + f.Data = io.NopCloser(bytes.NewBuffer(b)) } - return f.data.Read(b) + return f.Data.Read(b) } // ReadDir returns the directory entries for the directory file. If needed, the @@ -394,8 +368,8 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { if !fi.IsDir() { return nil, ErrNotDir } - if f.data == nil { - sys := f.info.Sys().(*sysFuture) + if f.Data == nil { + sys := f.Info.Sys().(*sysFuture) dirs, err = sys.fs.ReadDir(sys.path) if err != nil { return nil, err @@ -405,9 +379,9 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { if err != nil { return nil, err } - f.data = io.NopCloser(data) + f.Data = io.NopCloser(data) } else { - err = json.NewDecoder(f.data).Decode(&dirs) + err = json.NewDecoder(f.Data).Decode(&dirs) if err != nil { return nil, err } @@ -420,15 +394,10 @@ func (f *File) ReadDir(n int) ([]fs.DirEntry, error) { // Close closes the underlying file datasource. func (f *File) Close() error { -<<<<<<< Updated upstream - // directories won't have data - if f.data == nil { -======= if f.Data == nil { ->>>>>>> Stashed changes return nil } - return f.data.Close() + return f.Data.Close() } // Name returns the file name. diff --git a/server/http.go b/server/http.go index aa198c86..56ad8e63 100644 --- a/server/http.go +++ b/server/http.go @@ -13,7 +13,6 @@ import ( "strconv" "strings" - charmfs "github.com/charmbracelet/charm/fs" charm "github.com/charmbracelet/charm/proto" "github.com/charmbracelet/charm/server/db" "github.com/charmbracelet/charm/server/storage" @@ -93,7 +92,7 @@ func NewHTTPServer(cfg *Config) (*HTTPServer, error) { mux.HandleFunc(pat.Get("/v1/bio/:name"), s.handleGetUser) mux.HandleFunc(pat.Post("/v1/bio"), s.handlePostUser) mux.HandleFunc(pat.Post("/v1/encrypt-key"), s.handlePostEncryptKey) - mux.HandleFunc(pat.Head("/v1/fs/*"), s.handleHeadFile) + // NOTE: pat.Get handles both GET and HEAD requests. mux.HandleFunc(pat.Get("/v1/fs/*"), s.handleGetFile) mux.HandleFunc(pat.Post("/v1/fs/*"), s.handlePostFile) mux.HandleFunc(pat.Delete("/v1/fs/*"), s.handleDeleteFile) @@ -320,65 +319,54 @@ func (s *HTTPServer) handlePostFile(w http.ResponseWriter, r *http.Request) { s.cfg.Stats.FSFileWritten(u.CharmID, fh.Size) } -func (s *HTTPServer) writeFileHeaders(w http.ResponseWriter, f fs.FileInfo) { - w.Header().Set("X-Name", f.Name()) - w.Header().Set("X-File-Mode", fmt.Sprintf("%d", f.Mode())) - w.Header().Set("X-Is-Dir", fmt.Sprintf("%t", f.IsDir())) - w.Header().Set("X-Last-Modified", f.ModTime().Format(http.TimeFormat)) - w.Header().Set("X-Size", fmt.Sprintf("%d", f.Size())) - // Backwards compatibility with old clients - w.Header().Set("Last-Modified", f.ModTime().Format(http.TimeFormat)) -} - -func (s *HTTPServer) handleHeadFile(w http.ResponseWriter, r *http.Request) { - u := s.charmUserFromRequest(w, r) - path := pattern.Path(r.Context()) - f, err := s.cfg.FileStore.Stat(u.CharmID, path) - if errors.Is(err, fs.ErrNotExist) { - s.renderCustomError(w, "file not found", http.StatusNotFound) - return - } - if err != nil { - log.Printf("cannot get file: %s", err) - s.renderError(w) - return - } - s.writeFileHeaders(w, f) -} - func (s *HTTPServer) handleGetFile(w http.ResponseWriter, r *http.Request) { u := s.charmUserFromRequest(w, r) path := filepath.Clean(pattern.Path(r.Context())) - f, err := s.cfg.FileStore.Get(u.CharmID, path) + fi, err := s.cfg.FileStore.Stat(u.CharmID, path) if errors.Is(err, fs.ErrNotExist) { s.renderCustomError(w, "file not found", http.StatusNotFound) return } - if err != nil { - log.Printf("cannot get file: %s", err) - s.renderError(w) - return - } - defer f.Close() // nolint:errcheck - fi, err := f.Stat() if err != nil { log.Printf("cannot get file info: %s", err) s.renderError(w) return } - - switch f.(type) { - case *charmfs.DirFile: + w.Header().Set("X-Name", fi.Name()) + w.Header().Set("X-File-Mode", fmt.Sprintf("%d", fi.Mode())) + w.Header().Set("X-Is-Dir", fmt.Sprintf("%t", fi.IsDir())) + w.Header().Set("X-Last-Modified", fi.ModTime().Format(http.TimeFormat)) + w.Header().Set("X-Size", fmt.Sprintf("%d", fi.Size())) + // Backwards compatibility with old clients + w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat)) + if fi.IsDir() { w.Header().Set("Content-Type", "application/json") - default: + } else { w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat)) s.cfg.Stats.FSFileRead(u.CharmID, fi.Size()) } - s.writeFileHeaders(w, fi) - _, err = io.Copy(w, f) - if err != nil { - log.Printf("cannot copy file: %s", err) + switch r.Method { + case "HEAD": + case "GET": + f, err := s.cfg.FileStore.Get(u.CharmID, path) + if errors.Is(err, fs.ErrNotExist) { + s.renderCustomError(w, "file not found", http.StatusNotFound) + return + } + if err != nil { + log.Printf("cannot get file: %s", err) + s.renderError(w) + return + } + defer f.Close() // nolint:errcheck + _, err = io.Copy(w, f) + if err != nil { + log.Printf("cannot copy file: %s", err) + s.renderError(w) + return + } + default: s.renderError(w) return } diff --git a/server/storage/local/storage.go b/server/storage/local/storage.go index 8aa97a03..b08f6b44 100644 --- a/server/storage/local/storage.go +++ b/server/storage/local/storage.go @@ -110,14 +110,12 @@ func (lfs *LocalFileStore) Get(charmID string, path string) (fs.File, error) { Files: fis, } buf := bytes.NewBuffer(nil) - enc := json.NewEncoder(buf) - err = enc.Encode(dir) - if err != nil { + if err := json.NewEncoder(buf).Encode(dir); err != nil { return nil, err } - return &charmfs.DirFile{ - Buffer: buf, - FileInfo: info, + return &charmfs.File{ + Data: io.NopCloser(buf), + Info: info, }, nil } return f, nil From d8a78d397f93932934e35785b8e58d23f5182a23 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Thu, 5 May 2022 13:22:28 -0400 Subject: [PATCH 07/19] fix(fs): ensure dir name is empty when requesting root --- server/storage/local/storage.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/storage/local/storage.go b/server/storage/local/storage.go index b08f6b44..65731566 100644 --- a/server/storage/local/storage.go +++ b/server/storage/local/storage.go @@ -41,9 +41,13 @@ func (lfs *LocalFileStore) Stat(charmID, path string) (fs.FileInfo, error) { if err != nil { return nil, err } + name := i.Name() + if name == charmID { + name = "" + } in := &charmfs.FileInfo{ FileInfo: charm.FileInfo{ - Name: i.Name(), + Name: name, IsDir: i.IsDir(), Size: i.Size(), ModTime: i.ModTime(), @@ -69,7 +73,7 @@ func (lfs *LocalFileStore) Stat(charmID, path string) (fs.FileInfo, error) { // Get returns an fs.File for the given Charm ID and path. func (lfs *LocalFileStore) Get(charmID string, path string) (fs.File, error) { fp := filepath.Join(lfs.Path, charmID, path) - info, err := os.Stat(fp) + info, err := lfs.Stat(charmID, path) if os.IsNotExist(err) { return nil, fs.ErrNotExist } @@ -103,8 +107,8 @@ func (lfs *LocalFileStore) Get(charmID string, path string) (fs.File, error) { } dir := charm.FileInfo{ Name: info.Name(), - IsDir: true, - Size: 0, + IsDir: info.IsDir(), + Size: info.Size(), ModTime: info.ModTime(), Mode: info.Mode(), Files: fis, From 2e4483ce4f74e10bcaa386162fd0b84599fdbac4 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Thu, 5 May 2022 13:23:15 -0400 Subject: [PATCH 08/19] fix(fs): error on missing file info headers --- fs/fs.go | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/fs/fs.go b/fs/fs.go index dbc88a35..ef3ebc1d 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -96,12 +96,22 @@ func (cfs *FS) Stat(name string) (fs.FileInfo, error) { return nil, pathError("stat", name, err) } defer resp.Body.Close() // nolint:errcheck + eName := path.Base(ep) + if eName == "." { + eName = "" + } + // Error if the header file name doesn't match the expected encrypted name. + // This is a sanity check to ensure that the server is using the right + // version. fileName := resp.Header.Get("X-Name") - if fileName == "" { - fileName = name + if fileName != eName { + return nil, pathError("stat", name, fs.ErrInvalid) } fileMode := resp.Header.Get("X-File-Mode") - mode, _ := strconv.ParseInt(fileMode, 10, 64) + mode, err := strconv.ParseInt(fileMode, 10, 64) + if err != nil { + return nil, pathError("stat", name, err) + } isDir := resp.Header.Get("X-Is-Dir") lastModified := resp.Header.Get("X-Last-Modified") if lastModified == "" { @@ -112,7 +122,10 @@ func (cfs *FS) Stat(name string) (fs.FileInfo, error) { return nil, pathError("stat", name, err) } fileSize := resp.Header.Get("X-Size") - size, _ := strconv.ParseInt(fileSize, 10, 64) + size, err := strconv.ParseInt(fileSize, 10, 64) + if err != nil { + return nil, pathError("stat", name, err) + } info.FileInfo = charm.FileInfo{ Name: path.Base(fileName), Mode: fs.FileMode(mode), @@ -441,15 +454,22 @@ func (fi *FileInfo) Info() (fs.FileInfo, error) { } // EncryptPath returns the encrypted path for a given path. -func (cfs *FS) EncryptPath(path string) (string, error) { +func (cfs *FS) EncryptPath(p string) (string, error) { eps := make([]string, 0) - path = strings.TrimPrefix(path, "charm:") - ps := strings.Split(path, "/") + p = strings.TrimPrefix(p, "charm:") + p = path.Clean(p) + if p == "." || p == "/" { + p = "" + } + ps := strings.Split(p, "/") for _, p := range ps { ep, err := cfs.crypt.EncryptLookupField(p) if err != nil { return "", err } + if ep == "" { + continue + } eps = append(eps, ep) } return strings.Join(eps, "/"), nil From 5dcd2ddbfb30e1d4858d35521ed56fd48c341e5e Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Thu, 5 May 2022 13:23:50 -0400 Subject: [PATCH 09/19] fix: return errors on fs tree --- cmd/fs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/fs.go b/cmd/fs.go index 83d9f603..56b64041 100644 --- a/cmd/fs.go +++ b/cmd/fs.go @@ -322,6 +322,9 @@ func fsTree(cmd *cobra.Command, args []string) error { return err } err = fs.WalkDir(lsfs, args[0], func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } fmt.Println(path) return nil }) From 2b34f70b4f49b2c5ad2ad82c57d69c2dd0901407 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Thu, 5 May 2022 15:48:10 -0400 Subject: [PATCH 10/19] fix(fs): kv when dir not exist --- fs/fs.go | 3 +++ kv/kv.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/fs.go b/fs/fs.go index ef3ebc1d..dfa61519 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -188,6 +188,9 @@ func (cfs *FS) ReadFile(name string) ([]byte, error) { // ReadDir reads the named directory and returns a list of directory entries. func (cfs *FS) ReadDir(name string) ([]fs.DirEntry, error) { f, err := cfs.Open(name) + if errors.Is(err, fs.ErrNotExist) { + return []fs.DirEntry{}, nil + } if err != nil { return nil, err } diff --git a/kv/kv.go b/kv/kv.go index 0376deb4..ea599234 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -53,7 +53,7 @@ func OpenWithDefaults(name string) (*KV, error) { if err != nil { return nil, err } - pn := filepath.Join(dd, "/kv/", name) + pn := filepath.Join(dd, "kv", name) opts := badger.DefaultOptions(pn).WithLoggingLevel(badger.ERROR) // By default we have no logger as it will interfere with Bubble Tea From 53ca339681c55460826b66025e8c2c321cd81aa4 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Mon, 9 May 2022 17:48:07 -0400 Subject: [PATCH 11/19] feat: use sasquatch metadata to store fs files metadata --- crypt/crypt.go | 25 +++++++++--- fs/fs.go | 35 +++++++++++------ go.mod | 8 ++-- go.sum | 14 ++++--- server/http.go | 1 - server/storage/local/storage.go | 70 ++++++++++++++++++++++++++++++--- 6 files changed, 120 insertions(+), 33 deletions(-) diff --git a/crypt/crypt.go b/crypt/crypt.go index db77500e..ad897810 100644 --- a/crypt/crypt.go +++ b/crypt/crypt.go @@ -54,34 +54,49 @@ func NewCrypt() (*Crypt, error) { // NewDecryptedReader creates a new Reader that will read from and decrypt the // passed in io.Reader of encrypted data. func (cr *Crypt) NewDecryptedReader(r io.Reader) (*DecryptedReader, error) { + dr, _, err := cr.NewDecryptedReaderWithMetadata(r) + return dr, err +} + +// NewDecryptedReaderWithMetadata creates a new Reader that will read from and decrypt the +// passed in io.Reader of encrypted data and its metadata. +func (cr *Crypt) NewDecryptedReaderWithMetadata(r io.Reader) (*DecryptedReader, []byte, error) { var sdr io.Reader + var md []byte dr := &DecryptedReader{} for _, k := range cr.keys { id, err := sasquatch.NewScryptIdentity(k.Key) if err != nil { - return nil, err + return nil, nil, err } - sdr, err = sasquatch.Decrypt(r, id) + sdr, md, err = sasquatch.DecryptWithMetadata(r, id) if err == nil { break } } if sdr == nil { - return nil, ErrIncorrectEncryptKeys + return nil, nil, ErrIncorrectEncryptKeys } dr.r = sdr - return dr, nil + return dr, md, nil } // NewEncryptedWriter creates a new Writer that encrypts all data and writes // the encrypted data to the supplied io.Writer. func (cr *Crypt) NewEncryptedWriter(w io.Writer) (*EncryptedWriter, error) { + return cr.NewEncryptedWriterWithMetadata(w, nil) +} + +// NewEncryptedWriterWithMetadata creates a new Writer that encrypts all data +// and writes the encrypted data along with their metadata to the supplied +// io.Writer. +func (cr *Crypt) NewEncryptedWriterWithMetadata(w io.Writer, metadata []byte) (*EncryptedWriter, error) { ew := &EncryptedWriter{} rec, err := sasquatch.NewScryptRecipient(cr.keys[0].Key) if err != nil { return ew, err } - sew, err := sasquatch.Encrypt(w, rec) + sew, err := sasquatch.EncryptWithMetadata(w, metadata, rec) if err != nil { return ew, err } diff --git a/fs/fs.go b/fs/fs.go index dfa61519..f5ae74e4 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -3,6 +3,7 @@ package fs import ( "bytes" + "encoding/gob" "encoding/json" "errors" "fmt" @@ -246,7 +247,23 @@ func (cfs *FS) ReadDir(name string) ([]fs.DirEntry, error) { func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { src := bytes.NewBuffer(data) ebuf := bytes.NewBuffer(nil) - eb, err := cfs.crypt.NewEncryptedWriter(ebuf) + ep, err := cfs.EncryptPath(name) + if err != nil { + return err + } + fi := charm.FileInfo{ + Name: path.Base(ep), // NOTE this is not the actual file name + IsDir: false, + Size: int64(len(data)), + Mode: perm, + ModTime: time.Now(), + } + + md := bytes.NewBuffer(nil) + if err = gob.NewEncoder(md).Encode(fi); err != nil { + return err + } + eb, err := cfs.crypt.NewEncryptedWriterWithMetadata(ebuf, md.Bytes()) if err != nil { return err } @@ -281,6 +298,10 @@ func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { if _, err := databuf.Read(boun); err != nil { return err } + // TODO: stream the encrypted data to the server, we need to calculate the + // content length manually. That is [multipart header length] + [encrypted + // data length] + [multipart footer length]. + // // headlen is the length of the multipart part header, bounlen is the length of the multipart boundary footer. contentLength := int64(headlen) + int64(ebuf.Len()) + int64(bounlen) // pipe the multipart request to the server @@ -312,21 +333,13 @@ func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { return } }() - ep, err := cfs.EncryptPath(name) - if err != nil { - return err - } - // Deprecated: remove mode from request query in favor of X-File-Mode - // header. + // Deprecated: remove mode from request query in favor of sasquatch + // metadata. rp := fmt.Sprintf("/v1/fs/%s?mode=%d", ep, perm) headers := http.Header{ "Content-Type": {w.FormDataContentType()}, "Content-Length": {fmt.Sprintf("%d", contentLength)}, "X-File-Mode": {fmt.Sprintf("%d", perm)}, - // These headers are ignored by the server but implemented for consistency purposes. - "X-Is-Dir": {"false"}, - "X-Last-Modified": {time.Now().Format(http.TimeFormat)}, - "X-Size": {fmt.Sprintf("%d", ebuf.Len())}, } resp, err := cfs.cc.AuthedRequest("POST", rp, headers, rr) if err != nil { diff --git a/go.mod b/go.mod index 21202635..22fdd2e1 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,12 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/muesli/go-app-paths v0.2.1 github.com/muesli/reflow v0.3.0 - github.com/muesli/sasquatch v0.0.0-20200811221207-66979d92330a + github.com/muesli/sasquatch v0.0.0-20220506035923-a0043c9268b2 github.com/muesli/toktok v0.1.0 github.com/prometheus/client_golang v0.9.3 github.com/spf13/cobra v1.0.0 goji.io v2.0.2+incompatible - golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 + golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 gopkg.in/square/go-jose.v2 v2.6.0 modernc.org/sqlite v1.14.8 @@ -68,8 +68,8 @@ require ( go.opencensus.io v0.22.5 // indirect golang.org/x/mod v0.3.0 // indirect golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect - golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect - golang.org/x/term v0.0.0-20210422114643-f5beecf764ed // indirect + golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect + golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 // indirect golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.25.0 // indirect diff --git a/go.sum b/go.sum index d2b4cdb7..0cb214ab 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,8 @@ github.com/muesli/go-app-paths v0.2.1/go.mod h1:SxS3Umca63pcFcLtbjVb+J0oD7cl4ixQ github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68/go.mod h1:Xk+z4oIWdQqJzsxyjgl3P22oYZnHdZ8FFTHAQQt5BMQ= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= -github.com/muesli/sasquatch v0.0.0-20200811221207-66979d92330a h1:Hw/15RYEOUD6T9UCRkUmNBa33kJkH33Fui6hE4sRLKU= -github.com/muesli/sasquatch v0.0.0-20200811221207-66979d92330a/go.mod h1:+XG0ne5zXWBTSbbe7Z3/RWxaT8PZY6zaZ1dX6KjprYY= +github.com/muesli/sasquatch v0.0.0-20220506035923-a0043c9268b2 h1:zzsAab9Nc5xuP+dI5pfS2bGmwJd5kgZuV7YeNW1/YcE= +github.com/muesli/sasquatch v0.0.0-20220506035923-a0043c9268b2/go.mod h1:G725xzk62J8hTlQIq820kJNEQP32RtK2ThDy9DqphSU= github.com/muesli/termenv v0.8.1/go.mod h1:kzt/D/4a88RoheZmwfqorY3A+tnsSMA9HJC/fQSFKo0= github.com/muesli/termenv v0.9.0/go.mod h1:R/LzAKf+suGs4IsO95y7+7DpFHO0KABgnZqtlyx2mBw= github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0/go.mod h1:Bd5NYQ7pd+SrtBSrSNoBBmXlcY8+Xj4BMJgh8qcZrvs= @@ -295,13 +295,13 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 h1:syTAU9FwmvzEoIYMqcPHOcVm4H3U5u90WsvuYgwpETU= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -357,11 +357,13 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210422114643-f5beecf764ed h1:Ei4bQjjpYUsS4efOUz+5Nz++IVkHk87n2zBA0NxBWc0= golang.org/x/term v0.0.0-20210422114643-f5beecf764ed/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 h1:EH1Deb8WZJ0xc0WK//leUHXcX9aLE5SymusoTmMZye8= +golang.org/x/term v0.0.0-20220411215600-e5f449aeb171/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/server/http.go b/server/http.go index 56ad8e63..f162cb2b 100644 --- a/server/http.go +++ b/server/http.go @@ -343,7 +343,6 @@ func (s *HTTPServer) handleGetFile(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") } else { w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat)) s.cfg.Stats.FSFileRead(u.CharmID, fi.Size()) } switch r.Method { diff --git a/server/storage/local/storage.go b/server/storage/local/storage.go index 65731566..a7ecd3fb 100644 --- a/server/storage/local/storage.go +++ b/server/storage/local/storage.go @@ -2,6 +2,7 @@ package localstorage import ( "bytes" + "encoding/gob" "encoding/json" "fmt" "io" @@ -12,6 +13,7 @@ import ( charmfs "github.com/charmbracelet/charm/fs" charm "github.com/charmbracelet/charm/proto" "github.com/charmbracelet/charm/server/storage" + "github.com/muesli/sasquatch" ) // LocalFileStore is a FileStore implementation that stores files locally in a @@ -41,19 +43,18 @@ func (lfs *LocalFileStore) Stat(charmID, path string) (fs.FileInfo, error) { if err != nil { return nil, err } - name := i.Name() - if name == charmID { - name = "" - } in := &charmfs.FileInfo{ FileInfo: charm.FileInfo{ - Name: name, + Name: i.Name(), IsDir: i.IsDir(), Size: i.Size(), ModTime: i.ModTime(), Mode: i.Mode(), }, } + if i.Name() == charmID { + in.FileInfo.Name = "" + } // Get the actual size of the files in a directory if i.IsDir() { in.FileInfo.Size = 0 @@ -61,11 +62,48 @@ func (lfs *LocalFileStore) Stat(charmID, path string) (fs.FileInfo, error) { if info.IsDir() { return nil } - in.FileInfo.Size += info.Size() + f, err := os.Open(path) + if err != nil { + return err + } + md, err := sasquatch.Metadata(f) + if err != nil { + return err + } + var fi charm.FileInfo + err = gob.NewDecoder(bytes.NewReader(md)).Decode(&fi) + if err != nil && err == io.EOF { + // No metadata, use the actual file size instead. + in.FileInfo.Size = info.Size() + return nil + } + if err != nil { + return err + } + in.FileInfo.Size += fi.Size return nil }); err != nil { return nil, err } + } else { + f, err := os.Open(fp) + if err != nil { + return nil, err + } + md, err := sasquatch.Metadata(f) + if err != nil { + return nil, err + } + var fi charm.FileInfo + err = gob.NewDecoder(bytes.NewReader(md)).Decode(&fi) + if err != nil && err != io.EOF { + return nil, err + } + if err != io.EOF { + // Metadata found, use it. + in.FileInfo = fi + } + in.FileInfo.Name = i.Name() // override the name } return in, nil } @@ -92,10 +130,30 @@ func (lfs *LocalFileStore) Get(charmID string, path string) (fs.File, error) { } fis := make([]charm.FileInfo, 0) for _, v := range rds { + ff, err := os.Open(filepath.Join(fp, v.Name())) + if err != nil { + return nil, err + } fi, err := v.Info() if err != nil { return nil, err } + if !v.IsDir() { + md, err := sasquatch.Metadata(ff) + if err != nil { + return nil, err + } + var cfi charm.FileInfo + err = gob.NewDecoder(bytes.NewReader(md)).Decode(&cfi) + if err != nil && err != io.EOF { + return nil, err + } + if err != io.EOF { + fi = &charmfs.FileInfo{ + FileInfo: cfi, + } + } + } fin := charm.FileInfo{ Name: v.Name(), IsDir: fi.IsDir(), From 9ecb597194e67f9f27e3d1b378dfa9c6f01ae410 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Tue, 10 May 2022 14:28:20 -0400 Subject: [PATCH 12/19] feat: use encrypted files metadata to report file info --- crypt/crypt.go | 50 +++++++++++--- fs/fs.go | 47 +++++++++++-- proto/fs.go | 13 ++-- server/http.go | 7 ++ server/storage/local/storage.go | 117 ++++++++++++-------------------- 5 files changed, 139 insertions(+), 95 deletions(-) diff --git a/crypt/crypt.go b/crypt/crypt.go index ad897810..2f7c2fe7 100644 --- a/crypt/crypt.go +++ b/crypt/crypt.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "io" + "log" "github.com/charmbracelet/charm/client" charm "github.com/charmbracelet/charm/proto" @@ -109,6 +110,18 @@ func (cr *Crypt) Keys() []*charm.EncryptKey { return cr.keys } +// Encrypt encrypts data. +func (cr *Crypt) Encrypt(b []byte) ([]byte, error) { + if b == nil { + return nil, nil + } + ct, err := siv.Encrypt(nil, []byte(cr.keys[0].Key[:32]), b, nil) + if err != nil { + return nil, err + } + return ct, nil +} + // EncryptLookupField will deterministically encrypt a string and the same // encrypted value every time this string is encrypted with the same // EncryptKey. This is useful if you need to look up an encrypted value without @@ -118,11 +131,32 @@ func (cr *Crypt) EncryptLookupField(field string) (string, error) { if field == "" { return "", nil } - ct, err := siv.Encrypt(nil, []byte(cr.keys[0].Key[:32]), []byte(field), nil) + b, err := cr.Encrypt([]byte(field)) if err != nil { return "", err } - return hex.EncodeToString(ct), nil + return hex.EncodeToString(b), nil +} + +// Decrypt decrypts data encrypted with Encrypt. +func (cr *Crypt) Decrypt(b []byte) ([]byte, error) { + if b == nil { + return nil, nil + } + var err error + var pt []byte + for _, k := range cr.keys { + pt, err = siv.Decrypt([]byte(k.Key[:32]), b, nil) + if err == nil { + break + } else { + log.Print(err) + } + } + if len(pt) == 0 { + return nil, ErrIncorrectEncryptKeys + } + return pt, nil } // DecryptLookupField decrypts a string encrypted with EncryptLookupField. @@ -134,15 +168,9 @@ func (cr *Crypt) DecryptLookupField(field string) (string, error) { if err != nil { return "", err } - var pt []byte - for _, k := range cr.keys { - pt, err = siv.Decrypt([]byte(k.Key[:32]), ct, nil) - if err == nil { - break - } - } - if len(pt) == 0 { - return "", ErrIncorrectEncryptKeys + pt, err := cr.Decrypt(ct) + if err != nil { + return "", err } return string(pt), nil } diff --git a/fs/fs.go b/fs/fs.go index f5ae74e4..1f3fa41c 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -3,6 +3,7 @@ package fs import ( "bytes" + "encoding/base64" "encoding/gob" "encoding/json" "errors" @@ -134,6 +135,24 @@ func (cfs *FS) Stat(name string) (fs.FileInfo, error) { ModTime: modTime, Size: size, } + metadata := resp.Header.Get("X-Metadata") + if metadata != "" { + b64, err := base64.StdEncoding.DecodeString(metadata) + if err != nil { + return nil, pathError("stat", name, err) + } + md, err := cfs.crypt.Decrypt(b64) + if err == nil { + var fi charm.FileInfo + err = gob.NewDecoder(bytes.NewBuffer(md)).Decode(&fi) + if err != nil && err != io.EOF { + return nil, pathError("stat", name, err) + } + if err != io.EOF { + info.FileInfo = fi + } + } + } return info, nil } @@ -226,12 +245,26 @@ func (cfs *FS) ReadDir(name string) ([]fs.DirEntry, error) { if err != nil { return nil, pathError("open", name, err) } - e.Name = n + fi := e + fi.Name = n + if !e.IsDir && e.Metadata != nil { + md, err := cfs.crypt.Decrypt(e.Metadata) + if err == nil { + var ei charm.FileInfo + err = gob.NewDecoder(bytes.NewBuffer(md)).Decode(&ei) + if err != nil && err != io.EOF { + return nil, pathError("open", name, err) + } + if err != io.EOF { + fi = ei + } + } + } dirs = append(dirs, &FileInfo{ - FileInfo: e, + FileInfo: fi, sys: &sysFuture{ fs: cfs, - path: path.Join(name, e.Name), + path: path.Join(name, fi.Name), }, }) } @@ -252,7 +285,7 @@ func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { return err } fi := charm.FileInfo{ - Name: path.Base(ep), // NOTE this is not the actual file name + Name: path.Base(name), IsDir: false, Size: int64(len(data)), Mode: perm, @@ -263,7 +296,11 @@ func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { if err = gob.NewEncoder(md).Encode(fi); err != nil { return err } - eb, err := cfs.crypt.NewEncryptedWriterWithMetadata(ebuf, md.Bytes()) + mde, err := cfs.crypt.Encrypt(md.Bytes()) + if err != nil { + return err + } + eb, err := cfs.crypt.NewEncryptedWriterWithMetadata(ebuf, mde) if err != nil { return err } diff --git a/proto/fs.go b/proto/fs.go index e159efe0..9f523bbd 100644 --- a/proto/fs.go +++ b/proto/fs.go @@ -7,12 +7,13 @@ import ( // FileInfo describes a file and is returned by Stat. type FileInfo struct { - Name string `json:"name"` - IsDir bool `json:"is_dir"` - Size int64 `json:"size"` - ModTime time.Time `json:"modtime"` - Mode fs.FileMode `json:"mode"` - Files []FileInfo `json:"files,omitempty"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` + ModTime time.Time `json:"modtime"` + Mode fs.FileMode `json:"mode"` + Metadata []byte `json:"metadata,omitempty"` + Files []FileInfo `json:"files,omitempty"` } // Add execute permissions to an fs.FileMode to mirror read permissions. diff --git a/server/http.go b/server/http.go index f162cb2b..f595fd1f 100644 --- a/server/http.go +++ b/server/http.go @@ -2,6 +2,7 @@ package server import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -13,6 +14,7 @@ import ( "strconv" "strings" + charmfs "github.com/charmbracelet/charm/fs" charm "github.com/charmbracelet/charm/proto" "github.com/charmbracelet/charm/server/db" "github.com/charmbracelet/charm/server/storage" @@ -332,6 +334,11 @@ func (s *HTTPServer) handleGetFile(w http.ResponseWriter, r *http.Request) { s.renderError(w) return } + cfi, ok := fi.(*charmfs.FileInfo) + if ok && cfi.FileInfo.Metadata != nil { + b64 := base64.StdEncoding.EncodeToString(cfi.FileInfo.Metadata) + w.Header().Set("X-Metadata", b64) + } w.Header().Set("X-Name", fi.Name()) w.Header().Set("X-File-Mode", fmt.Sprintf("%d", fi.Mode())) w.Header().Set("X-Is-Dir", fmt.Sprintf("%t", fi.IsDir())) diff --git a/server/storage/local/storage.go b/server/storage/local/storage.go index a7ecd3fb..82de2326 100644 --- a/server/storage/local/storage.go +++ b/server/storage/local/storage.go @@ -2,7 +2,6 @@ package localstorage import ( "bytes" - "encoding/gob" "encoding/json" "fmt" "io" @@ -36,25 +35,38 @@ func NewLocalFileStore(path string) (*LocalFileStore, error) { // Stat returns the FileInfo for the given Charm ID and path. func (lfs *LocalFileStore) Stat(charmID, path string) (fs.FileInfo, error) { fp := filepath.Join(lfs.Path, charmID, path) - i, err := os.Stat(fp) + f, err := os.Open(fp) if os.IsNotExist(err) { return nil, fs.ErrNotExist } if err != nil { return nil, err } + i, err := f.Stat() + if err != nil { + return nil, err + } + var md []byte + if !i.IsDir() { + md, err = sasquatch.Metadata(f) + if err != nil { + return nil, err + } + } + name := i.Name() + if name == charmID { + name = "" + } in := &charmfs.FileInfo{ FileInfo: charm.FileInfo{ - Name: i.Name(), - IsDir: i.IsDir(), - Size: i.Size(), - ModTime: i.ModTime(), - Mode: i.Mode(), + Name: name, + IsDir: i.IsDir(), + Size: i.Size(), + ModTime: i.ModTime(), + Mode: i.Mode(), + Metadata: md, }, } - if i.Name() == charmID { - in.FileInfo.Name = "" - } // Get the actual size of the files in a directory if i.IsDir() { in.FileInfo.Size = 0 @@ -62,54 +74,18 @@ func (lfs *LocalFileStore) Stat(charmID, path string) (fs.FileInfo, error) { if info.IsDir() { return nil } - f, err := os.Open(path) - if err != nil { - return err - } - md, err := sasquatch.Metadata(f) - if err != nil { - return err - } - var fi charm.FileInfo - err = gob.NewDecoder(bytes.NewReader(md)).Decode(&fi) - if err != nil && err == io.EOF { - // No metadata, use the actual file size instead. - in.FileInfo.Size = info.Size() - return nil - } - if err != nil { - return err - } - in.FileInfo.Size += fi.Size + in.FileInfo.Size += info.Size() return nil }); err != nil { return nil, err } - } else { - f, err := os.Open(fp) - if err != nil { - return nil, err - } - md, err := sasquatch.Metadata(f) - if err != nil { - return nil, err - } - var fi charm.FileInfo - err = gob.NewDecoder(bytes.NewReader(md)).Decode(&fi) - if err != nil && err != io.EOF { - return nil, err - } - if err != io.EOF { - // Metadata found, use it. - in.FileInfo = fi - } - in.FileInfo.Name = i.Name() // override the name } return in, nil } // Get returns an fs.File for the given Charm ID and path. func (lfs *LocalFileStore) Get(charmID string, path string) (fs.File, error) { + data := bytes.NewBuffer(nil) fp := filepath.Join(lfs.Path, charmID, path) info, err := lfs.Stat(charmID, path) if os.IsNotExist(err) { @@ -130,36 +106,28 @@ func (lfs *LocalFileStore) Get(charmID string, path string) (fs.File, error) { } fis := make([]charm.FileInfo, 0) for _, v := range rds { - ff, err := os.Open(filepath.Join(fp, v.Name())) - if err != nil { - return nil, err - } fi, err := v.Info() if err != nil { return nil, err } + var md []byte if !v.IsDir() { - md, err := sasquatch.Metadata(ff) + sf, err := os.Open(filepath.Join(fp, v.Name())) if err != nil { return nil, err } - var cfi charm.FileInfo - err = gob.NewDecoder(bytes.NewReader(md)).Decode(&cfi) - if err != nil && err != io.EOF { + md, err = sasquatch.Metadata(sf) + if err != nil { return nil, err } - if err != io.EOF { - fi = &charmfs.FileInfo{ - FileInfo: cfi, - } - } } fin := charm.FileInfo{ - Name: v.Name(), - IsDir: fi.IsDir(), - Size: fi.Size(), - ModTime: fi.ModTime(), - Mode: fi.Mode(), + Name: v.Name(), + IsDir: fi.IsDir(), + Size: fi.Size(), + ModTime: fi.ModTime(), + Mode: fi.Mode(), + Metadata: md, } fis = append(fis, fin) } @@ -171,16 +139,19 @@ func (lfs *LocalFileStore) Get(charmID string, path string) (fs.File, error) { Mode: info.Mode(), Files: fis, } - buf := bytes.NewBuffer(nil) - if err := json.NewEncoder(buf).Encode(dir); err != nil { + if err := json.NewEncoder(data).Encode(dir); err != nil { + return nil, err + } + } else { + _, err := io.Copy(data, f) + if err != nil { return nil, err } - return &charmfs.File{ - Data: io.NopCloser(buf), - Info: info, - }, nil } - return f, nil + return &charmfs.File{ + Data: io.NopCloser(data), + Info: info, + }, nil } // Put reads from the provided io.Reader and stores the data with the Charm ID From b748af7ac8ab6112bbc13746467e65f6947fb1a9 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Tue, 10 May 2022 14:55:51 -0400 Subject: [PATCH 13/19] fix: don't use the same file mode for directories --- server/storage/local/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/storage/local/storage.go b/server/storage/local/storage.go index 82de2326..57ee981c 100644 --- a/server/storage/local/storage.go +++ b/server/storage/local/storage.go @@ -165,7 +165,7 @@ func (lfs *LocalFileStore) Put(charmID string, path string, r io.Reader, mode fs if mode.IsDir() { return storage.EnsureDir(fp, mode) } - err := storage.EnsureDir(filepath.Dir(fp), mode) + err := storage.EnsureDir(filepath.Dir(fp), 0o755) if err != nil { return err } From 8ed3fa05639851a416ea4a810b069c7e7a4578a8 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Tue, 10 May 2022 15:57:50 -0400 Subject: [PATCH 14/19] feat: charmfs mkdir --- cmd/fs.go | 13 +++++++------ crypt/crypt.go | 3 --- fs/fs.go | 23 ++++++++++++++++++++++- server/http.go | 26 +++++++++++++++++--------- 4 files changed, 46 insertions(+), 19 deletions(-) diff --git a/cmd/fs.go b/cmd/fs.go index 56b64041..55af7e8d 100644 --- a/cmd/fs.go +++ b/cmd/fs.go @@ -176,13 +176,14 @@ func (lrfs *localRemoteFS) write(name string, src fs.File) error { } } case remotePath: - if !stat.IsDir() { - buf, err := io.ReadAll(src) - if err != nil { - return err - } - return lrfs.cfs.WriteFile(p.path, buf, stat.Mode()) + if stat.IsDir() { + return lrfs.cfs.MkdirAll(p.path, stat.Mode()) + } + buf, err := io.ReadAll(src) + if err != nil { + return err } + return lrfs.cfs.WriteFile(p.path, buf, stat.Mode()) default: return fmt.Errorf("invalid path type") } diff --git a/crypt/crypt.go b/crypt/crypt.go index 2f7c2fe7..13f0014e 100644 --- a/crypt/crypt.go +++ b/crypt/crypt.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "fmt" "io" - "log" "github.com/charmbracelet/charm/client" charm "github.com/charmbracelet/charm/proto" @@ -149,8 +148,6 @@ func (cr *Crypt) Decrypt(b []byte) ([]byte, error) { pt, err = siv.Decrypt([]byte(k.Key[:32]), b, nil) if err == nil { break - } else { - log.Print(err) } } if len(pt) == 0 { diff --git a/fs/fs.go b/fs/fs.go index 1f3fa41c..1bf33313 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -329,7 +329,6 @@ func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { if err := w.Close(); err != nil { return err } - w.Close() //nolint:errcheck bounlen := databuf.Len() boun := make([]byte, bounlen) if _, err := databuf.Read(boun); err != nil { @@ -399,6 +398,28 @@ func (cfs *FS) Remove(name string) error { return resp.Body.Close() } +// MkdirAll creates a directory on the configured Charm Cloud server. +func (cfs *FS) MkdirAll(path string, perm fs.FileMode) error { + ep, err := cfs.EncryptPath(path) + if err != nil { + return err + } + if !perm.IsDir() { + return fmt.Errorf("%q is not a directory", path) + } + // Deprecated: remove mode from request query in favor of sasquatch + // metadata. + rp := fmt.Sprintf("/v1/fs/%s?mode=%d", ep, perm) + headers := http.Header{ + "X-File-Mode": {fmt.Sprintf("%d", perm)}, + } + resp, err := cfs.cc.AuthedRequest("POST", rp, headers, nil) + if err != nil { + return err + } + return resp.Body.Close() +} + // Client returns the underlying *client.Client. func (cfs *FS) Client() *client.Client { return cfs.cc diff --git a/server/http.go b/server/http.go index f595fd1f..9ce11dba 100644 --- a/server/http.go +++ b/server/http.go @@ -9,6 +9,7 @@ import ( "io" "io/fs" "log" + "mime/multipart" "net/http" "path/filepath" "strconv" @@ -294,14 +295,19 @@ func (s *HTTPServer) handlePostFile(w http.ResponseWriter, r *http.Request) { s.renderError(w) return } - f, fh, err := r.FormFile("data") - if err != nil { - log.Printf("cannot parse form data: %s", err) - s.renderError(w) - return + mode := fs.FileMode(m) + var f multipart.File + var fh *multipart.FileHeader + if !mode.IsDir() { + f, fh, err = r.FormFile("data") + if err != nil { + log.Printf("cannot parse form data: %s", err) + s.renderError(w) + return + } + defer f.Close() // nolint:errcheck } - defer f.Close() // nolint:errcheck - if s.cfg.UserMaxStorage > 0 { + if s.cfg.UserMaxStorage > 0 && fh != nil { stat, err := s.cfg.FileStore.Stat(u.CharmID, "") if err != nil { log.Printf("cannot stat user storage: %s", err) @@ -313,12 +319,14 @@ func (s *HTTPServer) handlePostFile(w http.ResponseWriter, r *http.Request) { return } } - if err := s.cfg.FileStore.Put(u.CharmID, path, f, fs.FileMode(m)); err != nil { + if err := s.cfg.FileStore.Put(u.CharmID, path, f, mode); err != nil { log.Printf("cannot post file: %s", err) s.renderError(w) return } - s.cfg.Stats.FSFileWritten(u.CharmID, fh.Size) + if fh != nil { + s.cfg.Stats.FSFileWritten(u.CharmID, fh.Size) + } } func (s *HTTPServer) handleGetFile(w http.ResponseWriter, r *http.Request) { From 7ec8c9051d4c3402874950eec7b026c0810d5947 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Thu, 12 May 2022 12:32:42 -0400 Subject: [PATCH 15/19] feat: add server version number --- main.go | 2 ++ server/http.go | 9 ++++++--- server/middleware.go | 8 ++++++++ server/server.go | 6 ++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 06b7139f..9affeaff 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/charm/client" "github.com/charmbracelet/charm/cmd" + "github.com/charmbracelet/charm/server" "github.com/charmbracelet/charm/ui" "github.com/charmbracelet/charm/ui/common" "github.com/spf13/cobra" @@ -62,6 +63,7 @@ func init() { } } rootCmd.Version = Version + server.Version = Version rootCmd.AddCommand( cmd.BioCmd, diff --git a/server/http.go b/server/http.go index 9ce11dba..0a8da680 100644 --- a/server/http.go +++ b/server/http.go @@ -52,9 +52,11 @@ type providerJSON struct { func NewHTTPServer(cfg *Config) (*HTTPServer, error) { healthMux := http.NewServeMux() // No auth health check endpoint - healthMux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "We live!") - })) + healthMux.Handle("/", versionMiddleware( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "We live!") + }), + )) health := &http.Server{ Addr: fmt.Sprintf("%s:%d", cfg.BindAddr, cfg.HealthPort), Handler: healthMux, @@ -86,6 +88,7 @@ func NewHTTPServer(cfg *Config) (*HTTPServer, error) { return nil, err } + mux.Use(versionMiddleware) mux.Use(babylogger.Middleware) mux.Use(PublicPrefixesMiddleware([]string{"/v1/public/", "/.well-known/"})) mux.Use(jwtMiddleware) diff --git a/server/middleware.go b/server/middleware.go index 85a511a4..8526f64f 100644 --- a/server/middleware.go +++ b/server/middleware.go @@ -20,6 +20,14 @@ var ctxUserKey contextKey = "charmUser" // MaxFSRequestSize is the maximum size of a request body for fs endpoints. var MaxFSRequestSize int64 = 1024 * 1024 * 1024 // 1GB +// versionMiddleware is a middleware that adds a version header to the response. +var versionMiddleware = func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Version", Version) + h.ServeHTTP(w, r) + }) +} + // RequestLimitMiddleware limits the request body size to the specified limit. func RequestLimitMiddleware() func(http.Handler) http.Handler { return func(h http.Handler) http.Handler { diff --git a/server/server.go b/server/server.go index 81b3da08..ae5b6586 100644 --- a/server/server.go +++ b/server/server.go @@ -23,6 +23,12 @@ import ( "golang.org/x/sync/errgroup" ) +var ( + // Version is the version of the Charm Cloud server. This is set at build + // time by main.go. + Version = "" +) + // Config is the configuration for the Charm server. type Config struct { BindAddr string `env:"CHARM_SERVER_BIND_ADDRESS" envDefault:""` From 0e373f773ce5c82ed8b2de98b76d3f6266b0a17a Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Thu, 12 May 2022 13:40:56 -0400 Subject: [PATCH 16/19] refactor: move server config into its own module --- cmd/post_news.go | 3 +- cmd/serve.go | 3 +- cmd/serve_migrate.go | 4 +- main.go | 4 +- server/auth.go | 2 +- server/config/config.go | 121 ++++++++++++++++++++++++++++++++++++ server/http.go | 25 ++++---- server/jwk.go | 32 ---------- server/jwt/jwt.go | 48 +++++++++++++++ server/middleware.go | 3 +- server/server.go | 128 +++++---------------------------------- server/ssh.go | 15 ++--- testserver/testserver.go | 3 +- 13 files changed, 217 insertions(+), 174 deletions(-) create mode 100644 server/config/config.go delete mode 100644 server/jwk.go create mode 100644 server/jwt/jwt.go diff --git a/cmd/post_news.go b/cmd/post_news.go index 343b7146..70b9cc5b 100644 --- a/cmd/post_news.go +++ b/cmd/post_news.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/charmbracelet/charm/server" + "github.com/charmbracelet/charm/server/config" "github.com/charmbracelet/keygen" "github.com/spf13/cobra" ) @@ -21,7 +22,7 @@ var ( Short: "Post news to the self-hosted Charm server.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - cfg := server.DefaultConfig() + cfg := config.DefaultConfig() if serverDataDir != "" { cfg.DataDir = serverDataDir } diff --git a/cmd/serve.go b/cmd/serve.go index eb02f868..0d9c3087 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -10,6 +10,7 @@ import ( "time" "github.com/charmbracelet/charm/server" + "github.com/charmbracelet/charm/server/config" "github.com/charmbracelet/keygen" "github.com/spf13/cobra" ) @@ -30,7 +31,7 @@ var ( Long: paragraph("Start the SSH and HTTP servers needed to power a SQLite-backed Charm Cloud."), Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - cfg := server.DefaultConfig() + cfg := config.DefaultConfig() if serverHTTPPort != 0 { cfg.HTTPPort = serverHTTPPort } diff --git a/cmd/serve_migrate.go b/cmd/serve_migrate.go index fec7be08..83b7014a 100644 --- a/cmd/serve_migrate.go +++ b/cmd/serve_migrate.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" - "github.com/charmbracelet/charm/server" + "github.com/charmbracelet/charm/server/config" "github.com/charmbracelet/charm/server/db/sqlite" "github.com/charmbracelet/charm/server/db/sqlite/migration" "github.com/spf13/cobra" @@ -24,7 +24,7 @@ var ServeMigrationCmd = &cobra.Command{ Long: paragraph("Run the server migration tool to migrate the database."), Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - cfg := server.DefaultConfig() + cfg := config.DefaultConfig() dp := filepath.Join(cfg.DataDir, "db", sqlite.DbName) _, err := os.Stat(dp) if err != nil { diff --git a/main.go b/main.go index 9affeaff..4d0d3342 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/charm/client" "github.com/charmbracelet/charm/cmd" - "github.com/charmbracelet/charm/server" + "github.com/charmbracelet/charm/server/config" "github.com/charmbracelet/charm/ui" "github.com/charmbracelet/charm/ui/common" "github.com/spf13/cobra" @@ -63,7 +63,7 @@ func init() { } } rootCmd.Version = Version - server.Version = Version + config.Version = Version rootCmd.AddCommand( cmd.BioCmd, diff --git a/server/auth.go b/server/auth.go index cda5a196..364f4117 100644 --- a/server/auth.go +++ b/server/auth.go @@ -59,7 +59,7 @@ func (me *SSHServer) handleAPIAuth(s ssh.Session) { me.errorLog.Printf("Error fetching encrypt keys: %s\n", err) return } - httpScheme := me.config.httpURL().Scheme + httpScheme := me.config.HTTPURL().Scheme _ = me.sendJSON(s, charm.Auth{ JWT: j, ID: u.CharmID, diff --git a/server/config/config.go b/server/config/config.go new file mode 100644 index 00000000..2cfd851d --- /dev/null +++ b/server/config/config.go @@ -0,0 +1,121 @@ +package config + +import ( + "crypto/tls" + "fmt" + "log" + "net/url" + + "github.com/caarlos0/env/v6" + charm "github.com/charmbracelet/charm/proto" + "github.com/charmbracelet/charm/server/db" + "github.com/charmbracelet/charm/server/jwt" + "github.com/charmbracelet/charm/server/stats" + "github.com/charmbracelet/charm/server/storage" +) + +var ( + // Version is the version of the Charm Cloud server. This is set at build + // time by main.go. + Version = "" +) + +// Config is the configuration for the Charm server. +type Config struct { + BindAddr string `env:"CHARM_SERVER_BIND_ADDRESS" envDefault:""` + Host string `env:"CHARM_SERVER_HOST" envDefault:"localhost"` + SSHPort int `env:"CHARM_SERVER_SSH_PORT" envDefault:"35353"` + HTTPPort int `env:"CHARM_SERVER_HTTP_PORT" envDefault:"35354"` + StatsPort int `env:"CHARM_SERVER_STATS_PORT" envDefault:"35355"` + HealthPort int `env:"CHARM_SERVER_HEALTH_PORT" envDefault:"35356"` + DataDir string `env:"CHARM_SERVER_DATA_DIR" envDefault:"data"` + UseTLS bool `env:"CHARM_SERVER_USE_TLS" envDefault:"false"` + TLSKeyFile string `env:"CHARM_SERVER_TLS_KEY_FILE"` + TLSCertFile string `env:"CHARM_SERVER_TLS_CERT_FILE"` + PublicURL string `env:"CHARM_SERVER_PUBLIC_URL"` + EnableMetrics bool `env:"CHARM_SERVER_ENABLE_METRICS" envDefault:"false"` + UserMaxStorage int64 `env:"CHARM_SERVER_USER_MAX_STORAGE" envDefault:"0"` + ErrorLog *log.Logger + PublicKey []byte + PrivateKey []byte + DB db.DB + FileStore storage.FileStore + Stats stats.Stats + LinkQueue charm.LinkQueue + TLSConfig *tls.Config + JWTKeyPair jwt.JWTKeyPair +} + +// DefaultConfig returns a Config with the values populated with the defaults +// or specified environment variables. +func DefaultConfig() *Config { + cfg := &Config{} + if err := env.Parse(cfg); err != nil { + log.Fatalf("could not read environment: %s", err) + } + + return cfg +} + +// WithDB returns a Config with the provided DB interface implementation. +func (cfg *Config) WithDB(db db.DB) *Config { + cfg.DB = db + return cfg +} + +// WithFileStore returns a Config with the provided FileStore implementation. +func (cfg *Config) WithFileStore(fs storage.FileStore) *Config { + cfg.FileStore = fs + return cfg +} + +// WithStats returns a Config with the provided Stats implementation. +func (cfg *Config) WithStats(s stats.Stats) *Config { + cfg.Stats = s + return cfg +} + +// WithKeys returns a Config with the provided public and private keys for the +// SSH server and JWT signing. +func (cfg *Config) WithKeys(publicKey []byte, privateKey []byte) *Config { + cfg.PublicKey = publicKey + cfg.PrivateKey = privateKey + return cfg +} + +// WithTLSConfig returns a Config with the provided TLS configuration. +func (cfg *Config) WithTLSConfig(c *tls.Config) *Config { + cfg.TLSConfig = c + return cfg +} + +// WithErrorLogger returns a Config with the provided error log for the server. +func (cfg *Config) WithErrorLogger(l *log.Logger) *Config { + cfg.ErrorLog = l + return cfg +} + +// WithLinkQueue returns a Config with the provided LinkQueue implementation. +func (cfg *Config) WithLinkQueue(q charm.LinkQueue) *Config { + cfg.LinkQueue = q + return cfg +} + +// WithJWTKeyPair returns a Config with the provided JWT key pair. +func (cfg *Config) WithJWTKeyPair(k jwt.JWTKeyPair) *Config { + cfg.JWTKeyPair = k + return cfg +} + +// HttpUrl returns the URL for the HTTP server. +func (cfg *Config) HTTPURL() *url.URL { + s := fmt.Sprintf("http://%s:%d", cfg.Host, cfg.HTTPPort) + if cfg.PublicURL != "" { + s = cfg.PublicURL + } + url, err := url.Parse(s) + if err != nil { + log.Fatalf("could not parse URL: %s", err) + } + return url +} diff --git a/server/http.go b/server/http.go index 0a8da680..4851d0a0 100644 --- a/server/http.go +++ b/server/http.go @@ -17,6 +17,7 @@ import ( charmfs "github.com/charmbracelet/charm/fs" charm "github.com/charmbracelet/charm/proto" + "github.com/charmbracelet/charm/server/config" "github.com/charmbracelet/charm/server/db" "github.com/charmbracelet/charm/server/storage" "github.com/meowgorithm/babylogger" @@ -33,7 +34,7 @@ const resultsPerPage = 50 type HTTPServer struct { db db.DB fstore storage.FileStore - cfg *Config + cfg *config.Config server *http.Server health *http.Server httpScheme string @@ -49,7 +50,7 @@ type providerJSON struct { } // NewHTTPServer returns a new *HTTPServer with the specified Config. -func NewHTTPServer(cfg *Config) (*HTTPServer, error) { +func NewHTTPServer(cfg *config.Config) (*HTTPServer, error) { healthMux := http.NewServeMux() // No auth health check endpoint healthMux.Handle("/", versionMiddleware( @@ -60,7 +61,7 @@ func NewHTTPServer(cfg *Config) (*HTTPServer, error) { health := &http.Server{ Addr: fmt.Sprintf("%s:%d", cfg.BindAddr, cfg.HealthPort), Handler: healthMux, - ErrorLog: cfg.errorLog, + ErrorLog: cfg.ErrorLog, } mux := goji.NewMux() s := &HTTPServer{ @@ -71,17 +72,17 @@ func NewHTTPServer(cfg *Config) (*HTTPServer, error) { s.server = &http.Server{ Addr: fmt.Sprintf("%s:%d", s.cfg.BindAddr, s.cfg.HTTPPort), Handler: mux, - ErrorLog: s.cfg.errorLog, + ErrorLog: s.cfg.ErrorLog, } if cfg.UseTLS { s.httpScheme = "https" - s.health.TLSConfig = s.cfg.tlsConfig - s.server.TLSConfig = s.cfg.tlsConfig + s.health.TLSConfig = s.cfg.TLSConfig + s.server.TLSConfig = s.cfg.TLSConfig } jwtMiddleware, err := JWTMiddleware( - cfg.jwtKeyPair.JWK.Public(), - cfg.httpURL().String(), + cfg.JWTKeyPair.JWK().Public(), + cfg.HTTPURL().String(), []string{"charm"}, ) if err != nil { @@ -172,14 +173,14 @@ func (s *HTTPServer) renderCustomError(w http.ResponseWriter, msg string, status } func (s *HTTPServer) handleJWKS(w http.ResponseWriter, r *http.Request) { - jwks := jose.JSONWebKeySet{Keys: []jose.JSONWebKey{s.cfg.jwtKeyPair.JWK.Public()}} + jwks := jose.JSONWebKeySet{Keys: []jose.JSONWebKey{s.cfg.JWTKeyPair.JWK().Public()}} w.Header().Set("Content-Type", "application/json") w.Header().Set("Access-Control-Allow-Origin", "*") _ = json.NewEncoder(w).Encode(jwks) } func (s *HTTPServer) handleOpenIDConfig(w http.ResponseWriter, r *http.Request) { - pj := providerJSON{JWKSURL: fmt.Sprintf("%s/v1/public/jwks", s.cfg.httpURL())} + pj := providerJSON{JWKSURL: fmt.Sprintf("%s/v1/public/jwks", s.cfg.HTTPURL())} w.Header().Set("Content-Type", "application/json") w.Header().Set("Access-Control-Allow-Origin", "*") _ = json.NewEncoder(w).Encode(pj) @@ -311,7 +312,7 @@ func (s *HTTPServer) handlePostFile(w http.ResponseWriter, r *http.Request) { defer f.Close() // nolint:errcheck } if s.cfg.UserMaxStorage > 0 && fh != nil { - stat, err := s.cfg.FileStore.Stat(u.CharmID, "") + stat, err := s.cfg.FileStore.Info(u.CharmID, "") if err != nil { log.Printf("cannot stat user storage: %s", err) s.renderError(w) @@ -335,7 +336,7 @@ func (s *HTTPServer) handlePostFile(w http.ResponseWriter, r *http.Request) { func (s *HTTPServer) handleGetFile(w http.ResponseWriter, r *http.Request) { u := s.charmUserFromRequest(w, r) path := filepath.Clean(pattern.Path(r.Context())) - fi, err := s.cfg.FileStore.Stat(u.CharmID, path) + fi, err := s.cfg.FileStore.Info(u.CharmID, path) if errors.Is(err, fs.ErrNotExist) { s.renderCustomError(w, "file not found", http.StatusNotFound) return diff --git a/server/jwk.go b/server/jwk.go deleted file mode 100644 index 19342417..00000000 --- a/server/jwk.go +++ /dev/null @@ -1,32 +0,0 @@ -package server - -import ( - "crypto/ed25519" - "crypto/sha256" - "fmt" - - "gopkg.in/square/go-jose.v2" -) - -// JSONWebKeyPair holds the ED25519 private key and JSON Web Key used in JWT -// operations. -type JSONWebKeyPair struct { - PrivateKey *ed25519.PrivateKey - JWK jose.JSONWebKey -} - -// NewJSONWebKeyPair creates a new JSONWebKeyPair from a given ED25519 private -// key. -func NewJSONWebKeyPair(pk *ed25519.PrivateKey) JSONWebKeyPair { - sum := sha256.Sum256([]byte(*pk)) - kid := fmt.Sprintf("%x", sum) - jwk := jose.JSONWebKey{ - Key: pk.Public(), - KeyID: kid, - Algorithm: "EdDSA", - } - return JSONWebKeyPair{ - PrivateKey: pk, - JWK: jwk, - } -} diff --git a/server/jwt/jwt.go b/server/jwt/jwt.go new file mode 100644 index 00000000..582e63e7 --- /dev/null +++ b/server/jwt/jwt.go @@ -0,0 +1,48 @@ +package jwt + +import ( + "crypto/ed25519" + "crypto/sha256" + "fmt" + + "gopkg.in/square/go-jose.v2" +) + +// JWTKeyPair is an interface for JWT signing and verification. +type JWTKeyPair interface { + PrivateKey() *ed25519.PrivateKey + JWK() *jose.JSONWebKey +} + +// JSONWebKeyPair holds the ED25519 private key and JSON Web Key used in JWT +// operations. +type JSONWebKeyPair struct { + privateKey *ed25519.PrivateKey + jwk *jose.JSONWebKey +} + +// PrivateKey implements the JWTKeyPair interface. +func (j JSONWebKeyPair) PrivateKey() *ed25519.PrivateKey { + return j.privateKey +} + +// JWK implements the JWTKeyPair interface. +func (j JSONWebKeyPair) JWK() *jose.JSONWebKey { + return j.jwk +} + +// NewJSONWebKeyPair creates a new JSONWebKeyPair from a given ED25519 private +// key. +func NewJSONWebKeyPair(pk *ed25519.PrivateKey) JWTKeyPair { + sum := sha256.Sum256([]byte(*pk)) + kid := fmt.Sprintf("%x", sum) + jwk := &jose.JSONWebKey{ + Key: pk.Public(), + KeyID: kid, + Algorithm: "EdDSA", + } + return JSONWebKeyPair{ + privateKey: pk, + jwk: jwk, + } +} diff --git a/server/middleware.go b/server/middleware.go index 8526f64f..5228ec4c 100644 --- a/server/middleware.go +++ b/server/middleware.go @@ -11,6 +11,7 @@ import ( jwtmiddleware "github.com/auth0/go-jwt-middleware/v2" "github.com/auth0/go-jwt-middleware/v2/validator" charm "github.com/charmbracelet/charm/proto" + "github.com/charmbracelet/charm/server/config" ) type contextKey string @@ -23,7 +24,7 @@ var MaxFSRequestSize int64 = 1024 * 1024 * 1024 // 1GB // versionMiddleware is a middleware that adds a version header to the response. var versionMiddleware = func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("X-Version", Version) + w.Header().Set("X-Version", config.Version) h.ServeHTTP(w, r) }) } diff --git a/server/server.go b/server/server.go index ae5b6586..062ad763 100644 --- a/server/server.go +++ b/server/server.go @@ -4,16 +4,13 @@ package server import ( "context" "crypto/ed25519" - "crypto/tls" "fmt" "log" - "net/url" "path/filepath" - "github.com/caarlos0/env/v6" - charm "github.com/charmbracelet/charm/proto" - "github.com/charmbracelet/charm/server/db" + "github.com/charmbracelet/charm/server/config" "github.com/charmbracelet/charm/server/db/sqlite" + "github.com/charmbracelet/charm/server/jwt" "github.com/charmbracelet/charm/server/stats" "github.com/charmbracelet/charm/server/stats/noop" "github.com/charmbracelet/charm/server/stats/prometheus" @@ -23,123 +20,18 @@ import ( "golang.org/x/sync/errgroup" ) -var ( - // Version is the version of the Charm Cloud server. This is set at build - // time by main.go. - Version = "" -) - -// Config is the configuration for the Charm server. -type Config struct { - BindAddr string `env:"CHARM_SERVER_BIND_ADDRESS" envDefault:""` - Host string `env:"CHARM_SERVER_HOST" envDefault:"localhost"` - SSHPort int `env:"CHARM_SERVER_SSH_PORT" envDefault:"35353"` - HTTPPort int `env:"CHARM_SERVER_HTTP_PORT" envDefault:"35354"` - StatsPort int `env:"CHARM_SERVER_STATS_PORT" envDefault:"35355"` - HealthPort int `env:"CHARM_SERVER_HEALTH_PORT" envDefault:"35356"` - DataDir string `env:"CHARM_SERVER_DATA_DIR" envDefault:"data"` - UseTLS bool `env:"CHARM_SERVER_USE_TLS" envDefault:"false"` - TLSKeyFile string `env:"CHARM_SERVER_TLS_KEY_FILE"` - TLSCertFile string `env:"CHARM_SERVER_TLS_CERT_FILE"` - PublicURL string `env:"CHARM_SERVER_PUBLIC_URL"` - EnableMetrics bool `env:"CHARM_SERVER_ENABLE_METRICS" envDefault:"false"` - UserMaxStorage int64 `env:"CHARM_SERVER_USER_MAX_STORAGE" envDefault:"0"` - errorLog *log.Logger - PublicKey []byte - PrivateKey []byte - DB db.DB - FileStore storage.FileStore - Stats stats.Stats - linkQueue charm.LinkQueue - tlsConfig *tls.Config - jwtKeyPair JSONWebKeyPair - httpScheme string -} - // Server contains the SSH and HTTP servers required to host the Charm Cloud. type Server struct { - Config *Config + Config *config.Config ssh *SSHServer http *HTTPServer } -// DefaultConfig returns a Config with the values populated with the defaults -// or specified environment variables. -func DefaultConfig() *Config { - cfg := &Config{httpScheme: "http"} - if err := env.Parse(cfg); err != nil { - log.Fatalf("could not read environment: %s", err) - } - - return cfg -} - -// WithDB returns a Config with the provided DB interface implementation. -func (cfg *Config) WithDB(db db.DB) *Config { - cfg.DB = db - return cfg -} - -// WithFileStore returns a Config with the provided FileStore implementation. -func (cfg *Config) WithFileStore(fs storage.FileStore) *Config { - cfg.FileStore = fs - return cfg -} - -// WithStats returns a Config with the provided Stats implementation. -func (cfg *Config) WithStats(s stats.Stats) *Config { - cfg.Stats = s - return cfg -} - -// WithKeys returns a Config with the provided public and private keys for the -// SSH server and JWT signing. -func (cfg *Config) WithKeys(publicKey []byte, privateKey []byte) *Config { - cfg.PublicKey = publicKey - cfg.PrivateKey = privateKey - return cfg -} - -// WithTLSConfig returns a Config with the provided TLS configuration. -func (cfg *Config) WithTLSConfig(c *tls.Config) *Config { - cfg.tlsConfig = c - return cfg -} - -// WithErrorLogger returns a Config with the provided error log for the server. -func (cfg *Config) WithErrorLogger(l *log.Logger) *Config { - cfg.errorLog = l - return cfg -} - -// WithLinkQueue returns a Config with the provided LinkQueue implementation. -func (cfg *Config) WithLinkQueue(q charm.LinkQueue) *Config { - cfg.linkQueue = q - return cfg -} - -func (cfg *Config) httpURL() *url.URL { - s := fmt.Sprintf("%s://%s:%d", cfg.httpScheme, cfg.Host, cfg.HTTPPort) - if cfg.PublicURL != "" { - s = cfg.PublicURL - } - url, err := url.Parse(s) - if err != nil { - log.Fatalf("could not parse URL: %s", err) - } - return url -} - // NewServer returns a *Server with the specified Config. -func NewServer(cfg *Config) (*Server, error) { +func NewServer(cfg *config.Config) (*Server, error) { s := &Server{Config: cfg} s.init(cfg) - pk, err := gossh.ParseRawPrivateKey(cfg.PrivateKey) - if err != nil { - return nil, err - } - cfg.jwtKeyPair = NewJSONWebKeyPair(pk.(*ed25519.PrivateKey)) ss, err := NewSSHServer(cfg) if err != nil { return nil, err @@ -203,7 +95,7 @@ func (srv *Server) Close() error { return nil } -func (srv *Server) init(cfg *Config) { +func (srv *Server) init(cfg *config.Config) { if cfg.DB == nil { dp := filepath.Join(cfg.DataDir, "db") err := storage.EnsureDir(dp, 0o700) @@ -223,9 +115,17 @@ func (srv *Server) init(cfg *Config) { if cfg.Stats == nil { srv.Config = cfg.WithStats(getStatsImpl(cfg)) } + if cfg.JWTKeyPair == nil { + pk, err := gossh.ParseRawPrivateKey(cfg.PrivateKey) + if err != nil { + log.Fatalf("could not parse private key: %s", err) + } + jwtKeyPair := jwt.NewJSONWebKeyPair(pk.(*ed25519.PrivateKey)) + srv.Config = cfg.WithJWTKeyPair(jwtKeyPair) + } } -func getStatsImpl(cfg *Config) stats.Stats { +func getStatsImpl(cfg *config.Config) stats.Stats { if cfg.EnableMetrics { return prometheus.NewStats(cfg.DB, cfg.StatsPort) } diff --git a/server/ssh.go b/server/ssh.go index 9ae5fe90..44874ec9 100644 --- a/server/ssh.go +++ b/server/ssh.go @@ -11,6 +11,7 @@ import ( "time" charm "github.com/charmbracelet/charm/proto" + "github.com/charmbracelet/charm/server/config" "github.com/charmbracelet/charm/server/db" "github.com/charmbracelet/wish" rm "github.com/charmbracelet/wish/recover" @@ -30,7 +31,7 @@ type SessionHandler func(s Session) // SSHServer serves the SSH protocol and handles requests to authenticate and // link Charm user accounts. type SSHServer struct { - config *Config + config *config.Config db db.DB server *ssh.Server errorLog *log.Logger @@ -38,11 +39,11 @@ type SSHServer struct { } // NewSSHServer creates a new SSHServer from the provided Config. -func NewSSHServer(cfg *Config) (*SSHServer, error) { +func NewSSHServer(cfg *config.Config) (*SSHServer, error) { s := &SSHServer{ config: cfg, - errorLog: cfg.errorLog, - linkQueue: cfg.linkQueue, + errorLog: cfg.ErrorLog, + linkQueue: cfg.LinkQueue, } if s.errorLog == nil { s.errorLog = log.Default() @@ -134,12 +135,12 @@ func (me *SSHServer) newJWT(charmID string, audience ...string) (string, error) claims := &jwt.RegisteredClaims{ Subject: charmID, ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour)), - Issuer: me.config.httpURL().String(), + Issuer: me.config.HTTPURL().String(), Audience: audience, } token := jwt.NewWithClaims(&jwt.SigningMethodEd25519{}, claims) - token.Header["kid"] = me.config.jwtKeyPair.JWK.KeyID - return token.SignedString(me.config.jwtKeyPair.PrivateKey) + token.Header["kid"] = me.config.JWTKeyPair.JWK().KeyID + return token.SignedString(me.config.JWTKeyPair.PrivateKey()) } // keyText is the base64 encoded public key for the glider.Session. diff --git a/testserver/testserver.go b/testserver/testserver.go index 46236f79..792005cd 100644 --- a/testserver/testserver.go +++ b/testserver/testserver.go @@ -13,6 +13,7 @@ import ( "github.com/charmbracelet/charm/client" "github.com/charmbracelet/charm/server" + "github.com/charmbracelet/charm/server/config" "github.com/charmbracelet/keygen" ) @@ -28,7 +29,7 @@ func SetupTestServer(tb testing.TB) *client.Client { sp := filepath.Join(td, ".ssh") clientData := filepath.Join(td, ".client-data") - cfg := server.DefaultConfig() + cfg := config.DefaultConfig() cfg.DataDir = filepath.Join(td, ".data") cfg.SSHPort = randomPort(tb) cfg.HTTPPort = randomPort(tb) From 8a25ec059ce3e5e43bfae45087eef3bc2a509faa Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Fri, 13 May 2022 12:13:11 -0400 Subject: [PATCH 17/19] feat: recursive file removal error when file does not exist and dir is not empty --- cmd/fs.go | 4 ++++ fs/fs.go | 18 +++++++++++++++--- server/http.go | 15 ++++++++++++--- server/storage/local/storage.go | 9 ++++++--- server/storage/storage.go | 2 +- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/cmd/fs.go b/cmd/fs.go index 55af7e8d..e05c5c31 100644 --- a/cmd/fs.go +++ b/cmd/fs.go @@ -264,6 +264,9 @@ func fsRemove(cmd *cobra.Command, args []string) error { if err != nil { return err } + if isRecursive { + return lsfs.RemoveAll(args[0]) + } return lsfs.Remove(args[0]) } @@ -363,6 +366,7 @@ func printDir(f fs.ReadDirFile) error { func init() { fsCopyCmd.Flags().BoolVarP(&isRecursive, "recursive", "r", false, "copy directories recursively") fsMoveCmd.Flags().BoolVarP(&isRecursive, "recursive", "r", false, "move directories recursively") + fsRemoveCmd.Flags().BoolVarP(&isRecursive, "recursive", "r", false, "remove files recursively") FSCmd.AddCommand(fsCatCmd) FSCmd.AddCommand(fsCopyCmd) diff --git a/fs/fs.go b/fs/fs.go index 1bf33313..cd039e94 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -384,20 +384,32 @@ func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { return resp.Body.Close() } -// Remove deletes a file from the Charm Cloud server. -func (cfs *FS) Remove(name string) error { +func (cfs *FS) remove(name string, all bool) error { ep, err := cfs.EncryptPath(name) if err != nil { return err } path := fmt.Sprintf("/v1/fs/%s", ep) - resp, err := cfs.cc.AuthedRequest("DELETE", path, nil, nil) + headers := http.Header{ + "X-Recursive": {fmt.Sprintf("%t", all)}, + } + resp, err := cfs.cc.AuthedRequest("DELETE", path, headers, nil) if err != nil { return err } return resp.Body.Close() } +// Remove deletes a file from the Charm Cloud server. +func (cfs *FS) Remove(name string) error { + return cfs.remove(name, false) +} + +// RemoveAll deletes a directory and all its contents from the Charm Cloud +func (cfs *FS) RemoveAll(name string) error { + return cfs.remove(name, true) +} + // MkdirAll creates a directory on the configured Charm Cloud server. func (cfs *FS) MkdirAll(path string, perm fs.FileMode) error { ep, err := cfs.EncryptPath(path) diff --git a/server/http.go b/server/http.go index 4851d0a0..02ebca5c 100644 --- a/server/http.go +++ b/server/http.go @@ -393,10 +393,19 @@ func (s *HTTPServer) handleGetFile(w http.ResponseWriter, r *http.Request) { func (s *HTTPServer) handleDeleteFile(w http.ResponseWriter, r *http.Request) { u := s.charmUserFromRequest(w, r) path := filepath.Clean(pattern.Path(r.Context())) - err := s.cfg.FileStore.Delete(u.CharmID, path) + all := r.Header.Get("X-Recursive") == "true" + err := s.cfg.FileStore.Delete(u.CharmID, path, all) if err != nil { - log.Printf("cannot delete file: %s", err) - s.renderError(w) + switch { + case errors.Is(err, fs.ErrNotExist): + s.renderCustomError(w, "file not found", http.StatusNotFound) + // Directory not empty + case errors.Is(err, fs.ErrExist): + s.renderCustomError(w, "directory not empty", http.StatusBadRequest) + default: + log.Printf("cannot delete file: %s", err) + s.renderError(w) + } return } } diff --git a/server/storage/local/storage.go b/server/storage/local/storage.go index 57ee981c..21853047 100644 --- a/server/storage/local/storage.go +++ b/server/storage/local/storage.go @@ -185,7 +185,10 @@ func (lfs *LocalFileStore) Put(charmID string, path string, r io.Reader, mode fs } // Delete deletes the file at the given path for the provided Charm ID. -func (lfs *LocalFileStore) Delete(charmID string, path string) error { - fp := filepath.Join(lfs.Path, charmID, path) - return os.RemoveAll(fp) +func (lfs *LocalFileStore) Delete(charmID string, path string, all bool) error { + fp := filepath.Join(lfs.path, charmID, path) + if all { + return os.RemoveAll(fp) + } + return os.Remove(fp) } diff --git a/server/storage/storage.go b/server/storage/storage.go index d1526531..90580b32 100644 --- a/server/storage/storage.go +++ b/server/storage/storage.go @@ -12,7 +12,7 @@ type FileStore interface { Stat(charmID string, path string) (fs.FileInfo, error) Get(charmID string, path string) (fs.File, error) Put(charmID string, path string, r io.Reader, mode fs.FileMode) error - Delete(charmID string, path string) error + Delete(charmID string, path string, all bool) error } // EnsureDir will create the directory for the provided path on the server From ea3644f6b04219931ff6d3e3c9396be4dbc6e8e9 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Mon, 16 May 2022 10:49:31 -0400 Subject: [PATCH 18/19] clean --- server/config/config.go | 4 ++-- server/http.go | 2 +- server/jwt/jwt.go | 6 +++--- server/middleware.go | 15 +++++++++------ 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 2cfd851d..3bb7edad 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -43,7 +43,7 @@ type Config struct { Stats stats.Stats LinkQueue charm.LinkQueue TLSConfig *tls.Config - JWTKeyPair jwt.JWTKeyPair + JWTKeyPair jwt.KeyPair } // DefaultConfig returns a Config with the values populated with the defaults @@ -102,7 +102,7 @@ func (cfg *Config) WithLinkQueue(q charm.LinkQueue) *Config { } // WithJWTKeyPair returns a Config with the provided JWT key pair. -func (cfg *Config) WithJWTKeyPair(k jwt.JWTKeyPair) *Config { +func (cfg *Config) WithJWTKeyPair(k jwt.KeyPair) *Config { cfg.JWTKeyPair = k return cfg } diff --git a/server/http.go b/server/http.go index 02ebca5c..6b1b33e6 100644 --- a/server/http.go +++ b/server/http.go @@ -89,8 +89,8 @@ func NewHTTPServer(cfg *config.Config) (*HTTPServer, error) { return nil, err } - mux.Use(versionMiddleware) mux.Use(babylogger.Middleware) + mux.Use(versionMiddleware) mux.Use(PublicPrefixesMiddleware([]string{"/v1/public/", "/.well-known/"})) mux.Use(jwtMiddleware) mux.Use(CharmUserMiddleware(s)) diff --git a/server/jwt/jwt.go b/server/jwt/jwt.go index 582e63e7..48f5b837 100644 --- a/server/jwt/jwt.go +++ b/server/jwt/jwt.go @@ -8,8 +8,8 @@ import ( "gopkg.in/square/go-jose.v2" ) -// JWTKeyPair is an interface for JWT signing and verification. -type JWTKeyPair interface { +// KeyPair is an interface for JWT signing and verification. +type KeyPair interface { PrivateKey() *ed25519.PrivateKey JWK() *jose.JSONWebKey } @@ -33,7 +33,7 @@ func (j JSONWebKeyPair) JWK() *jose.JSONWebKey { // NewJSONWebKeyPair creates a new JSONWebKeyPair from a given ED25519 private // key. -func NewJSONWebKeyPair(pk *ed25519.PrivateKey) JWTKeyPair { +func NewJSONWebKeyPair(pk *ed25519.PrivateKey) JSONWebKeyPair { sum := sha256.Sum256([]byte(*pk)) kid := fmt.Sprintf("%x", sum) jwk := &jose.JSONWebKey{ diff --git a/server/middleware.go b/server/middleware.go index 5228ec4c..b6de2feb 100644 --- a/server/middleware.go +++ b/server/middleware.go @@ -21,7 +21,8 @@ var ctxUserKey contextKey = "charmUser" // MaxFSRequestSize is the maximum size of a request body for fs endpoints. var MaxFSRequestSize int64 = 1024 * 1024 * 1024 // 1GB -// versionMiddleware is a middleware that adds a version header to the response. +// versionMiddleware is a middleware that adds the server version as header +// (X-Version) to the response. var versionMiddleware = func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-Version", config.Version) @@ -51,6 +52,12 @@ func RequestLimitMiddleware() func(http.Handler) http.Handler { } } +var publicCtxKey = struct{}{} + +func isPublic(r *http.Request) bool { + return r.Context().Value(publicCtxKey) == true +} + // PublicPrefixesMiddleware allows for the specification of non-authed URL // prefixes. These won't be checked for JWT bearers or Charm user accounts. func PublicPrefixesMiddleware(prefixes []string) func(http.Handler) http.Handler { @@ -62,7 +69,7 @@ func PublicPrefixesMiddleware(prefixes []string) func(http.Handler) http.Handler public = true } } - ctx := context.WithValue(r.Context(), "public", public) + ctx := context.WithValue(r.Context(), publicCtxKey, public) next.ServeHTTP(w, r.WithContext(ctx)) }) } @@ -116,10 +123,6 @@ func CharmUserMiddleware(s *HTTPServer) func(http.Handler) http.Handler { } } -func isPublic(r *http.Request) bool { - return r.Context().Value("public") == true -} - func charmIDFromRequest(r *http.Request) (string, error) { claims := r.Context().Value(jwtmiddleware.ContextKey{}) if claims == "" { From 550793160c7bf42c6d332ad4b90b72b6cb7bf64d Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Mon, 16 May 2022 12:30:27 -0400 Subject: [PATCH 19/19] feat(fs): rename files * ignore file info name * refactor: change storage.Stat to storage.Info * ignore the underlying fs file permissions and use the metadata ones * use a sane dmode & fmode permissions (777 & 666) --- fs/fs.go | 25 ++++++++- server/http.go | 79 ++++++++++++++++------------ server/server.go | 2 +- server/storage/local/storage.go | 53 ++++++++++++++----- server/storage/local/storage_test.go | 4 +- server/storage/storage.go | 3 +- 6 files changed, 113 insertions(+), 53 deletions(-) diff --git a/fs/fs.go b/fs/fs.go index cd039e94..be5fe685 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -150,6 +150,7 @@ func (cfs *FS) Stat(name string) (fs.FileInfo, error) { } if err != io.EOF { info.FileInfo = fi + info.FileInfo.Name = path.Base(fileName) } } } @@ -257,6 +258,7 @@ func (cfs *FS) ReadDir(name string) ([]fs.DirEntry, error) { } if err != io.EOF { fi = ei + fi.Name = n } } } @@ -285,7 +287,7 @@ func (cfs *FS) WriteFile(name string, data []byte, perm fs.FileMode) error { return err } fi := charm.FileInfo{ - Name: path.Base(name), + // Ignore the name in the metadata. IsDir: false, Size: int64(len(data)), Mode: perm, @@ -432,6 +434,27 @@ func (cfs *FS) MkdirAll(path string, perm fs.FileMode) error { return resp.Body.Close() } +// Rename renames a file on the configured Charm Cloud server. +func (cfs *FS) Rename(oldname, newname string) error { + op, err := cfs.EncryptPath(oldname) + if err != nil { + return err + } + np, err := cfs.EncryptPath(newname) + if err != nil { + return err + } + rp := fmt.Sprintf("/v1/fs/%s", np) + headers := http.Header{ + "X-Rename": {op}, + } + resp, err := cfs.cc.AuthedRequest("POST", rp, headers, nil) + if err != nil { + return err + } + return resp.Body.Close() +} + // Client returns the underlying *client.Client. func (cfs *FS) Client() *client.Client { return cfs.cc diff --git a/server/http.go b/server/http.go index 6b1b33e6..335bba93 100644 --- a/server/http.go +++ b/server/http.go @@ -287,49 +287,58 @@ func (s *HTTPServer) handlePostSeq(w http.ResponseWriter, r *http.Request) { func (s *HTTPServer) handlePostFile(w http.ResponseWriter, r *http.Request) { u := s.charmUserFromRequest(w, r) path := filepath.Clean(pattern.Path(r.Context())) - ms := r.Header.Get("X-File-Mode") - if ms == "" { - // Deprecated: remove in next release - ms = r.URL.Query().Get("mode") - log.Printf("deprecated: use X-File-Mode header instead of mode query param. Please update your client.") - } - m, err := strconv.ParseUint(ms, 10, 32) - if err != nil { - log.Printf("file mode not a number: %s", err) - s.renderError(w) - return - } - mode := fs.FileMode(m) - var f multipart.File - var fh *multipart.FileHeader - if !mode.IsDir() { - f, fh, err = r.FormFile("data") - if err != nil { - log.Printf("cannot parse form data: %s", err) - s.renderError(w) + src := r.Header.Get("X-Rename") + if src != "" { + if err := s.cfg.FileStore.Rename(u.CharmID, src, path); err != nil { + log.Printf("cannot rename file: %s", err) + s.renderCustomError(w, err.Error(), http.StatusBadRequest) return } - defer f.Close() // nolint:errcheck - } - if s.cfg.UserMaxStorage > 0 && fh != nil { - stat, err := s.cfg.FileStore.Info(u.CharmID, "") + } else { + ms := r.Header.Get("X-File-Mode") + if ms == "" { + // Deprecated: remove in next release + ms = r.URL.Query().Get("mode") + log.Printf("deprecated: use X-File-Mode header instead of mode query param. Please update your client.") + } + m, err := strconv.ParseUint(ms, 10, 32) if err != nil { - log.Printf("cannot stat user storage: %s", err) + log.Printf("file mode not a number: %s", err) s.renderError(w) return } - if stat.Size()+fh.Size > s.cfg.UserMaxStorage { - s.renderCustomError(w, "user storage limit exceeded", http.StatusForbidden) + mode := fs.FileMode(m) + var f multipart.File + var fh *multipart.FileHeader + if !mode.IsDir() { + f, fh, err = r.FormFile("data") + if err != nil { + log.Printf("cannot parse form data: %s", err) + s.renderError(w) + return + } + defer f.Close() // nolint:errcheck + } + if s.cfg.UserMaxStorage > 0 && fh != nil { + stat, err := s.cfg.FileStore.Info(u.CharmID, "") + if err != nil { + log.Printf("cannot stat user storage: %s", err) + s.renderError(w) + return + } + if stat.Size()+fh.Size > s.cfg.UserMaxStorage { + s.renderCustomError(w, "user storage limit exceeded", http.StatusForbidden) + return + } + } + if err := s.cfg.FileStore.Put(u.CharmID, path, f, mode); err != nil { + log.Printf("cannot post file: %s", err) + s.renderError(w) return } - } - if err := s.cfg.FileStore.Put(u.CharmID, path, f, mode); err != nil { - log.Printf("cannot post file: %s", err) - s.renderError(w) - return - } - if fh != nil { - s.cfg.Stats.FSFileWritten(u.CharmID, fh.Size) + if fh != nil { + s.cfg.Stats.FSFileWritten(u.CharmID, fh.Size) + } } } diff --git a/server/server.go b/server/server.go index 062ad763..0406aa2b 100644 --- a/server/server.go +++ b/server/server.go @@ -106,7 +106,7 @@ func (srv *Server) init(cfg *config.Config) { srv.Config = cfg.WithDB(db) } if cfg.FileStore == nil { - fs, err := lfs.NewLocalFileStore(filepath.Join(cfg.DataDir, "files")) + fs, err := lfs.NewLocalFileStore(srv.Config, filepath.Join(cfg.DataDir, "files")) if err != nil { log.Fatalf("could not init file path: %s", err) } diff --git a/server/storage/local/storage.go b/server/storage/local/storage.go index 21853047..d59af6b1 100644 --- a/server/storage/local/storage.go +++ b/server/storage/local/storage.go @@ -11,30 +11,42 @@ import ( charmfs "github.com/charmbracelet/charm/fs" charm "github.com/charmbracelet/charm/proto" + "github.com/charmbracelet/charm/server/config" "github.com/charmbracelet/charm/server/storage" "github.com/muesli/sasquatch" ) +var ( + // FileMode is the default mode for files. + FileMode fs.FileMode = 0o666 + // DirMode is the default mode for directories. + DirMode fs.FileMode = 0o777 +) + // LocalFileStore is a FileStore implementation that stores files locally in a // folder. type LocalFileStore struct { - Path string + cfg *config.Config + path string } // NewLocalFileStore creates a FileStore locally in the provided path. Files // will be encrypted client-side and stored as regular file system files and // folders. -func NewLocalFileStore(path string) (*LocalFileStore, error) { +func NewLocalFileStore(cfg *config.Config, path string) (*LocalFileStore, error) { err := storage.EnsureDir(path, 0o700) if err != nil { return nil, err } - return &LocalFileStore{path}, nil + return &LocalFileStore{ + path: path, + cfg: cfg, + }, nil } // Stat returns the FileInfo for the given Charm ID and path. -func (lfs *LocalFileStore) Stat(charmID, path string) (fs.FileInfo, error) { - fp := filepath.Join(lfs.Path, charmID, path) +func (lfs *LocalFileStore) Info(charmID, path string) (fs.FileInfo, error) { + fp := filepath.Join(lfs.path, charmID, path) f, err := os.Open(fp) if os.IsNotExist(err) { return nil, fs.ErrNotExist @@ -86,8 +98,8 @@ func (lfs *LocalFileStore) Stat(charmID, path string) (fs.FileInfo, error) { // Get returns an fs.File for the given Charm ID and path. func (lfs *LocalFileStore) Get(charmID string, path string) (fs.File, error) { data := bytes.NewBuffer(nil) - fp := filepath.Join(lfs.Path, charmID, path) - info, err := lfs.Stat(charmID, path) + fp := filepath.Join(lfs.path, charmID, path) + info, err := lfs.Info(charmID, path) if os.IsNotExist(err) { return nil, fs.ErrNotExist } @@ -161,11 +173,11 @@ func (lfs *LocalFileStore) Put(charmID string, path string, r io.Reader, mode fs return fmt.Errorf("invalid path specified: %s", cpath) } - fp := filepath.Join(lfs.Path, charmID, path) + fp := filepath.Join(lfs.path, charmID, path) if mode.IsDir() { - return storage.EnsureDir(fp, mode) + return storage.EnsureDir(fp, DirMode) } - err := storage.EnsureDir(filepath.Dir(fp), 0o755) + err := storage.EnsureDir(filepath.Dir(fp), DirMode) if err != nil { return err } @@ -178,17 +190,30 @@ func (lfs *LocalFileStore) Put(charmID string, path string, r io.Reader, mode fs if err != nil { return err } - if mode != 0 { - return f.Chmod(mode) - } return nil } // Delete deletes the file at the given path for the provided Charm ID. -func (lfs *LocalFileStore) Delete(charmID string, path string, all bool) error { +func (lfs *LocalFileStore) Delete(charmID, path string, all bool) error { fp := filepath.Join(lfs.path, charmID, path) if all { return os.RemoveAll(fp) } return os.Remove(fp) } + +// Rename renames the file at the given path for the provided Charm ID. If +// destination exists, an error is returned. +func (lfs *LocalFileStore) Rename(charmID, src, dst string) error { + srcfp := filepath.Join(lfs.path, charmID, src) + dstfp := filepath.Join(lfs.path, charmID, dst) + _, err := os.Stat(srcfp) + if err != nil { + return err + } + _, err = os.Stat(dstfp) + if !os.IsNotExist(err) { + return fs.ErrExist + } + return os.Rename(srcfp, dstfp) +} diff --git a/server/storage/local/storage_test.go b/server/storage/local/storage_test.go index f84893f3..f39cddc6 100644 --- a/server/storage/local/storage_test.go +++ b/server/storage/local/storage_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "github.com/charmbracelet/charm/server/config" "github.com/google/uuid" ) @@ -15,7 +16,8 @@ func TestPut(t *testing.T) { tdir := t.TempDir() charmID := uuid.New().String() buf := bytes.NewBufferString("") - lfs, err := NewLocalFileStore(tdir) + cfg := config.DefaultConfig() + lfs, err := NewLocalFileStore(cfg, tdir) if err != nil { t.Fatal(err) } diff --git a/server/storage/storage.go b/server/storage/storage.go index 90580b32..49de496b 100644 --- a/server/storage/storage.go +++ b/server/storage/storage.go @@ -9,9 +9,10 @@ import ( // FileStore is the interface storage backends need to implement to act as a // the datastore for the Charm Cloud server. type FileStore interface { - Stat(charmID string, path string) (fs.FileInfo, error) + Info(charmID string, path string) (fs.FileInfo, error) Get(charmID string, path string) (fs.File, error) Put(charmID string, path string, r io.Reader, mode fs.FileMode) error + Rename(charmID string, src string, dst string) error Delete(charmID string, path string, all bool) error }