Merge pull request #65 from xushiwei/cached

HttpOpener.Open: 404 => fs.ErrNotExist

http/fs/cached/cached.go

@@ -122,11 +122,11 @@ func (p *dir) Close() error {
 }
 
 func (p *dir) Read(b []byte) (n int, err error) {
-	return 0, os.ErrPermission
+	return 0, fs.ErrPermission
 }
 
 func (p *dir) Seek(offset int64, whence int) (int64, error) {
-	return 0, os.ErrPermission
+	return 0, fs.ErrPermission
 }
 
 func (p *dir) Stat() (fs.FileInfo, error) {

http/fs/file.go

@@ -31,11 +31,11 @@ func (p *dataFile) Close() error {
 }
 
 func (p *dataFile) ReadDir(n int) ([]fs.DirEntry, error) {
-	return nil, os.ErrInvalid
+	return nil, fs.ErrInvalid
 }
 
 func (p *dataFile) Readdir(count int) ([]fs.FileInfo, error) {
-	return nil, os.ErrInvalid
+	return nil, fs.ErrInvalid
 }
 
 func (p *dataFile) IsDir() bool {
@@ -93,7 +93,7 @@ func (p *filesDataFS) Open(name string) (f http.File, err error) {
 			return File(name, strings.NewReader(files[i+1])), nil
 		}
 	}
-	return nil, os.ErrNotExist
+	return nil, fs.ErrNotExist
 }
 
 // FilesWithContent implements a http.FileSystem by a list of file name and content.
@@ -116,7 +116,7 @@ func (p *filesFS) Open(name string) (f http.File, err error) {
 			return
 		}
 	}
-	return nil, os.ErrNotExist
+	return nil, fs.ErrNotExist
 }
 
 // Files implements a http.FileSystem by a list of file name and content file.

http/fs/fs.go

@@ -22,7 +22,7 @@ func (p *unionFS) Open(name string) (f http.File, err error) {
 			return
 		}
 	}
-	return nil, os.ErrNotExist
+	return nil, fs.ErrNotExist
 }
 
 // Union merge a list of http.FileSystem into a union http.FileSystem object.
@@ -120,7 +120,7 @@ func (p rootDir) Open(name string) (f http.File, err error) {
 	if name == "/" {
 		return rootDir{}, nil
 	}
-	return nil, os.ErrNotExist
+	return nil, fs.ErrNotExist
 }
 
 // Root implements a http.FileSystem that only have a root directory.

http/fs/http.go

@@ -3,6 +3,7 @@ package fs
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"path"
 	"strings"
@@ -34,11 +35,17 @@ func (p *HttpOpener) Open(ctx context.Context, url string) (file http.File, err
 		return nil, err
 	}
 	if resp.StatusCode >= 400 {
+		e := &fs.PathError{Op: "http.Get", Path: url}
+		if resp.StatusCode == 404 {
+			e.Err = fs.ErrNotExist
+			return nil, e
+		}
 		url := "url"
 		if req := resp.Request; req != nil {
 			url = req.URL.String()
 		}
-		return nil, fmt.Errorf("http.Get %s error: status %d (%s)", url, resp.StatusCode, resp.Status)
+		e.Err = fmt.Errorf("http.Get %s error: status %d (%s)", url, resp.StatusCode, resp.Status)
+		return nil, e
 	}
 	return HttpFile(resp.Request.URL.Path, resp), nil
 }

http/fs/ignore/ignore.go

@@ -0,0 +1,115 @@
+package ignore
+
+import (
+	"errors"
+	"io/fs"
+	"net/http"
+	"path"
+	"strings"
+)
+
+// -----------------------------------------------------------------------------------------
+
+func Matched(ignore []string, fullName, dir, fname string) bool {
+	for _, ign := range ignore {
+		if strings.HasPrefix(ign, "/") { // start with /
+			if fullName == "" {
+				fullName = path.Join(dir, fname)
+			}
+			if strings.HasSuffix(ign, "/") { // end with /
+				if strings.HasPrefix(fullName, ign) {
+					return true
+				}
+			} else if strings.HasSuffix(ign, "*") {
+				if strings.HasPrefix(fullName, ign[:len(ign)-1]) {
+					return true
+				}
+			} else if fullName == ign {
+				return true
+			}
+		} else {
+			if fname == "" {
+				dir, fname = path.Split(fullName)
+			}
+			if strings.HasSuffix(ign, "*") {
+				if strings.HasPrefix(fname, ign[:len(ign)-1]) {
+					return true
+				}
+			} else if strings.HasPrefix(ign, "*") {
+				if strings.HasSuffix(fname, ign[1:]) {
+					return true
+				}
+			} else if fname == ign {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// -----------------------------------------------------------------------------------------
+
+type ignDir struct {
+	http.File
+	dir    string
+	ignore []string
+}
+
+func (p *ignDir) ReadDir(count int) (fis []fs.DirEntry, err error) {
+	f, ok := p.File.(interface {
+		ReadDir(count int) ([]fs.DirEntry, error)
+	})
+	if !ok {
+		return nil, &fs.PathError{Op: "readdir", Path: p.dir, Err: errors.New("not implemented")}
+	}
+	if fis, err = f.ReadDir(count); err != nil {
+		return
+	}
+	n := 0
+	dir, ignore := p.dir, p.ignore
+	for _, fi := range fis {
+		if Matched(ignore, "", dir, fi.Name()) {
+			continue
+		}
+		fis[n] = fi
+		n++
+	}
+	return fis[:n], nil
+}
+
+func (p *ignDir) Readdir(count int) (fis []fs.FileInfo, err error) {
+	if fis, err = p.File.Readdir(count); err != nil {
+		return
+	}
+	n := 0
+	dir, ignore := p.dir, p.ignore
+	for _, fi := range fis {
+		if Matched(ignore, "", dir, fi.Name()) {
+			continue
+		}
+		fis[n] = fi
+		n++
+	}
+	return fis[:n], nil
+}
+
+type fsIgnore struct {
+	fs     http.FileSystem
+	ignore []string
+}
+
+func (p *fsIgnore) Open(name string) (f http.File, err error) {
+	if f, err = p.fs.Open(name); err != nil {
+		return
+	}
+	if fi, err := f.Stat(); err == nil && fi.IsDir() {
+		f = &ignDir{f, name, p.ignore}
+	}
+	return
+}
+
+func New(fs http.FileSystem, patterns ...string) http.FileSystem {
+	return &fsIgnore{fs, patterns}
+}
+
+// -----------------------------------------------------------------------------------------

http/fs/stream.go

@@ -29,6 +29,25 @@ func Download(destFile string, src http.File) (err error) {
 
 // -----------------------------------------------------------------------------------------
 
+// Unseekable convert a http.File into a io.ReadCloser object without io.Seeker.
+// Note you should stop using the origin http.File object to read data.
+// This method is used to reduce unnecessary memory usage.
+func Unseekable(file http.File) io.ReadCloser {
+	switch f := file.(type) {
+	case *stream:
+		if f.br == nil {
+			return f.file
+		}
+	case *httpFile:
+		if f.br == nil {
+			return f.file
+		}
+	}
+	return file
+}
+
+// -----------------------------------------------------------------------------------------
+
 type stream struct {
 	file io.ReadCloser
 	b    bytes.Buffer
@@ -42,19 +61,19 @@ func (p *stream) TryReader() *bytes.Reader {
 }
 
 func (p *stream) ReadDir(n int) ([]fs.DirEntry, error) {
-	return nil, os.ErrInvalid
+	return nil, fs.ErrInvalid
 }
 
 func (p *stream) Readdir(count int) ([]fs.FileInfo, error) {
-	return nil, os.ErrInvalid
+	return nil, fs.ErrInvalid
 }
 
 func (p *stream) IsDir() bool {
 	return false
 }
 
 func (p *stream) Mode() fs.FileMode {
-	return 0
+	return fs.ModeIrregular
 }
 
 func (p *stream) Name() string {
@@ -135,19 +154,19 @@ func (p *httpFile) TryReader() *bytes.Reader {
 }
 
 func (p *httpFile) ReadDir(n int) ([]fs.DirEntry, error) {
-	return nil, os.ErrInvalid
+	return nil, fs.ErrInvalid
 }
 
 func (p *httpFile) Readdir(count int) ([]fs.FileInfo, error) {
-	return nil, os.ErrInvalid
+	return nil, fs.ErrInvalid
 }
 
 func (p *httpFile) IsDir() bool {
 	return false
 }
 
 func (p *httpFile) Mode() fs.FileMode {
-	return 0
+	return fs.ModeIrregular
 }
 
 func (p *httpFile) Name() string {