prevent ulimit overflow when writing too (#348)

evanw · Aug 26, 2020 · 9882ae3 · 9882ae3
1 parent f02fb76
commit 9882ae3
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 10 deletions.
diff --git a/cmd/esbuild/service.go b/cmd/esbuild/service.go
@@ -255,18 +255,22 @@ func (service *serviceType) handleTransformRequest(id uint32, request map[string
 
 	if inputFS && len(result.JS) > 0 {
 		file := input + ".js"
+		fs.BeforeFileOpen()
 		if err := ioutil.WriteFile(file, result.JS, 0644); err == nil {
 			result.JS = []byte(file)
 			jsFS = true
 		}
+		fs.AfterFileClose()
 	}
 
 	if inputFS && len(result.JSSourceMap) > 0 {
 		file := input + ".map"
+		fs.BeforeFileOpen()
 		if err := ioutil.WriteFile(file, result.JSSourceMap, 0644); err == nil {
 			result.JSSourceMap = []byte(file)
 			jsSourceMapFS = true
 		}
+		fs.AfterFileClose()
 	}
 
 	return encodePacket(packet{

diff --git a/internal/fs/fs.go b/internal/fs/fs.go
@@ -48,8 +48,8 @@ func (e *Entry) stat() {
 	entryPath := filepath.Join(e.dir, e.base)
 
 	// Use "lstat" since we want information about symbolic links
-	beforeFileOpen()
-	defer afterFileClose()
+	BeforeFileOpen()
+	defer AfterFileClose()
 	stat, err := os.Lstat(entryPath)
 	if err != nil {
 		return
@@ -232,12 +232,12 @@ type realFS struct {
 // Limit the number of files open simultaneously to avoid ulimit issues
 var fileOpenLimit = make(chan bool, 32)
 
-func beforeFileOpen() {
+func BeforeFileOpen() {
 	// This will block if the number of open files is already at the limit
 	fileOpenLimit <- false
 }
 
-func afterFileClose() {
+func AfterFileClose() {
 	<-fileOpenLimit
 }
 
@@ -248,8 +248,8 @@ func realpath(path string) string {
 	}
 	dir = realpath(dir)
 	path = filepath.Join(dir, filepath.Base(path))
-	beforeFileOpen()
-	defer afterFileClose()
+	BeforeFileOpen()
+	defer AfterFileClose()
 	if link, err := os.Readlink(path); err == nil {
 		if filepath.IsAbs(link) {
 			return link
@@ -316,8 +316,8 @@ func (fs *realFS) ReadDirectory(dir string) map[string]*Entry {
 }
 
 func (fs *realFS) ReadFile(path string) (string, bool) {
-	beforeFileOpen()
-	defer afterFileClose()
+	BeforeFileOpen()
+	defer AfterFileClose()
 	buffer, err := ioutil.ReadFile(path)
 	return string(buffer), err == nil
 }
@@ -355,8 +355,8 @@ func (*realFS) Rel(base string, target string) (string, bool) {
 }
 
 func readdir(dirname string) ([]string, error) {
-	beforeFileOpen()
-	defer afterFileClose()
+	BeforeFileOpen()
+	defer AfterFileClose()
 	f, err := os.Open(dirname)
 	if err != nil {
 		return nil, err

diff --git a/pkg/api/api_impl.go b/pkg/api/api_impl.go
@@ -546,6 +546,8 @@ func buildImpl(buildOpts BuildOptions) BuildResult {
 						waitGroup.Add(len(outputFiles))
 						for _, outputFile := range outputFiles {
 							go func(outputFile OutputFile) {
+								fs.BeforeFileOpen()
+								defer fs.AfterFileClose()
 								if err := os.MkdirAll(filepath.Dir(outputFile.Path), 0755); err != nil {
 									log.AddError(nil, ast.Loc{}, fmt.Sprintf(
 										"Failed to create output directory: %s", err.Error()))