Skip to content

Commit

Permalink
Close files earlier when generating fields files (elastic#16681)
Browse files Browse the repository at this point in the history
Fields generator keeps all open files while generating the global
fields.yml file. This is not needed and may lead to "too many open
files" errors in some environments.
  • Loading branch information
jsoriano authored Feb 28, 2020
1 parent 25ac6ba commit 9524e4a
Showing 1 changed file with 20 additions and 12 deletions.
32 changes: 20 additions & 12 deletions libbeat/generator/fields/fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,24 +123,32 @@ func writeGeneratedFieldsYml(fieldFiles []*YmlFile, output io.Writer) error {
func GenerateFieldsYml(fieldFiles []*YmlFile) ([]byte, error) {
buf := bytes.NewBufferString("")
for _, p := range fieldFiles {
file, err := os.Open(p.Path)
err := generateFieldsYmlForFile(buf, p)
if err != nil {
return nil, err
}
defer file.Close()
}
return buf.Bytes(), nil
}

fs := bufio.NewScanner(file)
for fs.Scan() {
err = writeIndentedLine(buf, fs.Text()+"\n", p.Indent)
if err != nil {
return nil, err
}
}
if err := fs.Err(); err != nil {
return nil, err
func generateFieldsYmlForFile(buf *bytes.Buffer, p *YmlFile) error {
file, err := os.Open(p.Path)
if err != nil {
return err
}
defer file.Close()

fs := bufio.NewScanner(file)
for fs.Scan() {
err = writeIndentedLine(buf, fs.Text()+"\n", p.Indent)
if err != nil {
return err
}
}
return buf.Bytes(), nil
if err := fs.Err(); err != nil {
return err
}
return nil
}

func writeIndentedLine(buf *bytes.Buffer, line string, indent int) error {
Expand Down

0 comments on commit 9524e4a

Please sign in to comment.