From 98d720f436b20d28ad520c0a06da42140080971f Mon Sep 17 00:00:00 2001 From: Vee Zhang Date: Wed, 10 Jan 2024 14:44:02 +0800 Subject: [PATCH] feat: support comment in csv (#304) --- examples/basic/basic.int.delete.v3.yaml | 1 + examples/basic/basic.int.insert.v3.yaml | 1 + examples/basic/basic.int.update.v3.yaml | 1 + examples/basic/basic.string.delete.v3.yaml | 1 + examples/basic/basic.string.insert.v3.yaml | 1 + examples/basic/basic.string.update.v3.yaml | 1 + examples/basic/person.csv | 3 +++ .../testdata/basic/basic.int.delete.v3.yaml | 1 + .../testdata/basic/basic.int.insert.v3.yaml | 1 + .../testdata/basic/basic.int.update.v3.yaml | 1 + .../testdata/basic/basic.string.delete.v3.yaml | 1 + .../testdata/basic/basic.string.insert.v3.yaml | 1 + .../testdata/basic/basic.string.update.v3.yaml | 1 + integration-testing/testdata/basic/person.csv | 3 +++ pkg/reader/csv.go | 3 +++ pkg/reader/csv_test.go | 13 +++++++------ pkg/reader/testdata/local_delimiter.csv | 6 ++++++ pkg/source/config.go | 1 + 18 files changed, 35 insertions(+), 6 deletions(-) diff --git a/examples/basic/basic.int.delete.v3.yaml b/examples/basic/basic.int.delete.v3.yaml index 9b60f81a..c20fc290 100644 --- a/examples/basic/basic.int.delete.v3.yaml +++ b/examples/basic/basic.int.delete.v3.yaml @@ -25,6 +25,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person mode: DELETE diff --git a/examples/basic/basic.int.insert.v3.yaml b/examples/basic/basic.int.insert.v3.yaml index dc7b3e52..34d45b5d 100644 --- a/examples/basic/basic.int.insert.v3.yaml +++ b/examples/basic/basic.int.insert.v3.yaml @@ -35,6 +35,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person id: diff --git a/examples/basic/basic.int.update.v3.yaml b/examples/basic/basic.int.update.v3.yaml index 5e8c5e7c..3bbc4eb8 100644 --- a/examples/basic/basic.int.update.v3.yaml +++ b/examples/basic/basic.int.update.v3.yaml @@ -25,6 +25,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person mode: UPDATE diff --git a/examples/basic/basic.string.delete.v3.yaml b/examples/basic/basic.string.delete.v3.yaml index 4520698f..84595711 100644 --- a/examples/basic/basic.string.delete.v3.yaml +++ b/examples/basic/basic.string.delete.v3.yaml @@ -25,6 +25,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person mode: DELETE diff --git a/examples/basic/basic.string.insert.v3.yaml b/examples/basic/basic.string.insert.v3.yaml index ca38c759..aa0393ba 100644 --- a/examples/basic/basic.string.insert.v3.yaml +++ b/examples/basic/basic.string.insert.v3.yaml @@ -35,6 +35,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person id: diff --git a/examples/basic/basic.string.update.v3.yaml b/examples/basic/basic.string.update.v3.yaml index 25112a66..1c797976 100644 --- a/examples/basic/basic.string.update.v3.yaml +++ b/examples/basic/basic.string.update.v3.yaml @@ -25,6 +25,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person mode: UPDATE diff --git a/examples/basic/person.csv b/examples/basic/person.csv index cf5aa9c7..b33d2049 100644 --- a/examples/basic/person.csv +++ b/examples/basic/person.csv @@ -1,4 +1,7 @@ +# I'm comment +# I'm comment 933|Mahinda|Perera|male|1989-12-03|2010-02-14T15:32:10|119.235.7.103|Firefox +# I'm comment 1129|Carmen|Lepland|female|1984-02-18|2010-01-28T06:39:58|195.20.151.175|Internet Explorer 2199023256684|A.|Rao|female|1985-08-02|2010-04-23T22:52:26|49.202.188.25|Firefox 4398046512167|Gustavo|Arbelaez|male|1986-11-02|2010-06-16T20:53:47|190.96.189.165|Chrome diff --git a/integration-testing/testdata/basic/basic.int.delete.v3.yaml b/integration-testing/testdata/basic/basic.int.delete.v3.yaml index 8dd4817c..024fdfc8 100644 --- a/integration-testing/testdata/basic/basic.int.delete.v3.yaml +++ b/integration-testing/testdata/basic/basic.int.delete.v3.yaml @@ -25,6 +25,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person mode: DELETE diff --git a/integration-testing/testdata/basic/basic.int.insert.v3.yaml b/integration-testing/testdata/basic/basic.int.insert.v3.yaml index 811fd4a3..fa4e6785 100644 --- a/integration-testing/testdata/basic/basic.int.insert.v3.yaml +++ b/integration-testing/testdata/basic/basic.int.insert.v3.yaml @@ -35,6 +35,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person id: diff --git a/integration-testing/testdata/basic/basic.int.update.v3.yaml b/integration-testing/testdata/basic/basic.int.update.v3.yaml index be23bc9d..2551c9a0 100644 --- a/integration-testing/testdata/basic/basic.int.update.v3.yaml +++ b/integration-testing/testdata/basic/basic.int.update.v3.yaml @@ -25,6 +25,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person mode: UPDATE diff --git a/integration-testing/testdata/basic/basic.string.delete.v3.yaml b/integration-testing/testdata/basic/basic.string.delete.v3.yaml index d1f777ab..7f541394 100644 --- a/integration-testing/testdata/basic/basic.string.delete.v3.yaml +++ b/integration-testing/testdata/basic/basic.string.delete.v3.yaml @@ -25,6 +25,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person mode: DELETE diff --git a/integration-testing/testdata/basic/basic.string.insert.v3.yaml b/integration-testing/testdata/basic/basic.string.insert.v3.yaml index 7f78ba9e..16ae1758 100644 --- a/integration-testing/testdata/basic/basic.string.insert.v3.yaml +++ b/integration-testing/testdata/basic/basic.string.insert.v3.yaml @@ -35,6 +35,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person id: diff --git a/integration-testing/testdata/basic/basic.string.update.v3.yaml b/integration-testing/testdata/basic/basic.string.update.v3.yaml index cf4a052b..d5864633 100644 --- a/integration-testing/testdata/basic/basic.string.update.v3.yaml +++ b/integration-testing/testdata/basic/basic.string.update.v3.yaml @@ -25,6 +25,7 @@ sources: - path: ./person.csv csv: delimiter: "|" + comment: "#" tags: - name: Person mode: UPDATE diff --git a/integration-testing/testdata/basic/person.csv b/integration-testing/testdata/basic/person.csv index cf5aa9c7..b33d2049 100644 --- a/integration-testing/testdata/basic/person.csv +++ b/integration-testing/testdata/basic/person.csv @@ -1,4 +1,7 @@ +# I'm comment +# I'm comment 933|Mahinda|Perera|male|1989-12-03|2010-02-14T15:32:10|119.235.7.103|Firefox +# I'm comment 1129|Carmen|Lepland|female|1984-02-18|2010-01-28T06:39:58|195.20.151.175|Internet Explorer 2199023256684|A.|Rao|female|1985-08-02|2010-04-23T22:52:26|49.202.188.25|Firefox 4398046512167|Gustavo|Arbelaez|male|1986-11-02|2010-06-16T20:53:47|190.96.189.165|Chrome diff --git a/pkg/reader/csv.go b/pkg/reader/csv.go index f96316d7..60bad8b4 100644 --- a/pkg/reader/csv.go +++ b/pkg/reader/csv.go @@ -40,6 +40,9 @@ func NewCSVReader(s source.Source) RecordReader { if chars := []rune(c.CSV.Delimiter); len(chars) > 0 { cr.Comma = chars[0] } + if chars := []rune(c.CSV.Comment); len(chars) > 0 { + cr.Comment = chars[0] + } cr.LazyQuotes = c.CSV.LazyQuotes h.withHeader = c.CSV.WithHeader diff --git a/pkg/reader/csv_test.go b/pkg/reader/csv_test.go index 2db4223e..17837191 100644 --- a/pkg/reader/csv_test.go +++ b/pkg/reader/csv_test.go @@ -81,6 +81,7 @@ var _ = Describe("csvReader", func() { }, CSV: &source.CSVConfig{ Delimiter: "|", + Comment: "#", }, }) Expect(err).NotTo(HaveOccurred()) @@ -103,32 +104,32 @@ var _ = Describe("csvReader", func() { r := NewCSVReader(s) nBytes, err = r.Size() Expect(err).NotTo(HaveOccurred()) - Expect(nBytes).To(Equal(int64(33))) + Expect(nBytes).To(Equal(int64(117))) n, record, err = r.Read() Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(6)) + Expect(n).To(Equal(20)) Expect(record).To(Equal(spec.Record{"1", "2", "3"})) n, record, err = r.Read() Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(7)) + Expect(n).To(Equal(35)) Expect(record).To(Equal(spec.Record{"4", " 5", "6"})) n, record, err = r.Read() Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(8)) + Expect(n).To(Equal(22)) Expect(record).To(Equal(spec.Record{" 7", "8", " 9"})) n, record, err = r.Read() Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(12)) + Expect(n).To(Equal(26)) Expect(record).To(Equal(spec.Record{"10", " 11 ", " 12"})) n, record, err = r.Read() Expect(err).To(HaveOccurred()) Expect(stderrors.Is(err, io.EOF)).To(BeTrue()) - Expect(n).To(Equal(0)) + Expect(n).To(Equal(14)) Expect(record).To(BeEmpty()) }) }) diff --git a/pkg/reader/testdata/local_delimiter.csv b/pkg/reader/testdata/local_delimiter.csv index bc89fc15..f11fa3aa 100644 --- a/pkg/reader/testdata/local_delimiter.csv +++ b/pkg/reader/testdata/local_delimiter.csv @@ -1,4 +1,10 @@ +# I'm comment 1|2|3 +# I'm comment +# I'm comment 4| 5|6 +# I'm comment 7|8| 9 +# I'm comment 10| 11 | 12 +# I'm comment diff --git a/pkg/source/config.go b/pkg/source/config.go index b38c04cd..c19eaec1 100644 --- a/pkg/source/config.go +++ b/pkg/source/config.go @@ -15,6 +15,7 @@ type ( CSVConfig struct { Delimiter string `yaml:"delimiter,omitempty"` + Comment string `yaml:"comment,omitempty"` WithHeader bool `yaml:"withHeader,omitempty"` LazyQuotes bool `yaml:"lazyQuotes,omitempty"` }