Merge pull request #21 from nlnwa/update_gowarc

Update gowarc
nlnwa · Sep 30, 2021 · 851377e · 851377e
2 parents d02b4e8 + b893785
commit 851377e
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 5 deletions.
diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.15
 require (
 	github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect
 	github.com/coreos/etcd v3.3.13+incompatible
-	github.com/nlnwa/gowarc v1.0.0-alpha.12
+	github.com/nlnwa/gowarc v1.0.0-alpha.13
 	github.com/nlnwa/veidemann-api/go v0.0.0-20210414094839-b36ce92632fe
 	github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e
 	github.com/opentracing/opentracing-go v1.2.0

diff --git a/go.sum b/go.sum
@@ -278,8 +278,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nlnwa/gowarc v1.0.0-alpha.12 h1:1xKWGVr+jPqe5g9Sqgg8qScLE/7+o3NZ/IS2vZMYYKg=
-github.com/nlnwa/gowarc v1.0.0-alpha.12/go.mod h1:SUvT0iKudUMshYNnv9zGPW5MmxnInlr3ZshFj+UxaTI=
+github.com/nlnwa/gowarc v1.0.0-alpha.13 h1:Y6eZPLaFKvs0QS7YRXjCxq1e+JNDgTzYHWgmyYo6uBM=
+github.com/nlnwa/gowarc v1.0.0-alpha.13/go.mod h1:SUvT0iKudUMshYNnv9zGPW5MmxnInlr3ZshFj+UxaTI=
 github.com/nlnwa/veidemann-api/go v0.0.0-20210414094839-b36ce92632fe h1:yaxQ13HIpCE+I1ZvcVhM1g+sUAopKAAxtt0k1NBmo2Q=
 github.com/nlnwa/veidemann-api/go v0.0.0-20210414094839-b36ce92632fe/go.mod h1:UVGCJSmHATdV3Eohyq03lF3z86q9nRXRQNv3krrEC8I=
 github.com/nlnwa/whatwg-url v0.0.0-20200306110950-d1a95e2e8fc3 h1:iarpnapq+Q98GFlSqalcQq9Qc0f5p7hMO/r+65aitOs=

diff --git a/main.go b/main.go
@@ -24,7 +24,9 @@ func main() {
 	pflag.Int("port", 8080, "port the browser controller api listens to.")
 	pflag.String("host-name", "", "")
 	pflag.String("warc-dir", "", "")
+	pflag.String("warc-version", "1.1", "which WARC version to use for generated records. Allowed values: 1.0, 1.1")
 	pflag.Int("warc-writer-pool-size", 1, "")
+	pflag.Bool("flush-record", false, "if true, flush WARC-file to disk after each record.")
 	pflag.String("work-dir", "", "")
 	pflag.Int("termination-grace-period-seconds", 0, "")
 

diff --git a/server/warcwriter.go b/server/warcwriter.go
@@ -209,6 +209,7 @@ func (ww *warcWriter) initFileWriter() {
 		gowarc.WithWarcInfoFunc(ww.warcInfoGenerator),
 		gowarc.WithMaxConcurrentWriters(ww.settings.WarcWriterPoolSize()),
 		gowarc.WithAddWarcConcurrentToHeader(true),
+		gowarc.WithFlush(ww.settings.FlushRecord()),
 	}
 
 	ww.fileWriter = gowarc.NewWarcFileWriter(opts...)

diff --git a/settings/mock.go b/settings/mock.go
@@ -16,14 +16,17 @@
 
 package settings
 
-import "github.com/nlnwa/gowarc"
+import (
+	"github.com/nlnwa/gowarc"
+)
 
 type Mock struct {
 	hostName                      string
 	warcDir                       string
 	warcWriterPoolSize            int
 	workDir                       string
 	terminationGracePeriodSeconds int
+	flushRecord                   bool
 }
 
 func NewMock(warcDir string, warcWriterPoolSize int) *Mock {
@@ -53,3 +56,7 @@ func (m Mock) TerminationGracePeriodSeconds() int {
 func (m Mock) WarcVersion() *gowarc.WarcVersion {
 	return gowarc.V1_1
 }
+
+func (m Mock) FlushRecord() bool {
+	return m.flushRecord
+}
diff --git a/settings/settings.go b/settings/settings.go
@@ -28,6 +28,7 @@ type Settings interface {
 	WorkDir() string
 	TerminationGracePeriodSeconds() int
 	WarcVersion() *gowarc.WarcVersion
+	FlushRecord() bool
 }
 
 type ViperSettings struct{}
@@ -53,5 +54,17 @@ func (s ViperSettings) TerminationGracePeriodSeconds() int {
 }
 
 func (s ViperSettings) WarcVersion() *gowarc.WarcVersion {
-	return gowarc.V1_1
+	v := viper.GetString("warc-version")
+	switch v {
+	case "1.0":
+		return gowarc.V1_0
+	case "1.1":
+		return gowarc.V1_1
+	default:
+		panic("Unsupported WARC version: " + v)
+	}
+}
+
+func (s ViperSettings) FlushRecord() bool {
+	return viper.GetBool("flush-record")
 }