diff --git a/go.mod b/go.mod index 92d1e1c..7b618c6 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.8.0 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -37,5 +38,4 @@ require ( gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/main.go b/main.go index ee2469e..ab8a16d 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "io" "math/rand" "os" "path/filepath" @@ -14,6 +15,7 @@ import ( "github.com/honeycombio/libhoney-go" flag "github.com/jessevdk/go-flags" "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" "github.com/honeycombio/honeytail/httime" "github.com/honeycombio/honeytail/parsers/arangodb" @@ -51,58 +53,60 @@ var validParsers = []string{ // GlobalOptions has all the top level CLI flags that honeytail supports type GlobalOptions struct { APIHost string `long:"api_host" description:"Host for the Honeycomb API" default:"https://api.honeycomb.io/"` - TailSample bool `hidden:"true" description:"When true, sample while tailing. When false, sample post-parser events"` - - ConfigFile string `short:"c" long:"config" description:"Config file for honeytail in INI format." no-ini:"true"` - - SampleRate uint `short:"r" long:"samplerate" description:"Only send 1 / N log lines" default:"1"` - NumSenders uint `short:"P" long:"poolsize" description:"Number of concurrent connections to open to Honeycomb" default:"80"` - BatchFrequencyMs uint `long:"send_frequency_ms" description:"How frequently to flush batches" default:"100"` - BatchSize uint `long:"send_batch_size" description:"Maximum number of messages to put in a batch" default:"50"` - Debug bool `long:"debug" description:"Print debugging output"` - DebugOut bool `long:"debug_stdout" description:"Instead of sending events to Honeycomb, print them to STDOUT for debugging"` - StatusInterval uint `long:"status_interval" description:"How frequently, in seconds, to print out summary info" default:"60"` - Backfill bool `long:"backfill" description:"Configure honeytail to ingest old data in order to backfill Honeycomb. Sets the correct values for --backoff, --tail.read_from, and --tail.stop"` - RebaseTime bool `long:"rebase_time" description:"When backfilling data, rebase timestamps relative to the current time."` - - Localtime bool `long:"localtime" description:"When parsing a timestamp that has no time zone, assume it is in the same timezone as localhost instead of UTC (the default)"` - Timezone string `long:"timezone" description:"When parsing a timestamp use this time zone instead of UTC (the default). Must be specified in TZ format as seen here: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones"` - ScrubFields []string `long:"scrub_field" description:"For the field listed, apply a one-way hash to the field content. May be specified multiple times"` - DropFields []string `long:"drop_field" description:"Do not send the field to Honeycomb. May be specified multiple times"` - AddFields []string `long:"add_field" description:"Add the field to every event. Field should be key=val. May be specified multiple times"` - DAMapFile string `long:"da_map_file" description:"Data Augmentation Map file. Path to a file that contains JSON mapping of columns to augment, the values of the column, and new objects to be inserted into the event, eg to add hostname based on IP address or username based on user ID"` - RequestShape []string `long:"request_shape" description:"Identify a field that contains an HTTP request of the form 'METHOD /path HTTP/1.x' or just the request path. Break apart that field into subfields that contain components. May be specified multiple times. Defaults to 'request' when using the nginx parser"` - ShapePrefix string `long:"shape_prefix" description:"Prefix to use on fields generated from request_shape to prevent field collision"` - RequestPattern []string `long:"request_pattern" description:"A pattern for the request path on which to base the derived request_shape. May be specified multiple times. Patterns are considered in order; first match wins."` - RequestParseQuery string `long:"request_parse_query" description:"How to parse the request query parameters. 'whitelist' means only extract listed query keys. 'all' means to extract all query parameters as individual columns" default:"whitelist"` - RequestQueryKeys []string `long:"request_query_keys" description:"Request query parameter key names to extract, when request_parse_query is 'whitelist'. May be specified multiple times."` - BackOff bool `long:"backoff" description:"When rate limited by the API, back off and retry sending failed events. Otherwise failed events are dropped. When --backfill is set, it will override this option=true"` - PrefixRegex string `long:"log_prefix" description:"pass a regex to this flag to strip the matching prefix from the line before handing to the parser. Useful when log aggregation prepends a line header. Use named groups to extract fields into the event."` - DeterministicSample string `long:"deterministic_sampling" description:"Specify a field to deterministically sample on, i.e., every concurrent Honeytail instance will sample 1/N based on content."` - DynSample []string `long:"dynsampling" description:"enable dynamic sampling using the field listed in this option. May be specified multiple times; fields will be concatenated to form the dynsample key. WARNING increases CPU utilization dramatically over normal sampling"` - DynWindowSec int `long:"dynsample_window" description:"measurement window size for the dynsampler, in seconds" default:"30"` - PreSampledField string `long:"presampled" description:"If this log has already been sampled, specify the field containing the sample rate here and it will be passed along unchanged"` - GoalSampleRate int `hidden:"true" description:"used to hold the desired sample rate and set tailing sample rate to 1"` - MinSampleRate int `long:"dynsample_minimum" description:"if the rate of traffic falls below this, dynsampler won't sample" default:"1"` - JSONFields []string `long:"json_field" description:"JSON fields encoded as string to unescape and properly parse before sending"` - FilterFiles []string `short:"F" long:"filter-file" description:"Log file(s) to exclude from --file glob. Can specify multiple times, including multiple globs."` - RenameFields []string `long:"rename_field" description:"Format: 'before=after'. Rename field called 'before' from parsed lines to field name 'after' in Honeycomb events."` - - Reqs RequiredOptions `group:"Required Options"` - Modes OtherModes `group:"Other Modes"` - - Tail tail.TailOptions `group:"Tail Options" namespace:"tail"` - - ArangoDB arangodb.Options `group:"ArangoDB Parser Options" namespace:"arangodb"` - CSV csv.Options `group:"CSV Parser Options" namespace:"csv"` - JSON htjson.Options `group:"JSON Parser Options" namespace:"json"` - KeyVal keyval.Options `group:"KeyVal Parser Options" namespace:"keyval"` - Mongo mongodb.Options `group:"MongoDB Parser Options" namespace:"mongo"` - MySQL mysql.Options `group:"MySQL Parser Options" namespace:"mysql"` - Nginx nginx.Options `group:"Nginx Parser Options" namespace:"nginx"` - PostgreSQL postgresql.Options `group:"PostgreSQL Parser Options" namespace:"postgresql"` - Regex regex.Options `group:"Regex Parser Options" namespace:"regex"` - Syslog syslog.Options `group:"Syslog Parser Options" namespace:"syslog"` + TailSample bool `hidden:"true" description:"When true, sample while tailing. When false, sample post-parser events" yaml:"-"` + + ConfigFile string `short:"c" long:"config" description:"Config file for honeytail in INI format." no-ini:"true" yaml:"-"` + ConfigYaml string `long:"config_yaml" description:"Config file for honeytail in YAML format." yaml:"-"` + WriteYaml string `long:"write_yaml" description:"When specified (a filename), parse the existing config, then write a new YAML config to the specified YAML file and quit." yaml:"-"` + + SampleRate uint `short:"r" long:"samplerate" description:"Only send 1 / N log lines" default:"1" yaml:"samplerate"` + NumSenders uint `short:"P" long:"poolsize" description:"Number of concurrent connections to open to Honeycomb" default:"80" yaml:"poolsize"` + BatchFrequencyMs uint `long:"send_frequency_ms" description:"How frequently to flush batches" default:"100" yaml:"send_frequency_ms"` + BatchSize uint `long:"send_batch_size" description:"Maximum number of messages to put in a batch" default:"50" yaml:"send_batch_size"` + Debug bool `long:"debug" description:"Print debugging output" yaml:"debug,omitempty"` + DebugOut bool `long:"debug_stdout" description:"Instead of sending events to Honeycomb, print them to STDOUT for debugging" yaml:"debug_stdout,omitempty"` + StatusInterval uint `long:"status_interval" description:"How frequently, in seconds, to print out summary info" default:"60" yaml:"status_interval"` + Backfill bool `long:"backfill" description:"Configure honeytail to ingest old data in order to backfill Honeycomb. Sets the correct values for --backoff, --tail.read_from, and --tail.stop" yaml:"backfill,omitempty"` + RebaseTime bool `long:"rebase_time" description:"When backfilling data, rebase timestamps relative to the current time." yaml:"rebase_time,omitempty"` + + Localtime bool `long:"localtime" description:"When parsing a timestamp that has no time zone, assume it is in the same timezone as localhost instead of UTC (the default)" yaml:"localtime,omitempty"` + Timezone string `long:"timezone" description:"When parsing a timestamp use this time zone instead of UTC (the default). Must be specified in TZ format as seen here: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones" yaml:"timezone,omitempty"` + ScrubFields []string `long:"scrub_field" description:"For the field listed, apply a one-way hash to the field content. May have multiple values." yaml:"scrub_field,omitempty"` + DropFields []string `long:"drop_field" description:"Do not send the field to Honeycomb. May have multiple values." yaml:"drop_field,omitempty"` + AddFields []string `long:"add_field" description:"Add the field to every event. Field should be key=val. May have multiple values." yaml:"add_field,omitempty"` + DAMapFile string `long:"da_map_file" description:"Data Augmentation Map file. Path to a file that contains JSON mapping of columns to augment, the values of the column, and new objects to be inserted into the event, eg to add hostname based on IP address or username based on user ID." yaml:"da_map_file,omitempty"` + RequestShape []string `long:"request_shape" description:"Identify a field that contains an HTTP request of the form 'METHOD /path HTTP/1.x' or just the request path. Break apart that field into subfields that contain components. May have multiple values. Defaults to 'request' when using the nginx parser." yaml:"request_shape,omitempty"` + ShapePrefix string `long:"shape_prefix" description:"Prefix to use on fields generated from request_shape to prevent field collision" yaml:"shape_prefix,omitempty"` + RequestPattern []string `long:"request_pattern" description:"A pattern for the request path on which to base the derived request_shape. May have multiple values. Patterns are considered in order; first match wins." yaml:"request_pattern,omitempty"` + RequestParseQuery string `long:"request_parse_query" description:"How to parse the request query parameters. 'whitelist' means only extract listed query keys. 'all' means to extract all query parameters as individual columns" default:"whitelist" yaml:"request_parse_query"` + RequestQueryKeys []string `long:"request_query_keys" description:"Request query parameter key names to extract, when request_parse_query is 'whitelist'. May have multiple values." yaml:"request_query_keys,omitempty"` + BackOff bool `long:"backoff" description:"When rate limited by the API, back off and retry sending failed events. Otherwise failed events are dropped. When --backfill is set, it will force this to true." yaml:"backoff,omitempty"` + PrefixRegex string `long:"log_prefix" description:"pass a regex to this flag to strip the matching prefix from the line before handing to the parser. Useful when log aggregation prepends a line header. Use named groups to extract fields into the event." yaml:"log_prefix,omitempty"` + DeterministicSample string `long:"deterministic_sampling" description:"Specify a field to deterministically sample on, i.e., every concurrent Honeytail instance will sample 1/N based on content." yaml:"deterministic_sampling,omitempty"` + DynSample []string `long:"dynsampling" description:"Enable dynamic sampling using the field listed in this option. May have multiple values; fields will be concatenated to form the dynsample key. WARNING: increases CPU utilization dramatically over normal sampling." yaml:"dynsampling,omitempty"` + DynWindowSec int `long:"dynsample_window" description:"Set measurement window size for the dynsampler, in seconds." default:"30" yaml:"dynsample_window"` + PreSampledField string `long:"presampled" description:"If this log has already been sampled, specify the field containing the sample rate here and it will be passed along unchanged." yaml:"presampled,omitempty"` + GoalSampleRate int `hidden:"true" description:"Used to hold the desired sample rate and set tailing sample rate to 1." yaml:"-"` + MinSampleRate int `long:"dynsample_minimum" description:"If the rate of traffic falls below this, dynsampler won't sample." default:"1" yaml:"dynsample_minimum"` + JSONFields []string `long:"json_field" description:"JSON fields encoded as string to unescape and properly parse before sending. May have multiple values." yaml:"json_field,omitempty"` + FilterFiles []string `short:"F" long:"filter-file" description:"Log file(s) to exclude from --file glob. May have multiple values, including multiple globs." yaml:"filter-file,omitempty"` + RenameFields []string `long:"rename_field" description:"Format: 'before=after'. Rename field called 'before' from parsed lines to field name 'after' in Honeycomb events. May have multiple values." yaml:"rename_field,omitempty"` + + Reqs RequiredOptions `group:"Required Options" yaml:"required_options,omitempty"` + Modes OtherModes `group:"Other Modes" yaml:"-"` + + Tail tail.TailOptions `group:"Tail Options" namespace:"tail" yaml:",omitempty"` + + ArangoDB arangodb.Options `group:"ArangoDB Parser Options" namespace:"arangodb" yaml:",omitempty"` + CSV csv.Options `group:"CSV Parser Options" namespace:"csv" yaml:",omitempty"` + JSON htjson.Options `group:"JSON Parser Options" namespace:"json" yaml:",omitempty"` + KeyVal keyval.Options `group:"KeyVal Parser Options" namespace:"keyval" yaml:",omitempty"` + Mongo mongodb.Options `group:"MongoDB Parser Options" namespace:"mongo" yaml:",omitempty"` + MySQL mysql.Options `group:"MySQL Parser Options" namespace:"mysql" yaml:",omitempty"` + Nginx nginx.Options `group:"Nginx Parser Options" namespace:"nginx" yaml:",omitempty"` + PostgreSQL postgresql.Options `group:"PostgreSQL Parser Options" namespace:"postgresql" yaml:",omitempty"` + Regex regex.Options `group:"Regex Parser Options" namespace:"regex" yaml:",omitempty"` + Syslog syslog.Options `group:"Syslog Parser Options" namespace:"syslog" yaml:",omitempty"` } type RequiredOptions struct { @@ -113,19 +117,22 @@ type RequiredOptions struct { } type OtherModes struct { - Help bool `short:"h" long:"help" description:"Show this help message"` - ListParsers bool `short:"l" long:"list" description:"List available parsers"` - Version bool `short:"V" long:"version" description:"Show version"` - WriteDefaultConfig bool `long:"write_default_config" description:"Write a default config file to STDOUT" no-ini:"true"` - WriteCurrentConfig bool `long:"write_current_config" description:"Write out the current config to STDOUT" no-ini:"true"` - - WriteManPage bool `hidden:"true" long:"write-man-page" description:"Write out a man page"` + Help bool `short:"h" long:"help" description:"Show this help message."` + ListParsers bool `short:"l" long:"list" description:"List available parsers."` + Version bool `short:"V" long:"version" description:"Show version."` + WriteDefaultConfig bool `long:"write_default_config" description:"Write a default config file to STDOUT." no-ini:"true"` + WriteCurrentConfig bool `long:"write_current_config" description:"Write out the current config to STDOUT." no-ini:"true"` + WriteCurrentYaml bool `long:"write_current_yaml" description:"Write out the current config to STDOUT as YAML." no-ini:"true"` + + WriteManPage bool `hidden:"true" long:"write-man-page" description:"Write out a man page."` } func main() { var options GlobalOptions flagParser := flag.NewParser(&options, flag.PrintErrors) - flagParser.Usage = "-p -k -f -d [optional arguments]\n\nSee https://honeycomb.io/docs/connect/agent/ for more detailed usage instructions." + flagParser.Usage = `-p -k -f -d [optional arguments] + +See https://honeycomb.io/docs/connect/agent/ for more detailed usage instructions.` if extraArgs, err := flagParser.Parse(); err != nil || len(extraArgs) != 0 { fmt.Println("Error: failed to parse the command line.") @@ -149,6 +156,20 @@ func main() { } } + if options.ConfigYaml != "" { + f, err := os.Open(options.ConfigYaml) + if err != nil { + fmt.Println("error opening " + options.ConfigYaml) + os.Exit(1) + } + b, err := io.ReadAll(f) + if err != nil { + fmt.Println("error reading " + options.ConfigYaml) + os.Exit(1) + } + yaml.Unmarshal(b, &options) + } + rand.Seed(time.Now().UnixNano()) if options.Debug { @@ -179,7 +200,7 @@ func main() { } setVersionUserAgent(options.Backfill, options.Reqs.ParserName) - handleOtherModes(flagParser, options.Modes) + handleOtherModes(flagParser, options) addParserDefaultOptions(&options) sanityCheckOptions(&options) @@ -243,7 +264,8 @@ func setVersionUserAgent(backfill bool, parserName string) { // handleOtherModes takes care of all flags that say we should just do something // and exit rather than actually parsing logs -func handleOtherModes(fp *flag.Parser, modes OtherModes) { +func handleOtherModes(fp *flag.Parser, options GlobalOptions) { + modes := options.Modes if modes.Version { fmt.Println("Honeytail version", version) os.Exit(0) @@ -267,6 +289,15 @@ func handleOtherModes(fp *flag.Parser, modes OtherModes) { ip.Write(os.Stdout, flag.IniIncludeComments) os.Exit(0) } + if modes.WriteCurrentYaml { + y, err := yaml.Marshal(options) + if err != nil { + fmt.Println("unable to marshal options to YAML!") + os.Exit(1) + } + os.Stdout.Write(y) + os.Exit(0) + } if modes.ListParsers { fmt.Println("Available parsers:", strings.Join(validParsers, ", ")) diff --git a/parsers/csv/csv.go b/parsers/csv/csv.go index 183961a..1825462 100644 --- a/parsers/csv/csv.go +++ b/parsers/csv/csv.go @@ -17,10 +17,11 @@ import ( // Options defines the options relevant to the CSV parser type Options struct { Fields string `long:"fields" description:"Comma separated list of CSV fields, in order."` - TimeFieldName string `long:"timefield" description:"Name of the field that contains a timestamp"` - TimeFieldFormat string `long:"time_format" description:"Timestamp format to use (strftime and Golang time.Parse supported)"` - NumParsers int `hidden:"true" description:"number of csv parsers to spin up"` - TrimLeadingSpace bool `bool:"trim_leading_space" description:"trim leading whitespace in CSV fields and values" default:"false"` + TimeFieldName string `long:"timefield" description:"Name of the field that contains a timestamp" yaml:"timefield,omitempty"` + TimeFieldFormat string `long:"time_format" description:"Timestamp format to use (strftime and Golang time.Parse supported)" yaml:"time_format,omitempty"` + TrimLeadingSpace bool `long:"trim_leading_space" description:"trim leading whitespace in CSV fields and values" yaml:"trim_leading_space,omitempty"` + + NumParsers int `hidden:"true" description:"number of csv parsers to spin up" yaml:"-"` } // Parser implements the Parser interface diff --git a/parsers/htjson/htjson.go b/parsers/htjson/htjson.go index da9ff34..ba1cd01 100644 --- a/parsers/htjson/htjson.go +++ b/parsers/htjson/htjson.go @@ -15,10 +15,10 @@ import ( ) type Options struct { - TimeFieldName string `long:"timefield" description:"Name of the field that contains a timestamp"` - TimeFieldFormat string `long:"format" description:"Format of the timestamp found in timefield (supports strftime and Golang time formats)"` + TimeFieldName string `long:"timefield" description:"Name of the field that contains a timestamp" yaml:"timefield,omitempty"` + TimeFieldFormat string `long:"format" description:"Format of the timestamp found in timefield (supports strftime and Golang time formats)" yaml:"format,omitempty"` - NumParsers int `hidden:"true" description:"number of htjson parsers to spin up"` + NumParsers int `hidden:"true" description:"number of htjson parsers to spin up" yaml:"-"` } type Parser struct { diff --git a/parsers/keyval/keyval.go b/parsers/keyval/keyval.go index b9a1397..a217492 100644 --- a/parsers/keyval/keyval.go +++ b/parsers/keyval/keyval.go @@ -7,8 +7,8 @@ import ( "strings" "sync" - "github.com/sirupsen/logrus" "github.com/kr/logfmt" + "github.com/sirupsen/logrus" "github.com/honeycombio/honeytail/event" "github.com/honeycombio/honeytail/httime" @@ -16,12 +16,12 @@ import ( ) type Options struct { - TimeFieldName string `long:"timefield" description:"Name of the field that contains a timestamp"` - TimeFieldFormat string `long:"format" description:"Format of the timestamp found in timefield (supports strftime and Golang time formats)"` - FilterRegex string `long:"filter_regex" description:"a regular expression that will filter the input stream and only parse lines that match"` - InvertFilter bool `long:"invert_filter" description:"change the filter_regex to only process lines that do *not* match"` + TimeFieldName string `long:"timefield" description:"Name of the field that contains a timestamp" yaml:"timefield,omitempty"` + TimeFieldFormat string `long:"format" description:"Format of the timestamp found in timefield (supports strftime and Golang time formats)" yaml:"format,omitempty"` + FilterRegex string `long:"filter_regex" description:"a regular expression that will filter the input stream and only parse lines that match" yaml:"filter_regex,omitempty"` + InvertFilter bool `long:"invert_filter" description:"change the filter_regex to only process lines that do *not* match" yaml:"invert_filter,omitempty"` - NumParsers int `hidden:"true" description:"number of keyval parsers to spin up"` + NumParsers int `hidden:"true" description:"number of keyval parsers to spin up" yaml:"-"` } type Parser struct { diff --git a/parsers/mongodb/mongodb.go b/parsers/mongodb/mongodb.go index 7650017..e7660a6 100644 --- a/parsers/mongodb/mongodb.go +++ b/parsers/mongodb/mongodb.go @@ -7,9 +7,9 @@ import ( "sync" "time" - "github.com/sirupsen/logrus" "github.com/honeycombio/mongodbtools/logparser" queryshape "github.com/honeycombio/mongodbtools/queryshape" + "github.com/sirupsen/logrus" "github.com/honeycombio/honeytail/event" "github.com/honeycombio/honeytail/httime" @@ -48,9 +48,9 @@ var timestampFormats = []string{ } type Options struct { - LogPartials bool `long:"log_partials" description:"Send what was successfully parsed from a line (only if the error occured in the log line's message)."` + LogPartials bool `long:"log_partials" description:"Send what was successfully parsed from a line (only if the error occured in the log line's message)." yaml:"log_partials,omitempty"` - NumParsers int `hidden:"true" description:"number of mongo parsers to spin up"` + NumParsers int `hidden:"true" description:"number of mongo parsers to spin up" yaml:"-"` } type Parser struct { diff --git a/parsers/mysql/mysql.go b/parsers/mysql/mysql.go index 18bd792..5c2f9f8 100644 --- a/parsers/mysql/mysql.go +++ b/parsers/mysql/mysql.go @@ -11,9 +11,9 @@ import ( "sync" "time" - "github.com/sirupsen/logrus" _ "github.com/go-sql-driver/mysql" "github.com/honeycombio/mysqltools/query/normalizer" + "github.com/sirupsen/logrus" "github.com/honeycombio/honeytail/event" "github.com/honeycombio/honeytail/httime" @@ -132,9 +132,9 @@ type Options struct { Host string `long:"host" description:"MySQL host in the format (address:port)"` User string `long:"user" description:"MySQL username"` Pass string `long:"pass" description:"MySQL password"` - QueryInterval uint `long:"interval" description:"interval for querying the MySQL DB in seconds" default:"30"` + QueryInterval uint `long:"interval" description:"interval for querying the MySQL DB in seconds"` - NumParsers int `hidden:"true" description:"number of MySQL parsers to spin up"` + NumParsers int `hidden:"true" description:"number of MySQL parsers to spin up" yaml:"-"` } type Parser struct { @@ -178,9 +178,13 @@ func (p *Parser) Init(options interface{}) error { p.role = role // update hostedOn and readOnly every seconds + queryInterval := p.conf.QueryInterval + if queryInterval == 0 { + queryInterval = 30 + } go func() { defer db.Close() - ticker := time.NewTicker(time.Second * time.Duration(p.conf.QueryInterval)) + ticker := time.NewTicker(time.Second * time.Duration(queryInterval)) for _ = range ticker.C { readOnly, err := getReadOnly(db) if err != nil { diff --git a/parsers/regex/regex.go b/parsers/regex/regex.go index 53eff4e..483de42 100644 --- a/parsers/regex/regex.go +++ b/parsers/regex/regex.go @@ -23,10 +23,10 @@ type Options struct { // Note: `LineRegex` and `line_regex` are named as singular so that // it's less confusing to users to input them. // Might be worth making this consistent across the entire repo - LineRegex []string `long:"line_regex" description:"Regular expression with named capture groups representing the fields you want parsed (RE2 syntax). You can enter multiple regexes to match (--regex.line_regex=\"(?Pre)\" --regex.line_regex=\"(?P...)\"). Parses using the first regex to match a line, so list them in most-to-least-specific order."` - TimeFieldName string `long:"timefield" description:"Name of the field that contains a timestamp"` - TimeFieldFormat string `long:"time_format" description:"Timestamp format to use (strftime and Golang time.Parse supported)"` - NumParsers int `hidden:"true" description:"number of regex parsers to spin up"` + LineRegex []string `long:"line_regex" description:"Regular expression with named capture groups representing the fields you want parsed (RE2 syntax). You can enter multiple regexes to match (--regex.line_regex=\"(?Pre)\" --regex.line_regex=\"(?P...)\"). Parses using the first regex to match a line, so list them in most-to-least-specific order." yaml:"line_regex,omitempty"` + TimeFieldName string `long:"timefield" description:"Name of the field that contains a timestamp" yaml:"timefield,omitempty"` + TimeFieldFormat string `long:"time_format" description:"Timestamp format to use (strftime and Golang time.Parse supported)" yaml:"time_format,omitempty"` + NumParsers int `hidden:"true" description:"number of regex parsers to spin up" yaml:"-"` } type Parser struct { diff --git a/tail/tail.go b/tail/tail.go index 4a13a3a..ad38478 100644 --- a/tail/tail.go +++ b/tail/tail.go @@ -33,11 +33,11 @@ const ( ) type TailOptions struct { - ReadFrom string `long:"read_from" description:"Location in the file from which to start reading. Values: beginning, end, last. Last picks up where it left off, if the file has not been rotated, otherwise beginning. When --backfill is set, it will override this option=beginning" default:"last"` - Stop bool `long:"stop" description:"Stop reading the file after reaching the end rather than continuing to tail. When --backfill is set, it will override this option=true"` - Poll bool `long:"poll" description:"use poll instead of inotify to tail files"` - StateFile string `long:"statefile" description:"File in which to store the last read position. Defaults to a file in /tmp named $logfile.leash.state. If tailing multiple files, default is forced."` - HashStateFileDirPaths bool `long:"hash_statefile_paths" description:"Generates a hash of the directory path for each file that is used to uniquely identify each statefile. Prevents re-using the same statefile for tailed files that have the same name."` + ReadFrom string `long:"read_from" description:"Location in the file from which to start reading. Values: beginning, end, last. Last picks up where it left off, if the file has not been rotated, otherwise beginning. When --backfill is set, it will override this option to beginning" yaml:"read_from,omitempty"` + Stop bool `long:"stop" description:"Stop reading the file after reaching the end rather than continuing to tail. When --backfill is set, it will override this option=true" yaml:"stop,omitempty"` + Poll bool `long:"poll" description:"use poll instead of inotify to tail files" yaml:"poll,omitempty"` + StateFile string `long:"statefile" description:"File in which to store the last read position. Defaults to a file in /tmp named $logfile.leash.state. If tailing multiple files, default is forced." yaml:"statefile,omitempty"` + HashStateFileDirPaths bool `long:"hash_statefile_paths" description:"Generates a hash of the directory path for each file that is used to uniquely identify each statefile. Prevents re-using the same statefile for tailed files that have the same name." yaml:"hash_statefile_paths,omitempty"` } // Statefile mechanics when ReadFrom is 'last' @@ -349,7 +349,11 @@ func getTailer(conf Config, file string, stateFile string) (*tail.Tail, error) { // tail a real file var loc *tail.SeekInfo // 0 value means start at beginning var reOpen, follow bool = true, true - switch conf.Options.ReadFrom { + var readFrom = conf.Options.ReadFrom + if readFrom == "" { + readFrom = "last" + } + switch readFrom { case "start", "beginning": // 0 value for tail.SeekInfo means start at beginning case "end": @@ -360,8 +364,7 @@ func getTailer(conf Config, file string, stateFile string) (*tail.Tail, error) { case "last": loc = getStartLocation(stateFile, file) default: - errMsg := fmt.Sprintf("unknown option to --read_from: %s", - conf.Options.ReadFrom) + errMsg := fmt.Sprintf("unknown option to --read_from: %s", readFrom) return nil, errors.New(errMsg) } if conf.Options.Stop { @@ -391,10 +394,10 @@ func getTailer(conf Config, file string, stateFile string) (*tail.Tail, error) { // It might describe an existing file, an existing directory, or a new path. // // If tailing a single logfile, we will use the specified --tail.statefile: -// - if it points to an existing file, that statefile will be used directly -// - if it points to a new path, that path will be written to directly -// - if it points to an existing directory, the statefile will be placed inside -// the directory (and the statefile's name will be derived from the logfile). +// - if it points to an existing file, that statefile will be used directly +// - if it points to a new path, that path will be written to directly +// - if it points to an existing directory, the statefile will be placed inside +// the directory (and the statefile's name will be derived from the logfile). // // If honeytail is asked to tail multiple files, we will only respect the // third case, where --tail.statefile describes an existing directory.