Rename Auditbeat modules

This changes the audit.file and audit.kernel metricsets into modules
named file_integrity and auditd, respectively. This requires existing
users to update their configuration.

The dashboards need to be updated to account for these changes.

Closes elastic#5422 (see the issue for more details)

CHANGELOG.asciidoc

@@ -21,6 +21,9 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 *Auditbeat*
 
 - Changed `audit.file.path` to be a multi-field so that path is searchable. {pull}5625[5625]
+- Split the audit.kernel and audit.file metricsets into their own modules
+  named auditd and file_integrity, respectively. This change requires
+  existing users to update their config. {issue}5422[5422]
 
 *Filebeat*
 

Vagrantfile

@@ -52,16 +52,19 @@ echo 'Creating github.com/elastic in the GOPATH'
 mkdir -p ~/go/src/github.com/elastic
 echo 'Symlinking /vagrant to ~/go/src/github.com/elastic'
 cd ~/go/src/github.com/elastic
-if [ -d "/vagrant" ]; then ln -s /vagrant beats; fi
+if [ -d "/vagrant" ]  && [ ! -e "beats" ]; then ln -s /vagrant beats; fi
 SCRIPT
 
 # Linux GVM
 $linuxGvmProvision = <<SCRIPT
 mkdir -p ~/bin
-curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.0.1/gvm-linux-amd64
-chmod +x ~/bin/gvm
-echo 'export PATH=~/bin:$PATH' >> ~/.bash_profile
-echo 'eval "$(gvm 1.9.2)"' >> ~/.bash_profile
+if [ ! -e "~/bin/gvm" ]; then
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.0.5/gvm-linux-amd64
+  chmod +x ~/bin/gvm
+  echo 'export GOPATH=$HOME/go' >> ~/.bash_profile
+  echo 'export PATH=$HOME/bin:$GOPATH/bin:$PATH' >> ~/.bash_profile
+  echo 'eval "$(gvm 1.9.2)"' >> ~/.bash_profile
+fi
 SCRIPT
 
 Vagrant.configure(2) do |config|
@@ -119,16 +122,25 @@ Vagrant.configure(2) do |config|
     openbsd.vm.provision "shell", inline: $unixProvision, privileged: false
   end
 
-  # CentOS 7
-  config.vm.define "centos7", primary: true do |centos7|
-    #centos7.vm.box = "http://cloud.centos.org/centos/7/vagrant/x86_64/images/CentOS-7-x86_64-Vagrant-1706_02.VirtualBox.box"
-    centos7.vm.box = "ubuntu/precise64"
-    centos7.vm.network :forwarded_port, guest: 22,   host: 2226,  id: "ssh", auto_correct: true
+  config.vm.define "precise64", primary: true do |c|
+    c.vm.box = "ubuntu/precise64"
+    c.vm.network :forwarded_port, guest: 22,   host: 2226,  id: "ssh", auto_correct: true
 
-    centos7.vm.provision "shell", inline: $unixProvision, privileged: false
-    centos7.vm.provision "shell", inline: $linuxGvmProvision, privileged: false
+    c.vm.provision "shell", inline: $unixProvision, privileged: false
+    c.vm.provision "shell", inline: $linuxGvmProvision, privileged: false
 
-    centos7.vm.synced_folder ".", "/vagrant", type: "virtualbox"
+    c.vm.synced_folder ".", "/vagrant", type: "virtualbox"
+  end
+
+  config.vm.define "fedora26", primary: true do |c|
+    c.vm.box = "bento/fedora-26"
+    c.vm.network :forwarded_port, guest: 22,   host: 2227,  id: "ssh", auto_correct: true
+
+    c.vm.provision "shell", inline: $unixProvision, privileged: false
+    c.vm.provision "shell", inline: $linuxGvmProvision, privileged: false
+    config.vm.provision "shell", inline: "dnf install -y make gcc python-pip python-virtualenv git"
+
+    c.vm.synced_folder ".", "/vagrant", type: "virtualbox"
   end
 
 end

auditbeat/Makefile

@@ -4,6 +4,7 @@ BEAT_DESCRIPTION=Audit the activities of users and processes on your system.
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
 GOX_OS?=linux windows ## @Building List of all OS to be supported by "make crosscompile".
+DEV_OS?=linux
 
 # Path to the libbeat Makefile
 -include ../libbeat/scripts/Makefile
@@ -49,17 +50,17 @@ collect: fields collect-docs configs kibana
 .PHONY: fields
 fields: python-env
 	@mkdir -p _meta
-	@cp ${ES_BEATS}/metricbeat/_meta/fields.common.yml _meta/fields.generated.yml
+	@cp _meta/fields.common.yml _meta/fields.generated.yml
 	@${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml
 
 # Collects all module configs
 .PHONY: configs
 configs: python-env
 	@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
-	     <(go run scripts/generate_config.go -os linux -concat) \
+	     <(go run scripts/generate_config.go -os ${DEV_OS} -concat) \
 		 ${ES_BEATS}/auditbeat/_meta/common.p2.yml > _meta/beat.yml
 	@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
-	     <(go run scripts/generate_config.go -os linux -ref -concat) > _meta/beat.reference.yml
+	     <(go run scripts/generate_config.go -os ${DEV_OS} -ref -concat) > _meta/beat.reference.yml
 
 # Collects all module docs
 .PHONY: collect-docs

auditbeat/_meta/fields.common.yml

@@ -3,34 +3,7 @@
   description: >
     Contains common fields available in all event types.
   fields:
-
-    - name: metricset.module
+    - name: dataset.module
       description: >
         The name of the module that generated the event.
 
-    - name: metricset.name
-      description: >
-        The name of the metricset that generated the event.
-
-    - name: metricset.host
-      description: >
-        Hostname of the machine from which the metricset was collected. This
-        field may not be present when the data was collected locally.
-
-    - name: metricset.rtt
-      type: long
-      required: true
-      description: >
-        Event round trip time in microseconds.
-
-    - name: metricset.namespace
-      type: keyword
-      description: >
-        Namespace of dynamic metricsets.
-
-    - name: type
-      required: true
-      example: metricsets
-      description: >
-        The document type. Always set to "metricsets".
-

auditbeat/auditbeat.reference.yml

@@ -29,17 +29,16 @@ auditbeat.max_start_delay: 10s
 #==========================  Modules configuration =============================
 auditbeat.modules:
 
-# The kernel metricset collects events from the audit framework in the Linux
+# The auditd module collects events from the audit framework in the Linux
 # kernel. You need to specify audit rules for the events that you want to audit.
-- module: audit
-  metricsets: [kernel]
-  kernel.resolve_ids: true
-  kernel.failure_mode: silent
-  kernel.backlog_limit: 8196
-  kernel.rate_limit: 0
-  kernel.include_raw_message: false
-  kernel.include_warnings: false
-  kernel.audit_rules: |
+- module: auditd
+  resolve_ids: true
+  failure_mode: silent
+  backlog_limit: 8196
+  rate_limit: 0
+  include_raw_message: false
+  include_warnings: false
+  audit_rules: |
     ## Define audit rules here.
     ## Create file watches (-w) or syscall audits (-a or -A). Uncomment these
     ## examples or add your own rules.
@@ -65,11 +64,10 @@ auditbeat.modules:
     #-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access
     #-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access
 
-# The file integrity metricset sends events when files are changed (created,
+# The file integrity module sends events when files are changed (created,
 # updated, deleted). The events contain file metadata and hashes.
-- module: audit
-  metricsets: [file]
-  file.paths:
+- module: file_integrity
+  paths:
   - /bin
   - /usr/bin
   - /sbin
@@ -78,22 +76,23 @@ auditbeat.modules:
 
   # Scan over the configured file paths at startup and send events for new or
   # modified files since the last time Auditbeat was running.
-  file.scan_at_start: true
+  scan_at_start: true
 
   # Average scan rate. This throttles the amount of CPU and I/O that Auditbeat
   # consumes at startup while scanning. Default is "50 MiB".
-  file.scan_rate_per_sec: 50 MiB
+  scan_rate_per_sec: 50 MiB
 
   # Limit on the size of files that will be hashed. Default is "100 MiB".
-  file.max_file_size: 100 MiB
+  # Limit on the size of files that will be hashed. Default is "100 MiB".
+  max_file_size: 100 MiB
 
   # Hash types to compute when the file changes. Supported types are md5, sha1,
   # sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, sha3_256,
   # sha3_384 and sha3_512. Default is sha1.
-  file.hash_types: [sha1]
+  hash_types: [sha1]
 
   # Detect changes to files included in subdirectories. Disabled by default.
-  file.recursive: false
+  recursive: false
 
 
 #================================ General ======================================

auditbeat/auditbeat.yml

@@ -10,9 +10,8 @@
 #==========================  Modules configuration =============================
 auditbeat.modules:
 
-- module: audit
-  metricsets: [kernel]
-  kernel.audit_rules: |
+- module: auditd
+  audit_rules: |
     ## Define audit rules here.
     ## Create file watches (-w) or syscall audits (-a or -A). Uncomment these
     ## examples or add your own rules.
@@ -38,9 +37,8 @@ auditbeat.modules:
     #-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access
     #-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access
 
-- module: audit
-  metricsets: [file]
-  file.paths:
+- module: file_integrity
+  paths:
   - /bin
   - /usr/bin
   - /sbin

auditbeat/cmd/root.go

@@ -5,7 +5,9 @@ import (
 
 	"github.com/elastic/beats/metricbeat/beater"
 
+	"github.com/elastic/beats/auditbeat/core"
 	cmd "github.com/elastic/beats/libbeat/cmd"
+	"github.com/elastic/beats/metricbeat/mb/module"
 )
 
 // Name of the beat (auditbeat).
@@ -15,6 +17,11 @@ const Name = "auditbeat"
 var RootCmd *cmd.BeatsRootCmd
 
 func init() {
+	create := beater.Creator(
+		beater.WithModuleOptions(
+			module.WithEventModifier(core.AddDatasetToEvent),
+		),
+	)
 	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
-	RootCmd = cmd.GenRootCmdWithRunFlags(Name, "", beater.DefaultCreator(), runFlags)
+	RootCmd = cmd.GenRootCmdWithRunFlags(Name, "", create, runFlags)
 }

auditbeat/core/eventmod.go

@@ -0,0 +1,16 @@
+package core
+
+import (
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/metricbeat/mb"
+)
+
+// AddDatasetToEvent adds dataset information to the event. In particular this
+// adds the module name under dataset.module.
+func AddDatasetToEvent(module, metricSet string, event *mb.Event) {
+	if event.RootFields == nil {
+		event.RootFields = common.MapStr{}
+	}
+
+	event.RootFields.Put("dataset.module", module)
+}