Merge remote-tracking branch 'upstream/main' into citrix-timestamps

elastic · Aug 28, 2024 · 5877c49 · 5877c49
2 parents c1deaa6 + 930a301
commit 5877c49
Show file tree

Hide file tree

Showing 247 changed files with 38,124 additions and 15,281 deletions.
diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
@@ -6,7 +6,7 @@ env:
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "false" # not required to set since system tests are not running yet
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.30.0'
+  K8S_VERSION: 'v1.31.0'
   YQ_VERSION: 'v4.35.2'
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
   GH_CLI_VERSION: "2.29.0"

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -5,7 +5,7 @@ env:
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.2"
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.30.0'
+  K8S_VERSION: 'v1.31.0'
   YQ_VERSION: 'v4.35.2'
   JQ_VERSION: '1.7'
   GH_CLI_VERSION: "2.29.0"

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -206,6 +206,7 @@
 /packages/gcp/data_stream/vpcflow @elastic/security-service-integrations
 /packages/gcp_metrics @elastic/obs-ds-hosted-services
 /packages/gcp_pubsub @elastic/security-service-integrations
+/packages/gigamon @elastic/security-service-integrations
 /packages/github @elastic/security-service-integrations
 /packages/gitlab @elastic/security-service-integrations
 /packages/golang @elastic/obs-infraobs-integrations

diff --git a/.github/workflows/bump-elastic-stack-version.yml b/.github/workflows/bump-elastic-stack-version.yml
@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@a0c478c868a71d3e239a65714de69450aa1ee2c6 #v2.65.0
+        uses: updatecli/updatecli-action@d0950ebbe80f4f80c3392b288d6a218fae872f69 #v2.66.0
 
       - name: Select diff action
         if: ${{ github.event_name == 'pull_request' }}

diff --git a/packages/abnormal_security/_dev/deploy/docker/files/config.yml b/packages/abnormal_security/_dev/deploy/docker/files/config.yml
@@ -323,23 +323,7 @@ rules:
           Content-Type:
             - 'application/json'
         body: |-
-          {"threats":[{"threatId":"284712ab-6d8b-47b3-89d3-a314efef79e2"}],"pageNumber":2,"nextPageNumber":3}
-  - path: /v1/threats
-    methods: ['GET']
-    query_params:
-      filter: "{filter:.*}"
-      pageNumber: 3
-      pageSize: 1
-    request_headers:
-      Authorization:
-        - "Bearer xxxx"
-    responses:
-      - status_code: 200
-        headers:
-          Content-Type:
-            - 'application/json'
-        body: |-
-          {"threats":[],"pageNumber":3,"nextPageNumber":4}
+          {"threats":[{"threatId":"284712ab-6d8b-47b3-89d3-a314efef79e2"}]}
   - path: /v1/threats/184712ab-6d8b-47b3-89d3-a314efef79e2
     methods: ['GET']
     query_params:
@@ -402,26 +386,9 @@ rules:
                     "https://www.example.com/"
                   ]
                 }
-              ],
-              "pageNumber": 1,
-              "nextPageNumber": 2
+              ]
           }
           `}}
-  - path: /v1/threats/184712ab-6d8b-47b3-89d3-a314efef79e2
-    methods: ['GET']
-    query_params:
-      pageNumber: 2
-      pageSize: 1
-    request_headers:
-      Authorization:
-        - "Bearer xxxx"
-    responses:
-      - status_code: 200
-        headers:
-          Content-Type:
-            - 'application/json'
-        body: |-
-          {"threatId":"184712ab-6d8b-47b3-89d3-a314efef79e2","messages":[],"pageNumber":2,"nextPageNumber":3}
   - path: /v1/threats/284712ab-6d8b-47b3-89d3-a314efef79e2
     methods: ['GET']
     query_params:
@@ -483,23 +450,6 @@ rules:
                         "https://www.example.com/"
                     ]
                 }
-              ],
-              "pageNumber": 1,
-              "nextPageNumber": 2
+              ]
           }
           `}}
-  - path: /v1/threats/284712ab-6d8b-47b3-89d3-a314efef79e2
-    methods: ['GET']
-    query_params:
-      pageNumber: 2
-      pageSize: 1
-    request_headers:
-      Authorization:
-        - "Bearer xxxx"
-    responses:
-      - status_code: 200
-        headers:
-          Content-Type:
-            - 'application/json'
-        body: |-
-          {"threatId":"284712ab-6d8b-47b3-89d3-a314efef79e2","messages":[],"pageNumber":2,"nextPageNumber":3}
diff --git a/packages/abnormal_security/changelog.yml b/packages/abnormal_security/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.1.1"
+  changes:
+    - description: Update pagination termination condition in threat data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10860
 - version: "0.1.0"
   changes:
     - description: Initial release.

diff --git a/packages/abnormal_security/data_stream/threat/agent/stream/cel.yml.hbs b/packages/abnormal_security/data_stream/threat/agent/stream/cel.yml.hbs
@@ -50,7 +50,7 @@ program: |
           }
         }).do_request().as(resp, resp.StatusCode == 200 ?
             bytes(resp.Body).decode_json().as(body, {
-              "worklist": body.threats.map(e, e.threatId),
+              "worklist": body,
               "next": 0,
             })
           :
@@ -73,9 +73,9 @@ program: |
       ))
   ).as(state, state.with(
     !has(state.worklist) ? state : // Exit early due to GET failure.
-      state.next < size(state.worklist) ?
+      has(state.worklist.threats) && size(state.worklist.threats) > 0 ?
         request("GET",
-          state.url.trim_right("/") + "/v1/threats/" + string(state.worklist[state.next]) + "?" + {
+          state.url.trim_right("/") + "/v1/threats/" + string(state.worklist.threats[state.next].threatId) + "?" + {
             "pageSize": [string(state.page_size)],
             "pageNumber": [string(state.child_next_page)]
           }.format_query()
@@ -85,30 +85,25 @@ program: |
           }
         }).do_request().as(resp, resp.StatusCode == 200 ?
           bytes(resp.Body).decode_json().as(body,{
-            "events": (
-              size(body.messages) > 0 ?
-                body.messages.map(e,{
-                  "message": e.encode_json(),
-                })
-              :
-                [{"message":"retry"}]
-            ),
+            "events": body.messages.map(e,{
+              "message": e.encode_json(),
+            }),
             "cursor": {
               "last_timestamp": state.end_time
             },
-            "worklist": int(state.next) + 1 < size(state.worklist) || size(body.messages) > 0 ? state.worklist : [],
-            "child_next_page": size(body.messages) > 0 ? int(state.child_next_page) + 1 : 1,
+            "worklist": int(state.next) + 1 < size(state.worklist.threats) || has(body.nextPageNumber) ? state.worklist : {},
+            "child_next_page": has(body.nextPageNumber) ? body.nextPageNumber : 1,
             "next": (
-              size(body.messages) > 0 ?
+              has(body.nextPageNumber) ?
                 state.next
               :
-                int(state.next) + 1 < size(state.worklist) ?
+                int(state.next) + 1 < size(state.worklist.threats) ?
                   int(state.next) + 1
                 :
                   0
             ),
-            "next_page": int(state.next) + 1 < size(state.worklist) || size(body.messages) > 0 ? state.next_page : int(state.next_page) + 1,
-            "want_more": true,
+            "next_page": int(state.next) + 1 < size(state.worklist.threats) || has(body.nextPageNumber) ? state.next_page : has(state.worklist.nextPageNumber) ? state.worklist.nextPageNumber : 1,
+            "want_more": int(state.next) + 1 < size(state.worklist.threats) || has(body.nextPageNumber) || has(state.worklist.nextPageNumber),
           })
         :
           {

diff --git a/packages/abnormal_security/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/abnormal_security/data_stream/threat/elasticsearch/ingest_pipeline/default.yml
@@ -9,9 +9,6 @@ processors:
       tag: data_collection_error
       if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null
       message: error message set and no data to process.
-  - drop:
-      if: ctx.message == 'retry'
-      tag: drop_retry_events
   - rename:
       field: message
       tag: rename_message_to_event_original

diff --git a/packages/abnormal_security/data_stream/threat/sample_event.json b/packages/abnormal_security/data_stream/threat/sample_event.json
@@ -45,22 +45,22 @@
         }
     },
     "agent": {
-        "ephemeral_id": "3cfaa9dc-bca8-4e29-a807-77b68709b731",
-        "id": "7aaba523-565c-4597-bc42-59135436336b",
-        "name": "docker-fleet-agent",
+        "ephemeral_id": "b66f399f-ba1c-4fe5-af82-9ca7a0204545",
+        "id": "e2eadaf0-613d-41d9-913c-96125e06487a",
+        "name": "elastic-agent-55334",
         "type": "filebeat",
         "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "abnormal_security.threat",
-        "namespace": "37330",
+        "namespace": "45319",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "7aaba523-565c-4597-bc42-59135436336b",
+        "id": "e2eadaf0-613d-41d9-913c-96125e06487a",
         "snapshot": false,
         "version": "8.13.0"
     },
@@ -88,7 +88,7 @@
         ],
         "dataset": "abnormal_security.threat",
         "id": "2260288475997441000",
-        "ingested": "2024-08-08T06:53:48Z",
+        "ingested": "2024-08-23T05:40:07Z",
         "kind": "enrichment",
         "original": "{\"abxMessageId\":2260288475997441000,\"abxPortalUrl\":\"https://portal.abnormalsecurity.com/home/threat-center/remediation-history/3456765434567654\",\"attachmentCount\":0,\"attachmentNames\":[],\"attackStrategy\":\"Unknown Sender\",\"attackType\":\"Spam\",\"attackVector\":\"Link\",\"attackedParty\":\"Employee (Other)\",\"autoRemediated\":true,\"ccEmails\":[],\"fromAddress\":\"[email protected]\",\"fromName\":\"john\",\"impersonatedParty\":\"None / Others\",\"internetMessageId\":\"\\u003cAZz8NUMEST-qmuz77_koic@example\\u003e\",\"isRead\":false,\"postRemediated\":false,\"receivedTime\":\"2024-07-17T23:25:38Z\",\"recipientAddress\":\"[email protected]\",\"remediationStatus\":\"Auto-Remediated\",\"remediationTimestamp\":\"2024-07-17T23:25:45.73564Z\",\"replyToEmails\":[],\"returnPath\":\"[email protected]\",\"senderDomain\":\"example.com\",\"senderIpAddress\":\"81.2.69.142\",\"sentTime\":\"2024-07-17T23:25:29Z\",\"subject\":\"YoU.have.𝗪𝟬0𝗡𝗡 a K0baIt 215-piece_ToooI_Set_Noo0wW..#GBOB\",\"summaryInsights\":[\"Abnormal Email Body HTML\",\"Invisible characters found in Email\",\"Suspicious Link\",\"Unusual Sender\",\"Unusual Sender Domain\"],\"threatId\":\"bf255f2d-a2ad-3f50-5075-fdcc24308bbd\",\"toAddresses\":[\"[email protected]\"],\"urlCount\":1,\"urls\":[\"https://www.example.com/\"]}",
         "reference": "https://portal.abnormalsecurity.com/home/threat-center/remediation-history/3456765434567654",

diff --git a/packages/abnormal_security/docs/README.md b/packages/abnormal_security/docs/README.md
@@ -498,22 +498,22 @@ An example event for `threat` looks as following:
         }
     },
     "agent": {
-        "ephemeral_id": "3cfaa9dc-bca8-4e29-a807-77b68709b731",
-        "id": "7aaba523-565c-4597-bc42-59135436336b",
-        "name": "docker-fleet-agent",
+        "ephemeral_id": "b66f399f-ba1c-4fe5-af82-9ca7a0204545",
+        "id": "e2eadaf0-613d-41d9-913c-96125e06487a",
+        "name": "elastic-agent-55334",
         "type": "filebeat",
         "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "abnormal_security.threat",
-        "namespace": "37330",
+        "namespace": "45319",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "7aaba523-565c-4597-bc42-59135436336b",
+        "id": "e2eadaf0-613d-41d9-913c-96125e06487a",
         "snapshot": false,
         "version": "8.13.0"
     },
@@ -541,7 +541,7 @@ An example event for `threat` looks as following:
         ],
         "dataset": "abnormal_security.threat",
         "id": "2260288475997441000",
-        "ingested": "2024-08-08T06:53:48Z",
+        "ingested": "2024-08-23T05:40:07Z",
         "kind": "enrichment",
         "original": "{\"abxMessageId\":2260288475997441000,\"abxPortalUrl\":\"https://portal.abnormalsecurity.com/home/threat-center/remediation-history/3456765434567654\",\"attachmentCount\":0,\"attachmentNames\":[],\"attackStrategy\":\"Unknown Sender\",\"attackType\":\"Spam\",\"attackVector\":\"Link\",\"attackedParty\":\"Employee (Other)\",\"autoRemediated\":true,\"ccEmails\":[],\"fromAddress\":\"[email protected]\",\"fromName\":\"john\",\"impersonatedParty\":\"None / Others\",\"internetMessageId\":\"\\u003cAZz8NUMEST-qmuz77_koic@example\\u003e\",\"isRead\":false,\"postRemediated\":false,\"receivedTime\":\"2024-07-17T23:25:38Z\",\"recipientAddress\":\"[email protected]\",\"remediationStatus\":\"Auto-Remediated\",\"remediationTimestamp\":\"2024-07-17T23:25:45.73564Z\",\"replyToEmails\":[],\"returnPath\":\"[email protected]\",\"senderDomain\":\"example.com\",\"senderIpAddress\":\"81.2.69.142\",\"sentTime\":\"2024-07-17T23:25:29Z\",\"subject\":\"YoU.have.𝗪𝟬0𝗡𝗡 a K0baIt 215-piece_ToooI_Set_Noo0wW..#GBOB\",\"summaryInsights\":[\"Abnormal Email Body HTML\",\"Invisible characters found in Email\",\"Suspicious Link\",\"Unusual Sender\",\"Unusual Sender Domain\"],\"threatId\":\"bf255f2d-a2ad-3f50-5075-fdcc24308bbd\",\"toAddresses\":[\"[email protected]\"],\"urlCount\":1,\"urls\":[\"https://www.example.com/\"]}",
         "reference": "https://portal.abnormalsecurity.com/home/threat-center/remediation-history/3456765434567654",

diff --git a/packages/abnormal_security/manifest.yml b/packages/abnormal_security/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.2.1
 name: abnormal_security
 title: Abnormal Security
-version: 0.1.0
+version: 0.1.1
 description: Collect logs from Abnormal Security with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/amazon_security_lake/_dev/build/docs/README.md b/packages/amazon_security_lake/_dev/build/docs/README.md
@@ -4,8 +4,9 @@ This [Amazon Security Lake](https://aws.amazon.com/security-lake/) integration h
 
 Security Lake automates the collection of security-related log and event data from integrated AWS services and third-party services. It also helps you manage the lifecycle of data with customizable retention and replication settings. Security Lake converts ingested data into Apache Parquet format and a standard open-source schema called the Open Cybersecurity Schema Framework (OCSF). With OCSF support, Security Lake normalizes and combines security data from AWS and a broad range of enterprise security data sources.
 
-The Amazon Security Lake integration currently supports only one mode of log collection:
+The Amazon Security Lake integration can be used in two different modes to collect data:
 - AWS S3 polling mode: Amazon Security Lake writes data to S3, and Elastic Agent polls the S3 bucket by listing its contents and reading new files.
+- AWS S3 SQS mode: Amazon Security Lake writes data to S3, S3 sends a notification of a new object to SQS, the Elastic Agent receives the notification from SQS, and then reads the S3 object. Multiple agents can be used in this mode.
 
 ## Compatibility
 
@@ -37,6 +38,7 @@ The Amazon Security Lake integration collects logs from both [Third-party servic
    - For **Log and event sources**, choose which sources the subscriber is authorized to consume.
    - For **Data access method**, choose **S3** to set up data access for the subscriber.
    - For **Subscriber credentials**, provide the subscriber's **AWS account ID** and **external ID**.
+   - For **Notification details**, select **SQS queue**.
    - Choose Create.
 3. Above mentioned steps will create and provide the required details such as IAM roles/AWS role ID, external ID and queue URL to configure AWS Security Lake Integration.
 
@@ -48,11 +50,18 @@ The Amazon Security Lake integration collects logs from both [Third-party servic
 3. Click on the "Amazon Security Lake" integration from the search results.
 4. Click on the Add Amazon Security Lake Integration button to add the integration.
    ![Home Page](../img/home_page.png)
-5. The integration currently only supports collecting logs via AWS S3.
-6. While adding the integration, you have to configure the following details:
-   - bucket arn
+5. By default collect logs via S3 Bucket toggle will be off and collect logs for AWS SQS.
+   - queue url
+      ![Queue URL](../img/queue_url.png)
+   - collect logs via S3 Bucket toggled off
    - role ARN
    - external id
+      ![Role ARN and External ID](../img/role_arn_and_external_id.png)
+
+6. If you want to collect logs via AWS S3, then you have to put the following details:
+    - bucket arn
+    - role ARN
+    - external id
 
 **NOTE**:
 

diff --git a/packages/amazon_security_lake/changelog.yml b/packages/amazon_security_lake/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.0"
+  changes:
+    - description: Re-added SQS notification settings which were removed due to a prior update error.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10854
 - version: "1.4.1"
   changes:
     - description: "Remove confusing documentation remaining from previous change."

diff --git a/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs b/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs
@@ -1,3 +1,4 @@
+{{#if collect_s3_logs}}
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
@@ -11,10 +12,32 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+
+{{else}}
+
+{{#if queue_url}}
+queue_url: {{queue_url}}
+{{/if}}
+sqs.notification_parsing_script.source: {{event_parsing_script}}
+{{#if region}}
+region: {{region}}
+{{/if}}
+{{#if visibility_timeout}}
+visibility_timeout: {{visibility_timeout}}
+{{/if}}
+{{#if api_timeout}}
+api_timeout: {{api_timeout}}
+{{/if}}
+{{#if max_number_of_messages}}
+max_number_of_messages: {{max_number_of_messages}}
+{{/if}}
 {{#if file_selectors}}
 file_selectors:
 {{file_selectors}}
 {{/if}}
+
+{{/if}}
+
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}
@@ -56,6 +79,11 @@ proxy_url: {{proxy_url}}
 ssl: {{ssl}}
 {{/if}}
 tags:
+{{#if collect_s3_logs}}
+  - collect_s3_logs
+{{else}}
+  - collect_sqs_logs
+{{/if}}
 {{#if preserve_original_event}}
   - preserve_original_event
 {{/if}}