Adding WAF Log type (#617)

* Adding WAF Log type Signed-off-by: Sandesh Kumar <[email protected]> * Minor refactoring Signed-off-by: Sandesh Kumar <[email protected]> * Fix duplicate keys Signed-off-by: Sandesh Kumar <[email protected]> * Test fix Signed-off-by: Sandesh Kumar <[email protected]> * test fix Signed-off-by: Sandesh Kumar <[email protected]> * Adding custom log specific to aws waf Signed-off-by: Sandesh Kumar <[email protected]> * Add Security category to waf logtype Signed-off-by: Sandesh Kumar <[email protected]> --------- Signed-off-by: Sandesh Kumar <[email protected]> (cherry picked from commit 115ae95)
opensearch-project · Oct 5, 2023 · 3120910 · 3120910
1 parent 1b8ec5d
commit 3120910
Show file tree

Hide file tree

Showing 11 changed files with 353 additions and 32 deletions.
diff --git a/src/main/resources/OSMapping/logtypes.json b/src/main/resources/OSMapping/logtypes.json
@@ -196,5 +196,14 @@
     "tags": {
       "correlation_id": 23
     }
+  },
+  "waf": {
+    "name": "waf",
+    "description": "Web Application Firewall based logs",
+    "category": "Security",
+    "source": "Sigma",
+    "tags": {
+      "correlation_id": 24
+    }
   }
 }
diff --git a/src/main/resources/OSMapping/waf_logtype.json b/src/main/resources/OSMapping/waf_logtype.json
@@ -0,0 +1,55 @@
+{
+  "name": "waf",
+  "description": "Web Application Firewall Log Type",
+  "is_builtin": true,
+  "mappings": [
+    {
+      "raw_field":"cs-method",
+      "ecs":"waf.request.method"
+    },
+    {
+      "raw_field":"httpRequest.httpMethod",
+      "ecs":"waf.request.method"
+    },
+    {
+      "raw_field":"cs-uri-query",
+      "ecs":"waf.request.uri_query"
+    },
+    {
+      "raw_field":"httpRequest.uri",
+      "ecs":"waf.request.uri_query"
+    },
+    {
+      "raw_field":"httpRequest.args",
+      "ecs":"waf.request.uri_query"
+    },
+    {
+      "raw_field":"cs-user-agent",
+      "ecs":"waf.request.headers.user_agent"
+    },
+    {
+      "raw_field":"httpRequest.headers",
+      "ecs":"waf.request.headers"
+    },
+    {
+      "raw_field":"sc-status",
+      "ecs":"waf.response.code"
+    },
+    {
+      "raw_field":"responseCodeSent",
+      "ecs":"waf.response.code"
+    },
+    {
+      "raw_field":"timestamp",
+      "ecs":"timestamp"
+    },
+    {
+      "raw_field":"httpRequest.headers.value",
+      "ecs":"waf.request.headers.value"
+    },
+    {
+      "raw_field":"httpRequest.headers.name",
+      "ecs":"waf.request.headers.name"
+    }
+  ]
+}
diff --git a/src/main/resources/rules/waf/aws_waf/aws_waf_web_susp_useragents.yml b/src/main/resources/rules/waf/aws_waf/aws_waf_web_susp_useragents.yml
@@ -0,0 +1,29 @@
+title: Suspicious User-Agents Related To Recon Tools - AWS WAF
+id: 19aa4f58-94ca-45ff-bc34-92e533c0994b
+status: experimental
+description: Detects known suspicious (default) user-agents related to scanning/recon tools
+references:
+  - https://github.com/wpscanteam/wpscan/blob/196fbab5b1ce3870a43515153d4f07878a89d410/lib/wpscan/browser.rb
+  - https://github.com/xmendez/wfuzz/blob/1b695ee9a87d66a7d7bf6cae70d60a33fae51541/docs/user/basicusage.rst
+  - https://github.com/lanmaster53/recon-ng/blob/9e907dfe09fce2997f0301d746796408e01a60b7/recon/core/base.py#L92
+author: Nasreddine Bencherchali (Nextron Systems), Tim Shelton, Sandesh Kumar (Amazon)
+date: 2022/07/19
+modified: 2023/09/26
+tags:
+  - attack.initial_access
+  - attack.t1190
+logsource:
+  category: webserver
+detection:
+  selection:
+    waf.request.headers.name: 'User-agent'
+    waf.request.headers.value|contains:
+        # Add more tools as you see fit
+        - 'Wfuzz/'
+        - 'WPScan v'
+        - 'Recon-ng/v'
+        - 'GIS - AppSec Team - Project Vision'
+  condition: selection
+falsepositives:
+  - Unknown
+level: medium
diff --git a/src/main/resources/rules/waf/web_cve_2023_25717_ruckus_wireless_admin_exploit_attempt.yml b/src/main/resources/rules/waf/web_cve_2023_25717_ruckus_wireless_admin_exploit_attempt.yml
@@ -0,0 +1,30 @@
+title: Potential CVE-2023-25717 Exploitation Attempt
+id: 043c1609-0e32-4462-a6f2-5a0c2da3fafe
+status: experimental
+description: Detects a potential exploitation attempt of CVE-2023-25717 a Remote Code Execution via an unauthenticated HTTP GET Request, in Ruckus Wireless Admin
+references:
+  - https://cybir.com/2023/cve/proof-of-concept-ruckus-wireless-admin-10-4-unauthenticated-remote-code-execution-csrf-ssrf/
+author: Nasreddine Bencherchali (Nextron Systems)
+date: 2023/05/30
+tags:
+  - attack.initial_access
+  - attack.t1190
+  - cve.2023.25717
+  - detection.emerging_threats
+logsource:
+  category: webserver
+detection:
+  selection:
+    cs-method: 'GET'
+    cs-uri-query|contains|all:
+      - '/forms/doLogin'
+      - 'login_username'
+      - 'password'
+    cs-uri-query|contains:
+      - '$('
+      - '%24%28' # URL Encode version of "$("
+  condition: selection
+falsepositives:
+  - Vulnerability scanners
+  - Some rare false positives may occur if the password contains the characters "$(". Apply addition indicators such as executed commands to remove FP
+level: high
diff --git a/src/main/resources/rules/waf/web_sql_injection_in_access_logs.yml b/src/main/resources/rules/waf/web_sql_injection_in_access_logs.yml
@@ -0,0 +1,60 @@
+title: SQL Injection Strings In URI
+id: 5513deaf-f49a-46c2-a6c8-3f111b5cb453
+status: test
+description: Detects potential SQL injection attempts via GET requests in access logs.
+references:
+  - https://www.acunetix.com/blog/articles/exploiting-sql-injection-example/
+  - https://www.acunetix.com/blog/articles/using-logs-to-investigate-a-web-application-attack/
+  - https://brightsec.com/blog/sql-injection-payloads/
+  - https://github.com/payloadbox/sql-injection-payload-list
+  - https://book.hacktricks.xyz/pentesting-web/sql-injection/mysql-injection
+author: Saw Win Naung, Nasreddine Bencherchali (Nextron Systems), Thurein Oo (Yoma Bank)
+date: 2020/02/22
+modified: 2023/09/04
+tags:
+  - attack.initial_access
+  - attack.t1190
+logsource:
+  category: webserver
+detection:
+  selection:
+    cs-method: 'GET'
+  keywords:
+    - '@@version'
+    - '%271%27%3D%271'
+    - '=select '
+    - '=select('
+    - '=select%20'
+    - 'concat_ws('
+    - 'CONCAT(0x'
+    - 'from mysql.innodb_table_stats'
+    - 'from%20mysql.innodb_table_stats'
+    - 'group_concat('
+    - 'information_schema.tables'
+    - 'json_arrayagg('
+    - 'or 1=1#'
+    - 'or%201=1#'
+    - 'order by '
+    - 'order%20by%20'
+    - 'select * '
+    - 'select database()'
+    - 'select version()'
+    - 'select%20*%20'
+    - 'select%20database()'
+    - 'select%20version()'
+    - 'select%28sleep%2810%29'
+    - 'SELECTCHAR('
+    - 'table_schema'
+    - 'UNION ALL SELECT'
+    - 'UNION SELECT'
+    - 'UNION%20ALL%20SELECT'
+    - 'UNION%20SELECT'
+    - "'1'='1"
+  filter_main_status:
+    sc-status: 404
+  condition: selection and keywords and not 1 of filter_main_*
+falsepositives:
+  - Java scripts and CSS Files
+  - User searches in search boxes of the respective website
+  - Internal vulnerability scanners can cause some serious FPs when used, if you experience a lot of FPs due to this think of adding more filters such as "User Agent" strings and more response codes
+level: high
diff --git a/src/main/resources/rules/waf/web_susp_useragents.yml b/src/main/resources/rules/waf/web_susp_useragents.yml
@@ -0,0 +1,28 @@
+title: Suspicious User-Agents Related To Recon Tools
+id: 19aa4f58-94ca-45ff-bc34-92e533c0994a
+status: experimental
+description: Detects known suspicious (default) user-agents related to scanning/recon tools
+references:
+  - https://github.com/wpscanteam/wpscan/blob/196fbab5b1ce3870a43515153d4f07878a89d410/lib/wpscan/browser.rb
+  - https://github.com/xmendez/wfuzz/blob/1b695ee9a87d66a7d7bf6cae70d60a33fae51541/docs/user/basicusage.rst
+  - https://github.com/lanmaster53/recon-ng/blob/9e907dfe09fce2997f0301d746796408e01a60b7/recon/core/base.py#L92
+author: Nasreddine Bencherchali (Nextron Systems), Tim Shelton
+date: 2022/07/19
+modified: 2023/01/02
+tags:
+  - attack.initial_access
+  - attack.t1190
+logsource:
+  category: webserver
+detection:
+  selection:
+    cs-user-agent|contains:
+      # Add more tools as you see fit
+      - 'Wfuzz/'
+      - 'WPScan v'
+      - 'Recon-ng/v'
+      - 'GIS - AppSec Team - Project Vision'
+  condition: selection
+falsepositives:
+  - Unknown
+level: medium
diff --git a/src/main/resources/rules/waf/web_xss_in_access_logs.yml b/src/main/resources/rules/waf/web_xss_in_access_logs.yml
@@ -0,0 +1,48 @@
+title: Cross Site Scripting Strings
+id: 65354b83-a2ea-4ea6-8414-3ab38be0d409
+status: experimental
+description: Detects XSS attempts injected via GET requests in access logs
+references:
+  - https://github.com/payloadbox/xss-payload-list
+  - https://portswigger.net/web-security/cross-site-scripting/contexts
+author: Saw Win Naung, Nasreddine Bencherchali
+date: 2021/08/15
+modified: 2022/06/14
+tags:
+  - attack.initial_access
+  - attack.t1189
+logsource:
+  category: webserver
+detection:
+  select_method:
+    cs-method: 'GET'
+  keywords:
+    - '=<script>'
+    - '=%3Cscript%3E'
+    - '=%253Cscript%253E'
+    - '<iframe '
+    - '%3Ciframe '
+    - '<svg '
+    - '%3Csvg '
+    - 'document.cookie'
+    - 'document.domain'
+    - ' onerror='
+    - ' onresize='
+    - ' onload="'
+    - 'onmouseover='
+    - '${alert'
+    - 'javascript:alert'
+    - 'javascript%3Aalert'
+  filter:
+    sc-status: 404
+  condition: select_method and keywords and not filter
+fields:
+  - client_ip
+  - vhost
+  - url
+  - response
+falsepositives:
+  - JavaScripts,CSS Files and PNG files
+  - User searches in search boxes of the respective website
+  - Internal vulnerability scanners can cause some serious FPs when used, if you experience a lot of FPs due to this think of adding more filters such as "User Agent" strings and more response codes
+level: high
diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java
@@ -47,6 +47,7 @@
 
 public class MapperRestApiIT extends SecurityAnalyticsRestTestCase {
 
+    private String matchAllSearchBody = "{\"size\": 1000, \"query\" : {\"match_all\":{}}}";
 
     public void testGetMappingSuccess() throws IOException {
         String testIndexName1 = "my_index_1";
@@ -1526,13 +1527,7 @@ public void testAzureMappings() throws IOException {
         Detector detector = randomDetectorWithInputs(List.of(input), "azure");
         createDetector(detector);
 
-        String request = "{\n" +
-                "   \"size\": 1000,  " +
-                "   \"query\" : {\n" +
-                "     \"match_all\":{}\n" +
-                "   }\n" +
-                "}";
-        List<SearchHit> hits = executeSearch(".opensearch-sap-azure-detectors-queries-000001", request);
+        List<SearchHit> hits = executeSearch(".opensearch-sap-azure-detectors-queries-000001", matchAllSearchBody);
         Assert.assertEquals(60, hits.size());
     }
 
@@ -1557,13 +1552,7 @@ public void testADLDAPMappings() throws IOException {
         Detector detector = randomDetectorWithInputs(List.of(input), "ad_ldap");
         createDetector(detector);
 
-        String request = "{\n" +
-                "   \"size\": 1000,  " +
-                "   \"query\" : {\n" +
-                "     \"match_all\":{}\n" +
-                "   }\n" +
-                "}";
-        List<SearchHit> hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-000001", request);
+        List<SearchHit> hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-000001", matchAllSearchBody);
         Assert.assertEquals(11, hits.size());
     }
 
@@ -1588,13 +1577,7 @@ public void testCloudtrailMappings() throws IOException {
         Detector detector = randomDetectorWithInputs(List.of(input), "cloudtrail");
         createDetector(detector);
 
-        String request = "{\n" +
-                "   \"size\": 1000,  " +
-                "   \"query\" : {\n" +
-                "     \"match_all\":{}\n" +
-                "   }\n" +
-                "}";
-        List<SearchHit> hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-000001", request);
+        List<SearchHit> hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-000001", matchAllSearchBody);
         Assert.assertEquals(32, hits.size());
     }
 
@@ -1619,16 +1602,37 @@ public void testS3Mappings() throws IOException {
         Detector detector = randomDetectorWithInputs(List.of(input), "s3");
         createDetector(detector);
 
-        String request = "{\n" +
-                "   \"size\": 1000,  " +
-                "   \"query\" : {\n" +
-                "     \"match_all\":{}\n" +
-                "   }\n" +
-                "}";
-        List<SearchHit> hits = executeSearch(".opensearch-sap-s3-detectors-queries-000001", request);
+        List<SearchHit> hits = executeSearch(".opensearch-sap-s3-detectors-queries-000001", matchAllSearchBody);
         Assert.assertEquals(1, hits.size());
     }
 
+    public void testWAFMappings() throws IOException {
+        String indexName = "waf-test-index";
+        String sampleDoc = readResource("waf-sample.json");
+
+        createIndex(indexName, Settings.EMPTY);
+        indexDoc(indexName, "1", sampleDoc);
+
+        createMappingsAPI(indexName, "waf");
+
+        Map<String, Object> mappings = getIndexMappingsSAFlat(indexName);
+        assertFalse(mappings.containsKey("timestamp"));     // timestamp field not an alias as it exists in example log
+        assertFalse(mappings.containsKey("waf.request.headers.user_agent")); // no matching field in example log
+        assertTrue(mappings.containsKey("waf.request.method"));
+        assertTrue(mappings.containsKey("waf.request.uri_query"));
+        assertTrue(mappings.containsKey("waf.request.headers.name"));
+        assertTrue(mappings.containsKey("waf.request.headers.value"));
+
+        // Verify that all rules are working
+        DetectorInput input = new DetectorInput("waf detector for security analytics", List.of(indexName), List.of(),
+                getPrePackagedRules("waf").stream().map(DetectorRule::new).collect(Collectors.toList()));
+        Detector detector = randomDetectorWithInputs(List.of(input), "waf");
+        createDetector(detector);
+
+        List<SearchHit> hits = executeSearch(".opensearch-sap-waf-detectors-queries-000001", matchAllSearchBody);
+        Assert.assertEquals(5, hits.size());
+    }
+
     @SuppressWarnings("unchecked")
     private int recurProps(Map<String, Object> props) {
         int totalProps = 0;

diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java
@@ -464,7 +464,7 @@ public void testSearchLogTypes() throws IOException, InterruptedException {
         Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(searchResponse));
 
         Map<String, Object> responseBody = asMap(searchResponse);
-        Assert.assertEquals(23, ((Map<String, Object>) ((Map<String, Object>) responseBody.get("hits")).get("total")).get("value"));
+        Assert.assertEquals(24, ((Map<String, Object>) ((Map<String, Object>) responseBody.get("hits")).get("total")).get("value"));
 
         request = "{\n" +
                 "  \"query\": {\n" +
@@ -892,4 +892,4 @@ public void testCreateACustomLogTypeInvalidCategory() throws IOException {
             makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType));
         });
     }
-}
+}