Add support for Gobuster tool

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
             require_serial: true
             verbose: true
     - repo: https://github.com/Yelp/detect-secrets
-      rev: v1.3.0
+      rev: v1.4.0
       hooks:
           - id: detect-secrets
             args: ["--baseline", ".secrets.baseline"]

.secrets.baseline

@@ -1,5 +1,5 @@
 {
-  "version": "1.3.0",
+  "version": "1.4.0",
   "plugins_used": [
     {
       "name": "ArtifactoryDetector"
@@ -192,7 +192,7 @@
         "filename": "rekono/findings/enums.py",
         "hashed_secret": "8be3c943b1609fffbfc51aad666d0a04adf83c9d",
         "is_verified": false,
-        "line_number": 24,
+        "line_number": 25,
         "is_secret": false
       }
     ],
@@ -784,7 +784,7 @@
         "filename": "rekono/testing/executions/test_base_tool.py",
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
         "is_verified": false,
-        "line_number": 311,
+        "line_number": 312,
         "is_secret": false
       }
     ],
@@ -849,5 +849,5 @@
       }
     ]
   },
-  "generated_at": "2022-12-31T14:34:49Z"
+  "generated_at": "2023-01-05T19:32:50Z"
 }

README.md

@@ -59,6 +59,7 @@ Why not automate this process and focus on find vulnerabilities using your skill
 - [SSH Audit](https://github.com/jtesta/ssh-audit)
 - [SMBMap](https://github.com/ShawnDEvans/smbmap)
 - [Dirsearch](https://github.com/maurosoria/dirsearch)
+- [Gobuster](https://github.com/OJ/gobuster)
 - [GitLeaks](https://github.com/zricethezav/gitleaks) & [GitDumper](https://github.com/internetwache/GitTools/tree/master/Dumper)
 - [Log4j Scan](https://github.com/fullhunt/log4j-scan)
 - [Spring4Shell Scan](https://github.com/fullhunt/spring4shell-scan)

docker/kali/Dockerfile

@@ -16,7 +16,7 @@ RUN ln -s /usr/bin/python3 /usr/bin/python
 RUN pip install --upgrade pip
 RUN pip install -r /code/requirements.txt
 
-RUN apt install nmap dirsearch theharvester nikto sslscan sslyze cmseek zaproxy exploitdb metasploit-framework emailharvester joomscan gitleaks smbmap nuclei -y
+RUN apt install nmap dirsearch theharvester nikto sslscan sslyze cmseek zaproxy exploitdb metasploit-framework emailharvester joomscan gitleaks smbmap nuclei gobuster -y
 RUN setcap cap_net_raw,cap_net_admin,cap_net_bind_service+eip $(which nmap)
 RUN git clone https://github.com/fullhunt/log4j-scan /opt/log4j-scan
 RUN git clone https://github.com/fullhunt/spring4shell-scan.git /opt/spring4shell-scan
@@ -25,8 +25,7 @@ RUN pip install -r /opt/log4j-scan/requirements.txt
 RUN pip install -r /opt/spring4shell-scan/requirements.txt
 RUN pip install emailfinder ssh-audit
 
-RUN apt install wordlists seclists dirb -y
-RUN gzip -d /usr/share/wordlists/rockyou.txt.gz
+RUN apt install seclists dirb -y
 
 RUN adduser --disabled-password rekono
 RUN chown -R rekono:rekono /code

rekono/executions/filters.py

@@ -1,5 +1,6 @@
 from django_filters.rest_framework import FilterSet
 from django_filters.rest_framework.filters import OrderingFilter
+
 from executions.models import Execution
 
 
@@ -37,9 +38,9 @@ class Meta:
             'task__executor__username': ['exact', 'icontains'],
             'tool': ['exact'],
             'tool__name': ['exact', 'icontains'],
-            'tool__stage': ['exact'],
             'configuration': ['exact'],
             'configuration__name': ['exact', 'icontains'],
+            'configuration__stage': ['exact'],
             'status': ['exact'],
             'start': ['gte', 'lte', 'exact'],
             'end': ['gte', 'lte', 'exact']

rekono/findings/enums.py

@@ -16,6 +16,7 @@ class DataType(models.TextChoices):
 
     IP = 'IP'
     DOMAIN = 'Domain'
+    VHOST = 'VHOST'
     URL = 'Url'
     EMAIL = 'Email'
     LINK = 'Link'

rekono/findings/filters.py

@@ -4,6 +4,7 @@
 from django.db.models import QuerySet
 from django_filters.rest_framework import filters
 from django_filters.rest_framework.filters import OrderingFilter
+
 from findings.enums import OSType
 from findings.models import (OSINT, Credential, Exploit, Host, Path, Port,
                              Technology, Vulnerability)
@@ -33,7 +34,6 @@
     'executions__end': ['gte', 'lte', 'exact'],
     'detected_by': ['exact'],
     'detected_by__name': ['exact', 'icontains'],
-    'detected_by__stage': ['exact'],
     'first_seen': ['gte', 'lte', 'exact'],
     'last_seen': ['gte', 'lte', 'exact'],
     'is_active': ['exact'],
@@ -162,7 +162,9 @@ class PortFilter(FindingFilter):
     '''FilterSet to filter and sort Port entities.'''
 
     # Ordering fields including common ones
-    o = OrderingFilter(fields=FINDING_ORDERING + (('host__os_type', 'os_type'), 'host', 'port', 'protocol', 'service'))
+    o = OrderingFilter(
+        fields=FINDING_ORDERING + (('host__os_type', 'os_type'), 'host', 'port', 'protocol', 'service', 'status')
+    )
 
     class Meta:
         '''FilterSet metadata.'''
@@ -184,7 +186,7 @@ class PathFilter(FindingFilter):
     '''FilterSet to filter and sort Path entities.'''
 
     # Ordering fields including common ones
-    o = OrderingFilter(fields=FINDING_ORDERING + (('port__host', 'host'), 'port', 'path', 'status'))
+    o = OrderingFilter(fields=FINDING_ORDERING + (('port__host', 'host'), 'port', 'path', 'status', 'type'))
 
     class Meta:
         '''FilterSet metadata.'''

rekono/findings/migrations/0002_alter_osint_data_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2023-01-05 15:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('findings', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='osint',
+            name='data_type',
+            field=models.TextField(choices=[('IP', 'Ip'), ('Domain', 'Domain'), ('VHOST', 'Vhost'), ('Url', 'Url'), ('Email', 'Email'), ('Link', 'Link'), ('ASN', 'Asn'), ('Username', 'User'), ('Password', 'Password')], max_length=10),
+        ),
+    ]

rekono/findings/models.py

@@ -165,8 +165,12 @@ def filter(self, input: Input) -> bool:
         if not input.filter:
             return True
         try:
-            # Filter by address type
-            return cast(models.TextChoices, TargetType)[input.filter] == get_target_type(self.address)
+            distinct = input.filter[0] == '!'
+            filter_types = [
+                cast(models.TextChoices, TargetType)[f.upper()] for f in input.filter.replace('!', '').split(',s')
+            ]
+            host_type = get_target_type(self.address)
+            return host_type not in filter_types if distinct else host_type in filter_types
         except KeyError:
             return True
 

rekono/frontend/src/backend/RekonoApi.vue

@@ -78,7 +78,7 @@ export default {
       cancellableStatuses: ['Requested', 'Running'],
       timeUnits: ['Weeks', 'Days', 'Hours', 'Minutes'],
       authenticationTypes: ['None', 'Basic', 'Bearer', 'Cookie', 'Digest', 'JWT', 'NTLM'],
-      wordlistTypes: ['Endpoint'],
+      wordlistTypes: ['Endpoint', 'Subdomain'],
       nameRegex: /^[\wÀ-ÿ\s.\-[\]()]{0,100}$/,
       textRegex: /^[\wÀ-ÿ\s.:,+\-'"?¿¡!#%$€[\]()]{0,300}$/,
       cveRegex: /^CVE-\d{4}-\d{1,7}$/,

rekono/frontend/src/common/TableHeader.vue

@@ -31,12 +31,13 @@
         <b-col v-for="ft in filters" :key="ft.name">
           <b-form-group :description="ft.name">
             <b-form-tags :ref="ft.name.toLowerCase().replace(' ', '_')" no-outer-focus v-if="!ft.values && ft.type === 'tags'" :value="$route.query[ft.name.toLowerCase().replace(' ', '_')] ? $route.query[ft.name.toLowerCase().replace(' ', '_')].split(',') : null" placeholder="" remove-on-delete size="md" tag-variant="dark" @input="addFilter(ft.name, ft.filterField, $event)"/>
-            <b-form-input :ref="ft.name.toLowerCase().replace(' ', '_')" v-if="!ft.values && ft.type !== 'tags'" :value="$route.query[ft.name.toLowerCase().replace(' ', '_')] ? $route.query[ft.name.toLowerCase().replace(' ', '_')] : null" :type="ft.type" @input="addFilter(ft.name, ft.filterField, $event)"/>
+            <b-form-input :ref="ft.name.toLowerCase().replace(' ', '_')" v-if="!ft.values && ft.type !== 'tags' && ft.type !== 'checkbox'" :value="$route.query[ft.name.toLowerCase().replace(' ', '_')] ? $route.query[ft.name.toLowerCase().replace(' ', '_')] : null" :type="ft.type" @input="addFilter(ft.name, ft.filterField, $event)"/>
             <b-form-select :ref="ft.name.toLowerCase().replace(' ', '_')" v-if="ft.values" :value="$route.query[ft.name.toLowerCase().replace(' ', '_')] ? $route.query[ft.name.toLowerCase().replace(' ', '_')] : ft.default" :options="ft.values" :value-field="ft.valueField" :text-field="ft.textField" @change="addFilter(ft.name, ft.filterField, $event)">
               <template #first>
                 <b-form-select-option :value="null">Select {{ ft.name.toLowerCase().replace(' ', '_') }}</b-form-select-option>
               </template>
             </b-form-select>
+            <b-form-checkbox :ref="ft.name.toLowerCase().replace(' ', '_')" v-if="ft.type === 'checkbox'" @change="addFilter(ft.name, ft.filterField, $event)"/>
           </b-form-group>
         </b-col>
       </b-row>

rekono/frontend/src/components/Processes.vue

@@ -107,7 +107,7 @@ export default {
         { key: 'icon', label: '', sortable: false },
         { key: 'tool.name', label: 'Tool', sortable: true },
         { key: 'configuration.name', label: 'Configuration', sortable: true },
-        { key: 'tool.stage_name', label: 'Stage', sortable: true },
+        { key: 'configuration.stage_name', label: 'Stage', sortable: true },
         { key: 'priority', sortable: true },
         { key: 'actions', sortable: false }
       ],
@@ -132,9 +132,9 @@ export default {
     data () {
       this.filters = [
         { name: 'Tags', filterField: 'tags__name__in', type: 'tags' },
-        { name: 'Stage', values: this.stages, valueField: 'id', textField: 'value', filterField: 'steps__tool__stage' },
+        { name: 'Stage', values: this.stages, valueField: 'id', textField: 'value', filterField: 'steps__configuration__stage' },
         { name: 'Creator', filterField: 'creator__username__icontains', type: 'text' },
-        { name: 'Favourities', values: [{ value: true, text: 'True' }, { value: false, text: 'False' }], valueField: 'value', textField: 'text', filterField: 'liked' }
+        { name: 'Favourities', type: 'checkbox', filterField: 'liked' }
       ]
     }
   },

rekono/frontend/src/components/Tools.vue

@@ -8,6 +8,9 @@
           <b-img v-if="!row.item.icon" src="favicon.ico"/>
         </b-link>
       </template>
+      <template #cell(stages)="row">
+        <p>{{ Array.from(new Set(row.item.configurations.map(configuration => { return configuration.stage_name }))).join(', ') }}</p>
+      </template>
       <template #cell(intensities)="row">
         <div v-for="base in intensityByVariant" v-bind:key="base.intensity_rank" class="d-inline">
           <div v-for="i in row.item.intensities" v-bind:key="i.intensity_rank" class="d-inline">
@@ -16,7 +19,7 @@
         </div>
       </template>
       <template #cell(inputs)="row">
-        <p>{{ row.item.arguments.map(argument => { return argument.inputs.find(input => input.order === 1).type.name }).join(', ') }}</p>
+        <p>{{ Array.from(new Set(row.item.arguments.map(argument => { return argument.inputs.find(input => input.order === 1).type.name }))).join(', ') }}</p>
       </template>
       <template #cell(likes)="row">
         {{ row.item.likes }}
@@ -79,9 +82,9 @@ export default {
         { key: 'icon', label: '', sortable: false },
         { key: 'name', label: 'Tool', sortable: true },
         { key: 'command', sortable: true },
-        { key: 'stage_name', label: 'Stage', sortable: true },
+        { key: 'stages', label: 'Stages', sortable: false },
         { key: 'intensities', sortable: true },
-        { key: 'inputs', sortable: true },
+        { key: 'inputs', sortable: false },
         { key: 'likes', sortable: true },
         { key: 'actions', sortable: false }
       ],
@@ -107,16 +110,20 @@ export default {
   watch: {
     data () {
       this.filters = [
-        { name: 'Stage', values: this.stages, valueField: 'id', textField: 'value', filterField: 'stage' },
+        { name: 'Stage', values: this.stages, valueField: 'id', textField: 'value', filterField: 'configurations__stage' },
         { name: 'Input', values: this.inputTypes, valueField: 'value', textField: 'value', filterField: 'arguments__inputs__type__name' },
         { name: 'Output', values: this.inputTypes, valueField: 'value', textField: 'value', filterField: 'configurations__outputs__type__name' },
-        { name: 'Favourities', values: [{ value: true, text: 'True' }, { value: false, text: 'False' }], valueField: 'value', textField: 'text', filterField: 'liked' }
+        { name: 'Favourities', type: 'checkbox', filterField: 'liked' }
       ] 
     }
   },
   methods: {
     fetchData (params = null) {
-      return this.getOnePage('/api/tools/?o=stage,name', params)
+      if (!params) {
+        params = {}
+      }
+      params['o'] = 'configurations__stage,name'
+      return this.getOnePage('/api/tools/', params)
         .then(response => {
           this.data = response.data.results
           this.total = response.data.count

rekono/frontend/src/components/Wordlists.vue

@@ -67,10 +67,10 @@ export default {
   watch: {
     data () {
       this.filters = [
-        { name: 'Type', values: ['Endpoint', 'Password'], valueField: 'value', textField: 'value', filterField: 'type' },
+        { name: 'Type', values: ['Endpoint', 'Subdomain'], valueField: 'value', textField: 'value', filterField: 'type' },
         { name: 'Max Size', filterField: 'size__lte', type: 'number' },
         { name: 'Creator', filterField: 'creator__username__icontains', type: 'text' },
-        { name: 'Favourities', values: [{ value: true, text: 'True' }, { value: false, text: 'False' }], valueField: 'value', textField: 'text', filterField: 'liked' }
+        { name: 'Favourities', type: 'checkbox', filterField: 'liked' }
       ] 
     }
   },

rekono/frontend/src/components/findings/Finding.vue

@@ -153,7 +153,7 @@ export default {
     getFilter () {
       let filter = {}
       if (this.order) {
-        filter.order = this.order
+        filter.o = this.order
       }
       if (this.execution) {
         filter.executions = this.execution
@@ -178,15 +178,17 @@ export default {
         filter = this.getFilter()
       }
       filter.severity = this.severities[index]
-      this.getAllPages('/api/vulnerabilities/', filter).then(results => {
-        this.findings.push(...results)
-        index += 1
-        if (index < this.severities.length) {
-          this.fetchVulnerabilities(filter, index)
-        } else {
-          this.$emit('end')
-        }
-      })
+      this.getAllPages('/api/vulnerabilities/', filter)
+        .then(results => {
+          this.findings.push(...results)
+          index += 1
+          if (index < this.severities.length) {
+            this.fetchVulnerabilities(filter, index)
+          } else {
+            this.$emit('end')
+          }
+        })
+        .catch(() => this.$emit('end'))
     },
     fetchData () {
       if (this.types && this.types.length > 0 && !this.types.includes(this.name.toLowerCase())) {
@@ -200,6 +202,7 @@ export default {
               this.findings = results
               this.$emit('end')
             })
+            .catch(() => this.$emit('end'))
         } else {
           this.fetchVulnerabilities()
         }

rekono/frontend/src/components/findings/Findings.vue

@@ -203,7 +203,7 @@ export default {
   methods: {
     fetchTargets () {
       if (!this.task) {
-        this.getAllPages('/api/targets/', { project: this.$route.params.id, order: 'target' })
+        this.getAllPages('/api/targets/', { project: this.$route.params.id, o: 'target' })
           .then(results => {
             this.targets = results
             if (results.length > 0) {

rekono/frontend/src/modals/ProjectMember.vue

@@ -38,7 +38,7 @@ export default {
   },
   methods: {
     fetchUsers () {
-      this.getAllPages('/api/users/', { project__ne: this.projectId, order: 'username' }).then(results => this.users = results)
+      this.getAllPages('/api/users/', { project__ne: this.projectId, o: 'username' }).then(results => this.users = results)
     },
     check () {
       this.memberState = (this.member !== null)

rekono/frontend/src/modals/Step.vue

@@ -99,7 +99,7 @@ export default {
     initialized (initialized) {
       if (initialized) {
         if (!this.step && !this.tool) {
-          this.getAllPages('/api/tools/', { order: 'stage,name'}).then(results => this.tools = results)
+          this.getAllPages('/api/tools/', { o: 'configurations__stage,name'}).then(results => this.tools = results)
         } else if (this.step && !this.tool) {
           this.priority = this.step.priority
           this.step.tool.configurations = [this.step.configuration]
@@ -110,7 +110,7 @@ export default {
           this.selectTool(this.tool.id, this.tool)
         }
         if (!this.process) {
-          let filter = { order: 'name' }
+          let filter = { o: 'name' }
           if (this.$store.state.role !== 'Admin') {
             filter.creator = this.$store.state.user
           }

rekono/frontend/src/modals/Task.vue

@@ -49,7 +49,9 @@
             <b-icon icon="file-earmark-word-fill"/> Wordlists
           </template>
           <b-form-group description="Select wordlists to use">
-            <b-form-select v-model="wordlistsItems" :options="wordlists" multiple value-field="id" text-field="name" :select-size="10"/>
+            <b-form-select v-model="wordlistsItems" multiple :select-size="10">
+              <option v-for="wordlist in wordlists" :key="wordlist.id" :value="wordlist.id">{{ wordlist.name }} - {{ wordlist.type }}</option>
+            </b-form-select>
           </b-form-group>
         </b-tab>
         <b-tab title-link-class="text-secondary">
@@ -172,16 +174,16 @@ export default {
         } else if (this.process) {
           this.selectProcess(this.process.id, this.process)
         } else if (!this.tool && !this.process) {
-          this.getAllPages('/api/tools/', { order: 'stage,name'}).then(results => this.tools = results)
-          this.getAllPages('/api/processes/', { order: 'name' }).then(results => this.processes = results)
+          this.getAllPages('/api/tools/', { o: 'configurations__stage,name'}).then(results => this.tools = results)
+          this.getAllPages('/api/processes/', { o: 'name' }).then(results => this.processes = results)
         }
         if (this.target) {
           this.targetIds = [this.target.id]
           this.get(`/api/projects/${this.target.project}/`).then(response => { this.selectProject(this.target.project, response.data) })
         } else if (this.project) {
           this.get(`/api/projects/${this.project.id}/`).then(response => { this.selectProject(this.project.id, response.data) })
         } else {
-          this.getAllPages('/api/projects/', { order: 'name' }).then(results => this.projects = results)
+          this.getAllPages('/api/projects/', { o: 'name' }).then(results => this.projects = results)
         }
       }
     }
@@ -352,7 +354,7 @@ export default {
       }
     },
     updateWordlists () {
-      this.getAllPages('/api/resources/wordlists/', { order: 'type,name' }).then(results => this.wordlists = results)
+      this.getAllPages('/api/resources/wordlists/', { o: 'type,name' }).then(results => this.wordlists = results)
     },
     cleanScheduledIn () {
       this.scheduledIn = null