Skip to content

Commit

Permalink
Add intelligence command to the CLI client (#2864)
Browse files Browse the repository at this point in the history
* adding a intelligence sub command to the Timesketch CLI + documentation

* W0622: Redefining built-in 'type' (redefined-builtin

* Apply suggestions from code review

Co-authored-by: Johan Berggren <[email protected]>

* code review feedback

* Update cli_client/python/timesketch_cli_client/commands/intelligence.py

* Update cli_client/python/timesketch_cli_client/commands/intelligence.py

* Update cli_client/python/timesketch_cli_client/commands/intelligence.py

* s/type/ioc-type

---------

Co-authored-by: Johan Berggren <[email protected]>
  • Loading branch information
jaegeral and berggren authored Oct 4, 2023
1 parent 5b24c50 commit 84865e3
Show file tree
Hide file tree
Showing 4 changed files with 241 additions and 5 deletions.
2 changes: 2 additions & 0 deletions cli_client/python/timesketch_cli_client/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from timesketch_cli_client.commands import analyze
from timesketch_cli_client.commands import config
from timesketch_cli_client.commands import importer
from timesketch_cli_client.commands import intelligence
from timesketch_cli_client.commands import search
from timesketch_cli_client.commands import sketch as sketch_command
from timesketch_cli_client.commands import timelines
Expand Down Expand Up @@ -169,6 +170,7 @@ def cli(ctx, sketch, output):
cli.add_command(importer.importer)
cli.add_command(events.events_group)
cli.add_command(sigma.sigma_group)
cli.add_command(intelligence.intelligence_group)


# pylint: disable=no-value-for-parameter
Expand Down
145 changes: 145 additions & 0 deletions cli_client/python/timesketch_cli_client/commands/intelligence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
# Copyright 2023 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Commands for interacting with intelligence within a Sketch."""

import sys
import json
import click


@click.group("intelligence")
def intelligence_group():
"""Manage intelligence within a sketch."""


@intelligence_group.command("list")
@click.option(
"--header/--no-header",
default=True,
help="Include header in output. (default is to show header))",
)
@click.option(
"--columns",
default="ioc,type",
help="Comma separated list of columns to show. (default: ioc,type)",
)
@click.pass_context
def list_intelligence(ctx, header, columns):
"""List all intelligence.
Args:
ctx: Click context object.
header: Include header in output. (default is to show header)
columns: Comma separated list of columns to show. (default: ioc,type)
Other options: externalURI, tags
"""

if not columns:
columns = "ioc,type"

columns = columns.split(",")

output = ctx.obj.output_format
sketch = ctx.obj.sketch
try:
intelligence = sketch.get_intelligence_attribute()
except ValueError as e:
click.echo(e)
sys.exit(1)

if not intelligence:
click.echo("No intelligence found.")
ctx.exit(1)
if output == "json":
click.echo(json.dumps(intelligence, indent=4, sort_keys=True))
elif output == "text":
if header:
click.echo("\t".join(columns))
for entry in intelligence:
row = []
for column in columns:
if column == "tags":
row.append(",".join(entry.get(column, [])))
else:
row.append(entry.get(column, ""))
click.echo("\t".join(row))
elif output == "csv":
if header:
click.echo(",".join(columns))
for entry in intelligence:
row = []
for column in columns:
if column == "tags":
# Tags can be multiple values but they should only be
# one value on the csv so we join them with a comma
# surrounded the array by quotes
row.append(f'"{",".join(entry.get(column, []))}"')
else:
row.append(entry.get(column, ""))
click.echo(",".join(row))
else:
click.echo(f"Output format {output} not implemented.")


@intelligence_group.command("add")
@click.option(
"--ioc",
required=True,
help="Indicator Of Compromise (IOC) value.",
)
@click.option(
"--ioc-type",
required=False,
help="Type of the intelligence (ipv4, hash_sha256, hash_sha1, hash_md5, other).",
)
@click.option(
"--tags",
required=False,
help="Comma separated list of tags.",
)
@click.pass_context
def add_intelligence(ctx, ioc, tags, ioc_type="other"):
"""Add intelligence to a sketch.
A sketch can have multiple intelligence entries. Each entry consists of
an indicator, a type and a list of tags.
Reference: https://timesketch.org/guides/user/intelligence/
Args:
ctx: Click context object.
ioc: IOC value.
ioc_type: Type of the intelligence. This is defined in the ontology file.
If a string doesn't match any of the aforementioned IOC types,
the type will fall back to other.
tags: Comma separated list of tags.
"""
sketch = ctx.obj.sketch

# Create a tags dict from the comma separated list
if tags:
tags = tags.split(",")
tags = {tag: [] for tag in tags}
else:
tags = []

ioc_dict = {"ioc": ioc, "type": ioc_type, "tags": tags}
# Put the ioc in a nested object to match the format of the API
data = {"data": [ioc_dict]}
try:
sketch.add_attribute(name="intelligence", ontology="intelligence", value=data)
except ValueError as e:
click.echo(e)
sys.exit(1)
click.echo(f"Intelligence added: {ioc}")
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Copyright 2023 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for intelligence command."""

import unittest
import mock

from click.testing import CliRunner

# pylint: disable=import-error
from timesketch_api_client import test_lib as api_test_lib

# pylint: enable=import-error

from .. import test_lib
from .intelligence import intelligence_group


class IntelligenceTest(unittest.TestCase):
"""Test Sigma CLI command."""

@mock.patch("requests.Session", api_test_lib.mock_session)
def setUp(self):
"""Setup test case."""
self.ctx = test_lib.get_cli_context()

def test_list_intelligence(self):
"""Test to list Sigma rules."""
runner = CliRunner()
result = runner.invoke(
intelligence_group,
["list"],
obj=self.ctx,
)
assert 1 is result.exit_code
53 changes: 48 additions & 5 deletions docs/guides/user/cli-client.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ Name: ticket_id3: Ontology: 12345 Value: text

Or as JSON

```
```bash
timesketch --output-format json sketch attributes
{
"intelligence": {
Expand Down Expand Up @@ -283,6 +283,50 @@ To remove an attribute from a sketch
timesketch sketch remove_attribute
```

## Intelligence

Intelligence is always sketch specific. The same can be achieved using
`timesketch attributes` command, but then the ontology type and data needs
to be provided in the correct format.

Running `timesketch intelligence list` will show the intelligence added to a
sketch (if sketch id is set in the config file)

The putput format can also be changed as follows

```bash
timesketch --sketch 2 --output-format text intelligence list --columns ioc,externalURI,tags,type
ioc externalURI tags type
1.2.3.4 google.com foo ipv4
3.3.3.3 fobar.com aaaa ipv4
```

Or as CSV

```bash
timesketch --sketch 2 --output-format csv intelligence list --columns ioc,externalURI,tags,type
ioc,externalURI,tags,type
1.2.3.4,google.com,"foo",ipv4
3.3.3.3,fobar.com,"aaaa",ipv4
aaaa.com,foobar.de,"foo,aaaa",hostname
```

### Adding Intelligence

Adding an indicator works as following

```bash
timesketch --sketch 2 intelligence add --ioc 8.8.4.4 --type ipv4 --tags foo,bar,ext
```

### Removing all of Intelligence

To remove all intelligence indicators, run:

```bash
timesketch --sketch 2 --output-format text sketch attributes remove --name intelligence --ontology intelligence
Attribute removed: Name: intelligence Ontology: intelligence
```

## Run analyzers

Expand Down Expand Up @@ -343,7 +387,6 @@ Running analyzer [domain] on [timeline 1]:
..
Results
[domain] = 217 domains discovered (150 TLDs) and 1 known CDN networks found.

```

### List analyzer results
Expand All @@ -353,7 +396,7 @@ That can be done with `timesketch analyzer results`.

It can show only the analyzer results directly:

```
```bash
timesketch --output-format text analyze results --analyzer account_finder --timeline 3
Results for analyzer [account_finder] on [sigma_events]:
SUCCESS - NOTE - Account finder was unable to extract any accounts.
Expand Down Expand Up @@ -396,7 +439,6 @@ Dependent: DONE - None - Feature extraction [ssh_failed_username] extracted 0 fe
Dependent: DONE - None - Feature extraction [win_login_subject_domain] extracted 0 features.
Dependent: DONE - None - Feature extraction [win_login_subject_logon_id] extracted 0 features.
Dependent: DONE - None - Feature extraction [win_login_username] extracted 0 features.

```

To get a result in `json` that can be piped into other CLI tools run something
Expand Down Expand Up @@ -440,7 +482,8 @@ This new feature makes it easy to add events to Timesketch from the command line

It can also be called with a output format `json` like following.

```timesketch --output-format json events add --message "foobar-message" --date 2023-03-04T11:31:12 --timestamp-desc "test"
```bash
timesketch --output-format json events add --message "foobar-message" --date 2023-03-04T11:31:12 --timestamp-desc "test"
{'meta': {}, 'objects': [{'color': 'F37991', 'created_at': '2023-03-08T12:46:24.472587', 'datasources': [], 'deleted': None, 'description': 'internal timeline for user-created events', 'id': 19, 'label_string': '', 'name': 'Manual events', 'searchindex': {'created_at': '2023-03-08T12:46:24.047640', 'deleted': None, 'description': 'internal timeline for user-created events', 'id': 9, 'index_name': '49a318b0ba17867fd71b50903774a0c8', 'label_string': '', 'name': 'Manual events', 'status': [{'created_at': '2023-03-17T09:35:03.202520', 'id': 87, 'status': 'ready', 'updated_at': '2023-03-17T09:35:03.202520'}], 'updated_at': '2023-03-08T12:46:24.047640', 'user': {'active': True, 'admin': True, 'groups': [], 'username': 'dev'}}, 'status': [{'created_at': '2023-03-17T09:35:03.233973', 'id': 79, 'status': 'ready', 'updated_at': '2023-03-17T09:35:03.233973'}], 'updated_at': '2023-03-08T12:46:24.472587', 'user': {'active': True, 'admin': True, 'groups': [], 'username': 'dev'}}]}
Event added to sketch: timefocus test
```
Expand Down

0 comments on commit 84865e3

Please sign in to comment.