Skip to content

Commit

Permalink
sumologic: use milliseconds since epoch for timestamps
Browse files Browse the repository at this point in the history
  • Loading branch information
@pcoccoli
pcoccoli committed Mar 19, 2024
1 parent 97378da commit 25fe3c0
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 53 deletions.
28 changes: 17 additions & 11 deletions stix_shifter_modules/sumologic/stix_translation/query_constructor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,26 +224,32 @@ def parse_expression(self, pattern: Pattern):

def translate_pattern(pattern: Pattern, data_model_mapping, options):
query = QueryStringPatternTranslator(pattern, data_model_mapping).translated
query = re.sub("START", "START ", query)
query = re.sub("STOP", " STOP ", query)
query, from_time, to_time = convert_timestamp(query)
query_dict = {"query": query.replace("'", "\""), "fromTime": from_time, "toTime": to_time}
query_str = json.dumps(query_dict)
return [query_str]


def to_datetime(value):
if '.' in value:
time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ'
else:
time_pattern = '%Y-%m-%dT%H:%M:%SZ'
return datetime.datetime.strptime(value, time_pattern).replace(tzinfo=datetime.timezone.utc)


def to_epoch_ms(value):
return int(value.timestamp() * 1000)


def convert_timestamp(query):
if ('START' and 'STOP') in query:
x = re.search('(.*)(?= START )(.*)(?<=STOP )(.*)', query)
query = x.group(1)
from_time = x.group(2).replace(' START ', "").replace(" STOP ", "")
to_time = x.group(3)
from_time = datetime.datetime.strptime(from_time, "t'%Y-%m-%dT%H:%M:%S.%fZ'").strftime("%Y%m%dT%H%M%S")
to_time = datetime.datetime.strptime(to_time, "t'%Y-%m-%dT%H:%M:%S.%fZ'").strftime("%Y%m%dT%H%M%S")
x = re.search(r"(.*)\s*START\s*t'(.*)'\s*STOP\s*t'(.*)'", query)
query = x.group(1).strip(' ')
from_time = to_datetime(x.group(2))
to_time = to_datetime(x.group(3))
else:
to_time = datetime.datetime.utcnow()
from_time = (to_time - datetime.timedelta(minutes=15))
to_time = to_time.strftime("%Y%m%dT%H%M%S")
from_time = from_time.strftime("%Y%m%dT%H%M%S")

return query, from_time, to_time
return query, to_epoch_ms(from_time), to_epoch_ms(to_time)
99 changes: 57 additions & 42 deletions stix_shifter_modules/sumologic/tests/stix_translation/test_sumologic_stix_to_query.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import json
import unittest
from stix_shifter_modules.sumologic.stix_translation import query_constructor

Expand All @@ -10,150 +11,164 @@ def _test_query_assertions(query, queries):
assert query['queries'] == [queries]


def _timestamp_eq(lhs, rhs):
return lhs - rhs <= 1000


# Handle test race condition when no timestamp is given in pattern
def _test_query_assertions_no_timestamp(query, queries):
for i, q in enumerate(query['queries']):
actual_query = json.loads(q)
expected = json.loads(queries)
assert actual_query['query'] == expected['query']
assert _timestamp_eq(actual_query['fromTime'], expected['fromTime'])
assert _timestamp_eq(actual_query['toTime'], expected['toTime'])


class TestStixToQuery(unittest.TestCase, object):
def test_artifact_payload_query(self):
stix_pattern = "[artifact:payload_bin = 'Sep 26 05:29:06 sumologic NetworkManager[677]: <info>" \
" [1632614346.1076] dhcp4 (eth0)']"
query = translation.translate('sumologic', 'query', 'sumologic', stix_pattern, options={"result_limit": 100})
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"_raw = \\\"Sep 26 05:29:06 sumologic NetworkManager[677]: <info> [1632614346.1076] " \
"dhcp4 (eth0)\\\"\", \"fromTime\": \"%s\", \"toTime\": \"%s\"}" % (from_time, to_time)
_test_query_assertions(query, queries)
"dhcp4 (eth0)\\\"\", \"fromTime\": %s, \"toTime\": %s}" % (from_time, to_time)
_test_query_assertions_no_timestamp(query, queries)

def test_domain_and_userid_query(self):
stix_pattern = "[domain-name:value = 'sumologic.domain_name.com' AND user-account:user_id = '12345678'] " \
"START t'2021-09-01T00:00:00.000Z' STOP t'2021-09-26T10:16:00.000Z'"
query = translation.translate('sumologic', 'query', 'sumologic', stix_pattern, options={"result_limit": 100})
queries = "{\"query\": \"id = \\\"12345678\\\" AND _sourcehost = \\\"sumologic.domain_name.com\\\"\", " \
"\"fromTime\": \"20210901T000000\", \"toTime\": \"20210926T101600\"}"
"\"fromTime\": 1630454400000, \"toTime\": 1632651360000}"
_test_query_assertions(query, queries)

def test_domain_and_userid_query_no_timestamp(self):
stix_pattern = "[domain-name:value = 'sumologic.domain_name.com' AND user-account:user_id = '12345678']"
query = translation.translate('sumologic', 'query', 'sumologic', stix_pattern, options={"result_limit": 100})
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"id = \\\"12345678\\\" AND _sourcehost = \\\"sumologic.domain_name.com\\\"\", " \
"\"fromTime\": \"%s\", \"toTime\": \"%s\"}" % (from_time, to_time)
_test_query_assertions(query, queries)
"\"fromTime\": %s, \"toTime\": %s}" % (from_time, to_time)
_test_query_assertions_no_timestamp(query, queries)

def test_event_count_query(self):
stix_pattern = "[x-ibm-finding:event_count = '21'] START t'2021-01-28T12:24:01.009Z' " \
"STOP t'2021-07-25T12:54:01.009Z'"
query = translation.translate('sumologic', 'query', 'sumologic', stix_pattern)
queries = "{\"query\": \"_messagecount = \\\"21\\\"\", \"fromTime\": \"20210128T122401\", " \
"\"toTime\": \"20210725T125401\"}"
queries = "{\"query\": \"_messagecount = \\\"21\\\"\", \"fromTime\": 1611836641009, " \
"\"toTime\": 1627217641009}"
_test_query_assertions(query, queries)

def test_time_observed_query(self):
stix_pattern = "[x-ibm-finding:time_observed = '2021-09-23T11:34:07.255Z']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"_messagetime = \\\"2021-09-23T11:34:07.255Z\\\"\", \"fromTime\": \"%s\", " \
"\"toTime\": \"%s\"}" % (from_time, to_time)
_test_query_assertions(query, queries)
queries = "{\"query\": \"_messagetime = \\\"2021-09-23T11:34:07.255Z\\\"\", \"fromTime\": %s, " \
"\"toTime\": %s}" % (from_time, to_time)
_test_query_assertions_no_timestamp(query, queries)

def test_src_device_query(self):
stix_pattern = "[x-ibm-finding:src_device = 'sumologic.domain.com']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"_collector = \\\"sumologic.domain.com\\\"\", \"fromTime\": \"%s\", " \
"\"toTime\": \"%s\"}" % (from_time, to_time)
_test_query_assertions(query, queries)
queries = "{\"query\": \"_collector = \\\"sumologic.domain.com\\\"\", \"fromTime\": %s, " \
"\"toTime\": %s}" % (from_time, to_time)
_test_query_assertions_no_timestamp(query, queries)

def test_event_code_query(self):
stix_pattern = "[x-oca-event:code = '12345678']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"_messageid = \\\"12345678\\\"\", \"fromTime\": \"%s\", " \
"\"toTime\": \"%s\"}" % (from_time, to_time)
_test_query_assertions(query, queries)
queries = "{\"query\": \"_messageid = \\\"12345678\\\"\", \"fromTime\": %s, " \
"\"toTime\": %s}" % (from_time, to_time)
_test_query_assertions_no_timestamp(query, queries)

def test_event_agent_query(self):
stix_pattern = "[x-oca-event:agent='sumologic.domain.com']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"_collector = \\\"sumologic.domain.com\\\"\", \"fromTime\": \"%s\", " \
"\"toTime\": \"%s\"}" \
queries = "{\"query\": \"_collector = \\\"sumologic.domain.com\\\"\", \"fromTime\": %s, " \
"\"toTime\": %s}" \
% (from_time, to_time)
_test_query_assertions(query, queries)
_test_query_assertions_no_timestamp(query, queries)

def test_event_module_query(self):
stix_pattern = "[x-oca-event:module = 'Linux System_3']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"_source = \\\"Linux System_3\\\"\", \"fromTime\": \"%s\", " \
"\"toTime\": \"%s\"}" % (from_time, to_time)
_test_query_assertions(query, queries)
queries = "{\"query\": \"_source = \\\"Linux System_3\\\"\", \"fromTime\": %s, " \
"\"toTime\": %s}" % (from_time, to_time)
_test_query_assertions_no_timestamp(query, queries)

def test_event_provider_query(self):
stix_pattern = "[x-oca-event:provider = 'linux/system']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"_sourcecategory = \\\"linux/system\\\"\", \"fromTime\": \"%s\", \"toTime\": \"%s\"}" \
queries = "{\"query\": \"_sourcecategory = \\\"linux/system\\\"\", \"fromTime\": %s, \"toTime\": %s}" \
% (from_time, to_time)
_test_query_assertions(query, queries)
_test_query_assertions_no_timestamp(query, queries)

def test_custom_collector_id_query(self):
stix_pattern = "[x-sumologic-source:collectorid = '12345678']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"_collectorid = \\\"12345678\\\"\", \"fromTime\": \"%s\", \"toTime\": \"%s\"}" \
queries = "{\"query\": \"_collectorid = \\\"12345678\\\"\", \"fromTime\": %s, \"toTime\": %s}" \
% (from_time, to_time)
_test_query_assertions(query, queries)
_test_query_assertions_no_timestamp(query, queries)

def test_custom_sourcename_query(self):
stix_pattern = "[x-sumologic-source:sourcename = '/var/log/messages']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"_sourcename = \\\"/var/log/messages\\\"\", \"fromTime\": \"%s\", \"toTime\": \"%s\"}" \
queries = "{\"query\": \"_sourcename = \\\"/var/log/messages\\\"\", \"fromTime\": %s, \"toTime\": %s}" \
% (from_time, to_time)
_test_query_assertions(query, queries)
_test_query_assertions_no_timestamp(query, queries)

def test_user_id_query(self):
stix_pattern = "[user-account:user_id = '12345678']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"id = \\\"12345678\\\"\", \"fromTime\": \"%s\", \"toTime\": \"%s\"}" \
queries = "{\"query\": \"id = \\\"12345678\\\"\", \"fromTime\": %s, \"toTime\": %s}" \
% (from_time, to_time)
_test_query_assertions(query, queries)
_test_query_assertions_no_timestamp(query, queries)

def test_user_account_login_query(self):
stix_pattern = "[user-account:account_login = '[email protected]']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"email = \\\"[email protected]\\\"\", \"fromTime\": \"%s\", \"toTime\": \"%s\"}" \
queries = "{\"query\": \"email = \\\"[email protected]\\\"\", \"fromTime\": %s, \"toTime\": %s}" \
% (from_time, to_time)
_test_query_assertions(query, queries)
_test_query_assertions_no_timestamp(query, queries)

def test_user_display_name_query(self):
stix_pattern = "[user-account:display_name = 'abc def']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"displayName = \\\"abc def\\\"\", \"fromTime\": \"%s\", \"toTime\": \"%s\"}" \
queries = "{\"query\": \"displayName = \\\"abc def\\\"\", \"fromTime\": %s, \"toTime\": %s}" \
% (from_time, to_time)
_test_query_assertions(query, queries)
_test_query_assertions_no_timestamp(query, queries)

def test_user_account_created_query(self):
stix_pattern = "[user-account:account_created = '2021-09-23T11:34:07.255Z']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"createdAt = \\\"2021-09-23T11:34:07.255Z\\\"\", \"fromTime\": \"%s\", " \
"\"toTime\": \"%s\"}" % (from_time, to_time)
_test_query_assertions(query, queries)
queries = "{\"query\": \"createdAt = \\\"2021-09-23T11:34:07.255Z\\\"\", \"fromTime\": %s, " \
"\"toTime\": %s}" % (from_time, to_time)
_test_query_assertions_no_timestamp(query, queries)

def test_in_operator(self):
stix_pattern = "[user-account:display_name IN ('abc', 'def')]"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"displayName = \\\"abc\\\" OR displayName = \\\"def\\\"\", " \
"\"fromTime\": \"%s\", \"toTime\": \"%s\"}" \
"\"fromTime\": %s, \"toTime\": %s}" \
% (from_time, to_time)
_test_query_assertions(query, queries)
_test_query_assertions_no_timestamp(query, queries)

def test_user_account_last_login_query(self):
stix_pattern = "[user-account:account_last_login = '2021-10-04T13:51:09.958Z']"
query = translation.translate('sumologic', 'query', '{}', stix_pattern)
_, from_time, to_time = query_constructor.convert_timestamp(query)
queries = "{\"query\": \"lastLoginTimestamp = \\\"2021-10-04T13:51:09.958Z\\\"\", \"fromTime\": \"%s\", " \
"\"toTime\": \"%s\"}" % (from_time, to_time)
_test_query_assertions(query, queries)
queries = "{\"query\": \"lastLoginTimestamp = \\\"2021-10-04T13:51:09.958Z\\\"\", \"fromTime\": %s, " \
"\"toTime\": %s}" % (from_time, to_time)
_test_query_assertions_no_timestamp(query, queries)

0 comments on commit 25fe3c0

Please sign in to comment.