From 4660d5ea280333391014a4becb1ea3bda15091c6 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 17 Jul 2019 15:48:09 -0400 Subject: [PATCH 01/17] Update SCO specs per WD 05 specs --- stix2/test/v21/test_observed_data.py | 30 +++++++++++------------ stix2/v21/observables.py | 36 +++++++++++++++++++++------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 09e6a67f..e82ecbd4 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -693,16 +693,16 @@ def test_directory_example(): dir = stix2.v21.Directory( _valid_refs={"1": "file"}, path='/usr/lib', - created="2015-12-21T19:00:00Z", - modified="2015-12-24T19:00:00Z", - accessed="2015-12-21T20:00:00Z", + ctime="2015-12-21T19:00:00Z", + mtime="2015-12-24T19:00:00Z", + atime="2015-12-21T20:00:00Z", contains_refs=["1"], ) assert dir.path == '/usr/lib' - assert dir.created == dt.datetime(2015, 12, 21, 19, 0, 0, tzinfo=pytz.utc) - assert dir.modified == dt.datetime(2015, 12, 24, 19, 0, 0, tzinfo=pytz.utc) - assert dir.accessed == dt.datetime(2015, 12, 21, 20, 0, 0, tzinfo=pytz.utc) + assert dir.ctime == dt.datetime(2015, 12, 21, 19, 0, 0, tzinfo=pytz.utc) + assert dir.mtime == dt.datetime(2015, 12, 24, 19, 0, 0, tzinfo=pytz.utc) + assert dir.atime == dt.datetime(2015, 12, 21, 20, 0, 0, tzinfo=pytz.utc) assert dir.contains_refs == ["1"] @@ -711,9 +711,9 @@ def test_directory_example_ref_error(): stix2.v21.Directory( _valid_refs=[], path='/usr/lib', - created="2015-12-21T19:00:00Z", - modified="2015-12-24T19:00:00Z", - accessed="2015-12-21T20:00:00Z", + ctime="2015-12-21T19:00:00Z", + mtime="2015-12-24T19:00:00Z", + atime="2015-12-21T20:00:00Z", contains_refs=["1"], ) @@ -753,9 +753,9 @@ def test_file_example(): size=100, magic_number_hex="1C", mime_type="application/msword", - created="2016-12-21T19:00:00Z", - modified="2016-12-24T19:00:00Z", - accessed="2016-12-21T20:00:00Z", + ctime="2016-12-21T19:00:00Z", + mtime="2016-12-24T19:00:00Z", + atime="2016-12-21T20:00:00Z", ) assert f.name == "qwerty.dll" @@ -763,9 +763,9 @@ def test_file_example(): assert f.magic_number_hex == "1C" assert f.hashes["SHA-256"] == "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a" assert f.mime_type == "application/msword" - assert f.created == dt.datetime(2016, 12, 21, 19, 0, 0, tzinfo=pytz.utc) - assert f.modified == dt.datetime(2016, 12, 24, 19, 0, 0, tzinfo=pytz.utc) - assert f.accessed == dt.datetime(2016, 12, 21, 20, 0, 0, tzinfo=pytz.utc) + assert f.ctime == dt.datetime(2016, 12, 21, 19, 0, 0, tzinfo=pytz.utc) + assert f.mtime == dt.datetime(2016, 12, 24, 19, 0, 0, tzinfo=pytz.utc) + assert f.atime == dt.datetime(2016, 12, 21, 20, 0, 0, tzinfo=pytz.utc) def test_file_example_with_NTFSExt(): diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index f3838994..be2b127d 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -14,7 +14,7 @@ from ..properties import ( BinaryProperty, BooleanProperty, CallableValues, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, - HashesProperty, HexProperty, IntegerProperty, ListProperty, + HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty, ObjectReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) @@ -28,6 +28,7 @@ class Artifact(_Observable): _type = 'artifact' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), @@ -52,6 +53,7 @@ class AutonomousSystem(_Observable): _type = 'autonomous-system' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('number', IntegerProperty(required=True)), ('name', StringProperty()), ('rir', StringProperty()), @@ -68,12 +70,13 @@ class Directory(_Observable): _type = 'directory' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('path', StringProperty(required=True)), ('path_enc', StringProperty()), # these are not the created/modified timestamps of the object itself - ('created', TimestampProperty()), - ('modified', TimestampProperty()), - ('accessed', TimestampProperty()), + ('ctime', TimestampProperty()), + ('mtime', TimestampProperty()), + ('atime', TimestampProperty()), ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -88,6 +91,7 @@ class DomainName(_Observable): _type = 'domain-name' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), @@ -103,6 +107,7 @@ class EmailAddress(_Observable): _type = 'email-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('display_name', StringProperty()), ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), @@ -137,6 +142,7 @@ class EmailMessage(_Observable): _type = 'email-message' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), ('content_type', StringProperty()), @@ -170,7 +176,7 @@ class ArchiveExt(_Extension): _type = 'archive-ext' _properties = OrderedDict([ - ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types='file'), required=True)), + ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']), required=True)), ('comment', StringProperty()), ]) @@ -323,6 +329,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('hashes', HashesProperty(spec_version='2.1')), ('size', IntegerProperty(min=0)), ('name', StringProperty()), @@ -330,9 +337,9 @@ class File(_Observable): ('magic_number_hex', HexProperty()), ('mime_type', StringProperty()), # these are not the created/modified timestamps of the object itself - ('created', TimestampProperty()), - ('modified', TimestampProperty()), - ('accessed', TimestampProperty()), + ('ctime', TimestampProperty()), + ('mtime', TimestampProperty()), + ('atime', TimestampProperty()), ('parent_directory_ref', ObjectReferenceProperty(valid_types='directory')), ('contains_refs', ListProperty(ObjectReferenceProperty)), ('content_ref', ObjectReferenceProperty(valid_types='artifact')), @@ -353,6 +360,7 @@ class IPv4Address(_Observable): _type = 'ipv4-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), @@ -369,6 +377,7 @@ class IPv6Address(_Observable): _type = 'ipv6-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), @@ -385,6 +394,7 @@ class MACAddress(_Observable): _type = 'mac-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -399,6 +409,7 @@ class Mutex(_Observable): _type = 'mutex' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('name', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -505,6 +516,7 @@ class NetworkTraffic(_Observable): _type = 'network-traffic' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('start', TimestampProperty()), ('end', TimestampProperty()), ('is_active', BooleanProperty()), @@ -624,6 +636,7 @@ class Process(_Observable): _type = 'process' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('is_hidden', BooleanProperty()), ('pid', IntegerProperty()), # this is not the created timestamps of the object itself @@ -663,6 +676,7 @@ class Software(_Observable): _type = 'software' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('name', StringProperty(required=True)), ('cpe', StringProperty()), ('languages', ListProperty(StringProperty)), @@ -681,6 +695,7 @@ class URL(_Observable): _type = 'url' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -710,6 +725,7 @@ class UserAccount(_Observable): _type = 'user-account' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('user_id', StringProperty()), ('credential', StringProperty()), ('account_login', StringProperty()), @@ -767,10 +783,11 @@ class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('key', StringProperty()), ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself - ('modified', TimestampProperty()), + ('modified_time', TimestampProperty()), ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), ('number_of_subkeys', IntegerProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), @@ -818,6 +835,7 @@ class X509Certificate(_Observable): _type = 'x509-certificate' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('is_self_signed', BooleanProperty()), ('hashes', HashesProperty(spec_version='2.1')), ('version', StringProperty()), From ec55463398ae224fc756b131366681bd981fe091 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Mon, 19 Aug 2019 09:39:13 -0400 Subject: [PATCH 02/17] Update SCO stuff to WD 05 --- stix2/base.py | 47 +++ stix2/org/__init__.py | 0 stix2/org/webpki/__init__.py | 0 stix2/org/webpki/json/Canonicalize.py | 503 ++++++++++++++++++++++++++ stix2/org/webpki/json/LICENSE | 254 +++++++++++++ stix2/org/webpki/json/NumberToJson.py | 95 +++++ stix2/org/webpki/json/__init__.py | 0 stix2/test/v21/test_observables.py | 0 stix2/v21/observables.py | 52 ++- 9 files changed, 934 insertions(+), 17 deletions(-) create mode 100644 stix2/org/__init__.py create mode 100644 stix2/org/webpki/__init__.py create mode 100644 stix2/org/webpki/json/Canonicalize.py create mode 100644 stix2/org/webpki/json/LICENSE create mode 100644 stix2/org/webpki/json/NumberToJson.py create mode 100644 stix2/org/webpki/json/__init__.py create mode 100644 stix2/test/v21/test_observables.py diff --git a/stix2/base.py b/stix2/base.py index 9fe1617a..ac9d1624 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -3,10 +3,13 @@ import collections import copy import datetime as dt +import uuid import simplejson as json import six +from stix2.org.webpki.json.Canonicalize import canonicalize + from .exceptions import ( AtLeastOnePropertyError, CustomContentError, DependentPropertiesError, ExtraPropertiesError, ImmutableError, InvalidObjRefError, @@ -309,6 +312,11 @@ def __init__(self, **kwargs): self.__allow_custom = kwargs.get('allow_custom', False) self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False) + if 'id' not in kwargs: + possible_id = self._generate_id(kwargs) + if possible_id is not None: + kwargs['id'] = possible_id + super(_Observable, self).__init__(**kwargs) def _check_ref(self, ref, prop, prop_name): @@ -347,6 +355,45 @@ def _check_property(self, prop_name, prop, kwargs): for ref in kwargs[prop_name]: self._check_ref(ref, prop, prop_name) + def _generate_id(self, kwargs): + required_prefix = self._type + "--" + namespace = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") + + properties_to_use = self._id_contributing_properties + if properties_to_use: + streamlined_object = {} + if "hashes" in kwargs and "hashes" in properties_to_use: + possible_hash = self._choose_one_hash(kwargs["hashes"]) + if possible_hash: + streamlined_object["hashes"] = possible_hash + for key in kwargs.keys(): + if key in properties_to_use and key != "hashes": + streamlined_object[key] = kwargs[key] + + if streamlined_object: + data = canonicalize(streamlined_object) + else: + return None + + return required_prefix + str(uuid.uuid5(namespace, str(data))) + else: + return None + + def _choose_one_hash(self, hash_dict): + if "MD5" in hash_dict: + return {"MD5": hash_dict["MD5"]} + elif "SHA-1" in hash_dict: + return {"SHA-1": hash_dict["SHA-1"]} + elif "SHA-256" in hash_dict: + return {"SHA-256": hash_dict["SHA-256"]} + elif "SHA-512" in hash_dict: + return {"SHA-512": hash_dict["SHA-512"]} + else: + # Cheesy way to pick the first item in the dictionary, since its not indexable + for (k, v) in hash_dict.items(): + break + return {k: v} + class _Extension(_STIXBase): diff --git a/stix2/org/__init__.py b/stix2/org/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stix2/org/webpki/__init__.py b/stix2/org/webpki/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stix2/org/webpki/json/Canonicalize.py b/stix2/org/webpki/json/Canonicalize.py new file mode 100644 index 00000000..cda810f1 --- /dev/null +++ b/stix2/org/webpki/json/Canonicalize.py @@ -0,0 +1,503 @@ +############################################################################## +# # +# Copyright 2006-2019 WebPKI.org (http://webpki.org). # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# https://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# # +############################################################################## + +################################################# +# JCS compatible JSON serializer for Python 3.x # +################################################# + +# This file has been modified to be compatible with Python 2.x as well + +import re + +from stix2.org.webpki.json.NumberToJson import convert2Es6Format + +try: + from _json import encode_basestring_ascii as c_encode_basestring_ascii +except ImportError: + c_encode_basestring_ascii = None +try: + from _json import encode_basestring as c_encode_basestring +except ImportError: + c_encode_basestring = None +try: + from _json import make_encoder as c_make_encoder +except ImportError: + c_make_encoder = None + +ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +HAS_UTF8 = re.compile(b'[\x80-\xff]') +ESCAPE_DCT = { + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) + +INFINITY = float('inf') + + +def py_encode_basestring(s): + """Return a JSON representation of a Python string + + """ + def replace(match): + return ESCAPE_DCT[match.group(0)] + return '"' + ESCAPE.sub(replace, s) + '"' + + +encode_basestring = (c_encode_basestring or py_encode_basestring) + + +def py_encode_basestring_ascii(s): + """Return an ASCII-only JSON representation of a Python string + + """ + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + return '\\u{0:04x}'.format(n) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) + return '"' + ESCAPE_ASCII.sub(replace, s) + '"' + + +encode_basestring_ascii = ( + c_encode_basestring_ascii or py_encode_basestring_ascii +) + + +class JSONEncoder(object): + """Extensible JSON encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str | string | + +-------------------+---------------+ + | int, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + + """ + item_separator = ', ' + key_separator = ': ' + + def __init__( + self, skipkeys=False, ensure_ascii=False, + check_circular=True, allow_nan=True, sort_keys=True, + indent=None, separators=(',', ':'), default=None, + ): + """Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is false, then it is a TypeError to attempt + encoding of keys that are not str, int, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is true, the output is guaranteed to be str + objects with all incoming non-ASCII characters escaped. If + ensure_ascii is false, the output can contain non-ASCII characters. + + If check_circular is true, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is true, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is true, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a non-negative integer, then JSON array + elements and object members will be pretty-printed with that + indent level. An indent level of 0 will only insert newlines. + None is the most compact representation. + + If specified, separators should be an (item_separator, key_separator) + tuple. The default is (', ', ': ') if *indent* is ``None`` and + (',', ': ') otherwise. To get the most compact JSON representation, + you should specify (',', ':') to eliminate whitespace. + + If specified, default is a function that gets called for objects + that can't otherwise be serialized. It should return a JSON encodable + version of the object or raise a ``TypeError``. + + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.indent = indent + if separators is not None: + self.item_separator, self.key_separator = separators + elif indent is not None: + self.item_separator = ',' + if default is not None: + self.default = default + + def default(self, o): + """Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + # Let the base class default method raise the TypeError + return JSONEncoder.default(self, o) + + """ + raise TypeError( + "Object of type '%s' is not JSON serializable" % + o.__class__.__name__, + ) + + def encode(self, o): + """Return a JSON string representation of a Python data structure. + + >>> from json.encoder import JSONEncoder + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo": ["bar", "baz"]}' + + """ + # This is for extremely simple cases and benchmarks. + if isinstance(o, str): + if self.ensure_ascii: + return encode_basestring_ascii(o) + else: + return encode_basestring(o) + # This doesn't pass the iterator directly to ''.join() because the + # exceptions aren't as detailed. The list call should be roughly + # equivalent to the PySequence_Fast that ''.join() would do. + chunks = self.iterencode(o, _one_shot=False) + if not isinstance(chunks, (list, tuple)): + chunks = list(chunks) + return ''.join(chunks) + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + + def floatstr( + o, allow_nan=self.allow_nan, + _repr=float.__repr__, _inf=INFINITY, _neginf=-INFINITY, + ): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on the + # internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o), + ) + + return text + + if ( + _one_shot and c_make_encoder is not None + and self.indent is None + ): + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan, + ) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot, + ) + return _iterencode(o, 0) + + +def _make_iterencode( + markers, _default, _encoder, _indent, _floatstr, + _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, + # HACK: hand-optimized bytecode; turn globals into locals + ValueError=ValueError, + dict=dict, + float=float, + id=id, + int=int, + isinstance=isinstance, + list=list, + str=str, + tuple=tuple, + _intstr=int.__str__, +): + + if _indent is not None and not isinstance(_indent, str): + _indent = ' ' * _indent + + def _iterencode_list(lst, _current_indent_level): + if not lst: + yield '[]' + return + if markers is not None: + markerid = id(lst) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = lst + buf = '[' + if _indent is not None: + _current_indent_level += 1 + newline_indent = '\n' + _indent * _current_indent_level + separator = _item_separator + newline_indent + buf += newline_indent + else: + newline_indent = None + separator = _item_separator + first = True + for value in lst: + if first: + first = False + else: + buf = separator + if isinstance(value, str): + yield buf + _encoder(value) + elif value is None: + yield buf + 'null' + elif value is True: + yield buf + 'true' + elif value is False: + yield buf + 'false' + elif isinstance(value, int): + # Subclasses of int/float may override __str__, but we still + # want to encode them as integers/floats in JSON. One example + # within the standard library is IntEnum. + yield buf + convert2Es6Format(value) + elif isinstance(value, float): + # see comment above for int + yield buf + convert2Es6Format(value) + else: + yield buf + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + # yield from chunks + for chunk in chunks: + yield chunk + if newline_indent is not None: + _current_indent_level -= 1 + yield '\n' + _indent * _current_indent_level + yield ']' + if markers is not None: + del markers[markerid] + + def _iterencode_dict(dct, _current_indent_level): + if not dct: + yield '{}' + return + if markers is not None: + markerid = id(dct) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = dct + yield '{' + if _indent is not None: + _current_indent_level += 1 + newline_indent = '\n' + _indent * _current_indent_level + item_separator = _item_separator + newline_indent + yield newline_indent + else: + newline_indent = None + item_separator = _item_separator + first = True + if _sort_keys: + items = sorted(dct.items(), key=lambda kv: kv[0].encode('utf-16_be')) + else: + items = dct.items() + for key, value in items: + if isinstance(key, str): + pass + # JavaScript is weakly typed for these, so it makes sense to + # also allow them. Many encoders seem to do something like this. + elif isinstance(key, float): + # see comment for int/float in _make_iterencode + key = convert2Es6Format(key) + elif key is True: + key = 'true' + elif key is False: + key = 'false' + elif key is None: + key = 'null' + elif isinstance(key, int): + # see comment for int/float in _make_iterencode + key = convert2Es6Format(key) + elif _skipkeys: + continue + else: + raise TypeError("key " + repr(key) + " is not a string") + if first: + first = False + else: + yield item_separator + yield _encoder(key) + yield _key_separator + if isinstance(value, str): + yield _encoder(value) + elif value is None: + yield 'null' + elif value is True: + yield 'true' + elif value is False: + yield 'false' + elif isinstance(value, int): + # see comment for int/float in _make_iterencode + yield convert2Es6Format(value) + elif isinstance(value, float): + # see comment for int/float in _make_iterencode + yield convert2Es6Format(value) + else: + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + # yield from chunks + for chunk in chunks: + yield chunk + if newline_indent is not None: + _current_indent_level -= 1 + yield '\n' + _indent * _current_indent_level + yield '}' + if markers is not None: + del markers[markerid] + + def _iterencode(o, _current_indent_level): + if isinstance(o, str): + yield _encoder(o) + elif o is None: + yield 'null' + elif o is True: + yield 'true' + elif o is False: + yield 'false' + elif isinstance(o, int): + # see comment for int/float in _make_iterencode + yield convert2Es6Format(o) + elif isinstance(o, float): + # see comment for int/float in _make_iterencode + yield convert2Es6Format(o) + elif isinstance(o, (list, tuple)): + # yield from _iterencode_list(o, _current_indent_level) + for thing in _iterencode_list(o, _current_indent_level): + yield thing + elif isinstance(o, dict): + # yield from _iterencode_dict(o, _current_indent_level) + for thing in _iterencode_dict(o, _current_indent_level): + yield thing + else: + if markers is not None: + markerid = id(o) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = o + o = _default(o) + # yield from _iterencode(o, _current_indent_level) + for thing in _iterencode(o, _current_indent_level): + yield thing + if markers is not None: + del markers[markerid] + return _iterencode + + +def canonicalize(obj, utf8=True): + textVal = JSONEncoder(sort_keys=True).encode(obj) + if utf8: + return textVal.encode() + return textVal + + +def serialize(obj, utf8=True): + textVal = JSONEncoder(sort_keys=False).encode(obj) + if utf8: + return textVal.encode() + return textVal diff --git a/stix2/org/webpki/json/LICENSE b/stix2/org/webpki/json/LICENSE new file mode 100644 index 00000000..1afbedba --- /dev/null +++ b/stix2/org/webpki/json/LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All +Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/stix2/org/webpki/json/NumberToJson.py b/stix2/org/webpki/json/NumberToJson.py new file mode 100644 index 00000000..cea54d06 --- /dev/null +++ b/stix2/org/webpki/json/NumberToJson.py @@ -0,0 +1,95 @@ +############################################################################## +# # +# Copyright 2006-2019 WebPKI.org (http://webpki.org). # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# https://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# # +############################################################################## + + +################################################################## +# Convert a Python double/float into an ES6/V8 compatible string # +################################################################## +def convert2Es6Format(value): + # Convert double/float to str using the native Python formatter + fvalue = float(value) + + # Zero is a special case. The following line takes "-0" case as well + if fvalue == 0: + return '0' + + # The rest of the algorithm works on the textual representation only + pyDouble = str(fvalue) + + # The following line catches the "inf" and "nan" values returned by str(fvalue) + if pyDouble.find('n') >= 0: + raise ValueError("Invalid JSON number: " + pyDouble) + + # Save sign separately, it doesn't have any role in the algorithm + pySign = '' + if pyDouble.find('-') == 0: + pySign = '-' + pyDouble = pyDouble[1:] + + # Now we should only have valid non-zero values + pyExpStr = '' + pyExpVal = 0 + q = pyDouble.find('e') + if q > 0: + # Grab the exponent and remove it from the number + pyExpStr = pyDouble[q:] + if pyExpStr[2:3] == '0': + # Supress leading zero on exponents + pyExpStr = pyExpStr[:2] + pyExpStr[3:] + pyDouble = pyDouble[0:q] + pyExpVal = int(pyExpStr[1:]) + + # Split number in pyFirst + pyDot + pyLast + pyFirst = pyDouble + pyDot = '' + pyLast = '' + q = pyDouble.find('.') + if q > 0: + pyDot = '.' + pyFirst = pyDouble[:q] + pyLast = pyDouble[q + 1:] + + # Now the string is split into: pySign + pyFirst + pyDot + pyLast + pyExpStr + if pyLast == '0': + # Always remove trailing .0 + pyDot = '' + pyLast = '' + + if pyExpVal > 0 and pyExpVal < 21: + # Integers are shown as is with up to 21 digits + pyFirst += pyLast + pyLast = '' + pyDot = '' + pyExpStr = '' + q = pyExpVal - len(pyFirst) + while q >= 0: + q -= 1 + pyFirst += '0' + elif pyExpVal < 0 and pyExpVal > -7: + # Small numbers are shown as 0.etc with e-6 as lower limit + pyLast = pyFirst + pyLast + pyFirst = '0' + pyDot = '.' + pyExpStr = '' + q = pyExpVal + while q < -1: + q += 1 + pyLast = '0' + pyLast + + # The resulting sub-strings are concatenated + return pySign + pyFirst + pyDot + pyLast + pyExpStr diff --git a/stix2/org/webpki/json/__init__.py b/stix2/org/webpki/json/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stix2/test/v21/test_observables.py b/stix2/test/v21/test_observables.py new file mode 100644 index 00000000..e69de29b diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index be2b127d..438fed26 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -28,7 +28,7 @@ class Artifact(_Observable): _type = 'artifact' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), @@ -37,6 +37,7 @@ class Artifact(_Observable): ('decryption_key', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["hashes", "payload_bin"] def _check_object_constraints(self): super(Artifact, self)._check_object_constraints() @@ -53,12 +54,13 @@ class AutonomousSystem(_Observable): _type = 'autonomous-system' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('number', IntegerProperty(required=True)), ('name', StringProperty()), ('rir', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["number"] class Directory(_Observable): @@ -70,7 +72,7 @@ class Directory(_Observable): _type = 'directory' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('path', StringProperty(required=True)), ('path_enc', StringProperty()), # these are not the created/modified timestamps of the object itself @@ -80,6 +82,7 @@ class Directory(_Observable): ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["path"] class DomainName(_Observable): @@ -91,11 +94,12 @@ class DomainName(_Observable): _type = 'domain-name' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class EmailAddress(_Observable): @@ -107,12 +111,13 @@ class EmailAddress(_Observable): _type = 'email-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('display_name', StringProperty()), ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class EmailMIMEComponent(_STIXBase): @@ -142,7 +147,7 @@ class EmailMessage(_Observable): _type = 'email-message' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), ('content_type', StringProperty()), @@ -159,6 +164,7 @@ class EmailMessage(_Observable): ('raw_email_ref', ObjectReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["from_ref", "subject", "body"] def _check_object_constraints(self): super(EmailMessage, self)._check_object_constraints() @@ -329,7 +335,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('hashes', HashesProperty(spec_version='2.1')), ('size', IntegerProperty(min=0)), ('name', StringProperty()), @@ -345,6 +351,7 @@ class File(_Observable): ('content_ref', ObjectReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["hashes", "name", "extensions"] def _check_object_constraints(self): super(File, self)._check_object_constraints() @@ -360,12 +367,13 @@ class IPv4Address(_Observable): _type = 'ipv4-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class IPv6Address(_Observable): @@ -377,12 +385,13 @@ class IPv6Address(_Observable): _type = 'ipv6-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class MACAddress(_Observable): @@ -394,10 +403,11 @@ class MACAddress(_Observable): _type = 'mac-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class Mutex(_Observable): @@ -409,10 +419,11 @@ class Mutex(_Observable): _type = 'mutex' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["name"] class HTTPRequestExt(_Extension): @@ -516,7 +527,7 @@ class NetworkTraffic(_Observable): _type = 'network-traffic' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('start', TimestampProperty()), ('end', TimestampProperty()), ('is_active', BooleanProperty()), @@ -536,6 +547,7 @@ class NetworkTraffic(_Observable): ('encapsulates_by_ref', ObjectReferenceProperty(valid_types='network-traffic')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["start", "src_ref", "dst_ref", "src_port", "dst_port", "protocols"] def _check_object_constraints(self): super(NetworkTraffic, self)._check_object_constraints() @@ -651,6 +663,7 @@ class Process(_Observable): ('child_refs', ListProperty(ObjectReferenceProperty('process'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = [] def _check_object_constraints(self): # no need to check windows-service-ext, since it has a required property @@ -676,7 +689,7 @@ class Software(_Observable): _type = 'software' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), ('cpe', StringProperty()), ('languages', ListProperty(StringProperty)), @@ -684,6 +697,7 @@ class Software(_Observable): ('version', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["name", "cpe", "vendor", "version"] class URL(_Observable): @@ -695,10 +709,11 @@ class URL(_Observable): _type = 'url' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class UNIXAccountExt(_Extension): @@ -725,7 +740,7 @@ class UserAccount(_Observable): _type = 'user-account' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('user_id', StringProperty()), ('credential', StringProperty()), ('account_login', StringProperty()), @@ -742,6 +757,7 @@ class UserAccount(_Observable): ('account_last_login', TimestampProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["account_type", "user_id", "account_login"] class WindowsRegistryValueType(_STIXBase): @@ -783,7 +799,7 @@ class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('key', StringProperty()), ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself @@ -792,6 +808,7 @@ class WindowsRegistryKey(_Observable): ('number_of_subkeys', IntegerProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["key", "values"] @property def values(self): @@ -835,7 +852,7 @@ class X509Certificate(_Observable): _type = 'x509-certificate' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('is_self_signed', BooleanProperty()), ('hashes', HashesProperty(spec_version='2.1')), ('version', StringProperty()), @@ -851,6 +868,7 @@ class X509Certificate(_Observable): ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["hashes", "serial_number"] def CustomObservable(type='x-custom-observable', properties=None): From 46359ead69b49d8e4fcd79499609c468089730a3 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Mon, 19 Aug 2019 13:35:17 -0400 Subject: [PATCH 03/17] Modify a few things --- stix2/base.py | 37 ++- .../json => canonicalization}/Canonicalize.py | 2 +- .../json => canonicalization}/NumberToJson.py | 0 stix2/{org => canonicalization}/__init__.py | 0 stix2/org/webpki/__init__.py | 0 stix2/org/webpki/json/LICENSE | 254 ------------------ stix2/org/webpki/json/__init__.py | 0 stix2/test/v21/test_observables.py | 0 stix2/v21/observables.py | 2 +- 9 files changed, 20 insertions(+), 275 deletions(-) rename stix2/{org/webpki/json => canonicalization}/Canonicalize.py (99%) rename stix2/{org/webpki/json => canonicalization}/NumberToJson.py (100%) rename stix2/{org => canonicalization}/__init__.py (100%) delete mode 100644 stix2/org/webpki/__init__.py delete mode 100644 stix2/org/webpki/json/LICENSE delete mode 100644 stix2/org/webpki/json/__init__.py delete mode 100644 stix2/test/v21/test_observables.py diff --git a/stix2/base.py b/stix2/base.py index ac9d1624..580823d0 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -8,7 +8,7 @@ import simplejson as json import six -from stix2.org.webpki.json.Canonicalize import canonicalize +from stix2.canonicalization.Canonicalize import canonicalize from .exceptions import ( AtLeastOnePropertyError, CustomContentError, DependentPropertiesError, @@ -359,24 +359,23 @@ def _generate_id(self, kwargs): required_prefix = self._type + "--" namespace = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") - properties_to_use = self._id_contributing_properties - if properties_to_use: - streamlined_object = {} - if "hashes" in kwargs and "hashes" in properties_to_use: - possible_hash = self._choose_one_hash(kwargs["hashes"]) - if possible_hash: - streamlined_object["hashes"] = possible_hash - for key in kwargs.keys(): - if key in properties_to_use and key != "hashes": - streamlined_object[key] = kwargs[key] - - if streamlined_object: - data = canonicalize(streamlined_object) - else: - return None - - return required_prefix + str(uuid.uuid5(namespace, str(data))) - else: + try: + properties_to_use = self._id_contributing_properties + if properties_to_use: + streamlined_object = {} + if "hashes" in kwargs and "hashes" in properties_to_use: + possible_hash = self._choose_one_hash(kwargs["hashes"]) + if possible_hash: + streamlined_object["hashes"] = possible_hash + for key in kwargs.keys(): + if key in properties_to_use and key != "hashes": + streamlined_object[key] = kwargs[key] + + if streamlined_object: + data = canonicalize(streamlined_object, utf8=False) + return required_prefix + str(uuid.uuid5(namespace, str(data))) + return None + except AttributeError: return None def _choose_one_hash(self, hash_dict): diff --git a/stix2/org/webpki/json/Canonicalize.py b/stix2/canonicalization/Canonicalize.py similarity index 99% rename from stix2/org/webpki/json/Canonicalize.py rename to stix2/canonicalization/Canonicalize.py index cda810f1..8bf25295 100644 --- a/stix2/org/webpki/json/Canonicalize.py +++ b/stix2/canonicalization/Canonicalize.py @@ -24,7 +24,7 @@ import re -from stix2.org.webpki.json.NumberToJson import convert2Es6Format +from stix2.canonicalization.NumberToJson import convert2Es6Format try: from _json import encode_basestring_ascii as c_encode_basestring_ascii diff --git a/stix2/org/webpki/json/NumberToJson.py b/stix2/canonicalization/NumberToJson.py similarity index 100% rename from stix2/org/webpki/json/NumberToJson.py rename to stix2/canonicalization/NumberToJson.py diff --git a/stix2/org/__init__.py b/stix2/canonicalization/__init__.py similarity index 100% rename from stix2/org/__init__.py rename to stix2/canonicalization/__init__.py diff --git a/stix2/org/webpki/__init__.py b/stix2/org/webpki/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/stix2/org/webpki/json/LICENSE b/stix2/org/webpki/json/LICENSE deleted file mode 100644 index 1afbedba..00000000 --- a/stix2/org/webpki/json/LICENSE +++ /dev/null @@ -1,254 +0,0 @@ -A. HISTORY OF THE SOFTWARE -========================== - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands -as a successor of a language called ABC. Guido remains Python's -principal author, although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) -in Reston, Virginia where he released several versions of the -software. - -In May 2000, Guido and the Python core development team moved to -BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations, which became -Zope Corporation. In 2001, the Python Software Foundation (PSF, see -https://www.python.org/psf/) was formed, a non-profit organization -created specifically to own Python-related Intellectual Property. -Zope Corporation was a sponsoring member of the PSF. - -All Python releases are Open Source (see http://www.opensource.org for -the Open Source Definition). Historically, most, but not all, Python -releases have also been GPL-compatible; the table below summarizes -the various releases. - - Release Derived Year Owner GPL- - from compatible? (1) - - 0.9.0 thru 1.2 1991-1995 CWI yes - 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes - 1.6 1.5.2 2000 CNRI no - 2.0 1.6 2000 BeOpen.com no - 1.6.1 1.6 2001 CNRI yes (2) - 2.1 2.0+1.6.1 2001 PSF no - 2.0.1 2.0+1.6.1 2001 PSF yes - 2.1.1 2.1+2.0.1 2001 PSF yes - 2.1.2 2.1.1 2002 PSF yes - 2.1.3 2.1.2 2002 PSF yes - 2.2 and above 2.1.1 2001-now PSF yes - -Footnotes: - -(1) GPL-compatible doesn't mean that we're distributing Python under - the GPL. All Python licenses, unlike the GPL, let you distribute - a modified version without making your changes open source. The - GPL-compatible licenses make it possible to combine Python with - other software that is released under the GPL; the others don't. - -(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, - because its license has a choice of law clause. According to - CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 - is "not incompatible" with the GPL. - -Thanks to the many outside volunteers who have worked under Guido's -direction to make these releases possible. - - -B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON -=============================================================== - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All -Rights Reserved" are retained in Python alone or in any derivative version -prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6.1 software in -source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python 1.6.1 -alone or in any derivative version, provided, however, that CNRI's -License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) -1995-2001 Corporation for National Research Initiatives; All Rights -Reserved" are retained in Python 1.6.1 alone or in any derivative -version prepared by Licensee. Alternately, in lieu of CNRI's License -Agreement, Licensee may substitute the following text (omitting the -quotes): "Python 1.6.1 is made available subject to the terms and -conditions in CNRI's License Agreement. This Agreement together with -Python 1.6.1 may be located on the Internet using the following -unique, persistent identifier (known as a handle): 1895.22/1013. This -Agreement may also be obtained from a proxy server on the Internet -using the following URL: http://hdl.handle.net/1895.22/1013". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6.1 or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python 1.6.1. - -4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by the federal -intellectual property law of the United States, including without -limitation the federal copyright law, and, to the extent such -U.S. federal law does not apply, by the law of the Commonwealth of -Virginia, excluding Virginia's conflict of law provisions. -Notwithstanding the foregoing, with regard to derivative works based -on Python 1.6.1 that incorporate non-separable material that was -previously distributed under the GNU General Public License (GPL), the -law of the Commonwealth of Virginia shall govern this License -Agreement only as to issues arising under or with respect to -Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this -License Agreement shall be deemed to create any relationship of -agency, partnership, or joint venture between CNRI and Licensee. This -License Agreement does not grant permission to use CNRI trademarks or -trade name in a trademark sense to endorse or promote products or -services of Licensee, or any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6.1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - - ACCEPT - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/stix2/org/webpki/json/__init__.py b/stix2/org/webpki/json/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/stix2/test/v21/test_observables.py b/stix2/test/v21/test_observables.py deleted file mode 100644 index e69de29b..00000000 diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 438fed26..3d9c1d5d 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -648,7 +648,7 @@ class Process(_Observable): _type = 'process' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('is_hidden', BooleanProperty()), ('pid', IntegerProperty()), # this is not the created timestamps of the object itself From bf1b8b567d89a7772c147769a7d939a9deb5d0d3 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 21 Aug 2019 02:00:41 -0400 Subject: [PATCH 04/17] Updates to allow existing tests to pass --- stix2/base.py | 26 ++++++++++++++++++++++-- stix2/canonicalization/Canonicalize.py | 5 +++++ stix2/test/v21/test_datastore_filters.py | 5 +++-- stix2/test/v21/test_observed_data.py | 17 ++++++++++++++++ stix2/v21/observables.py | 2 +- 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 580823d0..eb468ca1 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -128,6 +128,11 @@ def _check_at_least_one_property(self, list_of_properties=None): list_of_properties.remove('type') current_properties = self.properties_populated() list_of_properties_populated = set(list_of_properties).intersection(current_properties) + + if list_of_properties_populated == set(['id']) and isinstance(self, _Observable): + # Do not count the auto-generated id as a user-specified property + list_of_properties_populated = None + if list_of_properties and (not list_of_properties_populated or list_of_properties_populated == set(['extensions'])): raise AtLeastOnePropertyError(self.__class__, list_of_properties) @@ -369,13 +374,24 @@ def _generate_id(self, kwargs): streamlined_object["hashes"] = possible_hash for key in kwargs.keys(): if key in properties_to_use and key != "hashes": - streamlined_object[key] = kwargs[key] + if type(kwargs[key]) is dict: + for otherKey in kwargs[key]: + if isinstance(kwargs[key][otherKey], _STIXBase): + streamlined_object[key] = self._embed_obj_to_json(kwargs[key][otherKey]) + else: + streamlined_object[key] = kwargs[key] + else: + if isinstance(kwargs[key], _STIXBase): + streamlined_object[key] = self._embed_obj_to_json(kwargs[key]) + else: + streamlined_object[key] = kwargs[key] if streamlined_object: - data = canonicalize(streamlined_object, utf8=False) + data = canonicalize(str(streamlined_object), utf8=False) return required_prefix + str(uuid.uuid5(namespace, str(data))) return None except AttributeError: + # We ideally end up here if handling a 2.0 SCO return None def _choose_one_hash(self, hash_dict): @@ -393,6 +409,12 @@ def _choose_one_hash(self, hash_dict): break return {k: v} + def _embed_obj_to_json(self, obj): + tmp_obj = dict(copy.deepcopy(obj)) + for prop_name in obj._defaulted_optional_properties: + del tmp_obj[prop_name] + return tmp_obj + class _Extension(_STIXBase): diff --git a/stix2/canonicalization/Canonicalize.py b/stix2/canonicalization/Canonicalize.py index 8bf25295..daf06165 100644 --- a/stix2/canonicalization/Canonicalize.py +++ b/stix2/canonicalization/Canonicalize.py @@ -360,6 +360,7 @@ def _iterencode_list(lst, _current_indent_level): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) + # Below line commented-out for python2 compatibility # yield from chunks for chunk in chunks: yield chunk @@ -441,6 +442,7 @@ def _iterencode_dict(dct, _current_indent_level): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) + # Below line commented-out for python2 compatibility # yield from chunks for chunk in chunks: yield chunk @@ -467,10 +469,12 @@ def _iterencode(o, _current_indent_level): # see comment for int/float in _make_iterencode yield convert2Es6Format(o) elif isinstance(o, (list, tuple)): + # Below line commented-out for python2 compatibility # yield from _iterencode_list(o, _current_indent_level) for thing in _iterencode_list(o, _current_indent_level): yield thing elif isinstance(o, dict): + # Below line commented-out for python2 compatibility # yield from _iterencode_dict(o, _current_indent_level) for thing in _iterencode_dict(o, _current_indent_level): yield thing @@ -481,6 +485,7 @@ def _iterencode(o, _current_indent_level): raise ValueError("Circular reference detected") markers[markerid] = o o = _default(o) + # Below line commented-out for python2 compatibility # yield from _iterencode(o, _current_indent_level) for thing in _iterencode(o, _current_indent_level): yield thing diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index b96aa4dc..92d035ba 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -86,6 +86,7 @@ "objects": { "0": { "type": "file", + "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec", "name": "HAL 9000.exe", }, }, @@ -109,8 +110,8 @@ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe"}}), - Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe"}), + Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe", "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec"}}), + Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe", "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec"}), Filter("labels", "contains", "heartbleed"), ] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 9916b799..64c9859e 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -25,6 +25,7 @@ "objects": { "0": { "type": "file", + "id": "file--500d9a03-9d03-5c31-82b2-2be8aacec481", "name": "foo.exe" } } @@ -64,10 +65,12 @@ def test_observed_data_example(): "objects": { "0": { "type": "file", + "id": "file--500d9a03-9d03-5c31-82b2-2be8aacec481", "name": "foo.exe" }, "1": { "type": "directory", + "id": "directory--ed959127-2df3-5999-99b6-df7614398c1c", "path": "/usr/home", "contains_refs": [ "0" @@ -1391,3 +1394,17 @@ def test_objects_deprecation(): }, }, ) + + +# def test_deterministic_id_same_extra_prop_vals(): +# email_addr_1 = stix2.v21.EmailAddress( +# value="john@example.com", +# display_name="Johnny Doe" +# ) + +# email_addr_2 = stix2.v21.EmailAddress( +# value="john@example.com", +# display_name="Johnny Doe" +# ) + +# assert email_addr_1.id == email_addr_2.id diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 3d9c1d5d..db7a3c78 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -41,7 +41,7 @@ class Artifact(_Observable): def _check_object_constraints(self): super(Artifact, self)._check_object_constraints() - self._check_mutually_exclusive_properties(['payload_bin', 'url']) + self._check_mutually_exclusive_properties(['payload_bin', 'url'], at_least_one=False) self._check_properties_dependency(['hashes'], ['url']) From 5e9d6a6a14193db21a9ddf05995dbc4806b82a24 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 21 Aug 2019 08:49:33 -0400 Subject: [PATCH 05/17] Fix small indentation error --- stix2/base.py | 2 +- stix2/test/v21/test_observed_data.py | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index eb468ca1..03d808e1 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -382,7 +382,7 @@ def _generate_id(self, kwargs): streamlined_object[key] = kwargs[key] else: if isinstance(kwargs[key], _STIXBase): - streamlined_object[key] = self._embed_obj_to_json(kwargs[key]) + streamlined_object[key] = self._embed_obj_to_json(kwargs[key]) else: streamlined_object[key] = kwargs[key] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 64c9859e..90a494a4 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1396,15 +1396,15 @@ def test_objects_deprecation(): ) -# def test_deterministic_id_same_extra_prop_vals(): -# email_addr_1 = stix2.v21.EmailAddress( -# value="john@example.com", -# display_name="Johnny Doe" -# ) - -# email_addr_2 = stix2.v21.EmailAddress( -# value="john@example.com", -# display_name="Johnny Doe" -# ) - -# assert email_addr_1.id == email_addr_2.id +def test_deterministic_id_same_extra_prop_vals(): + email_addr_1 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Johnny Doe", + ) + + email_addr_2 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Johnny Doe", + ) + + assert email_addr_1.id == email_addr_2.id From 364daec40a220858ca3eb281033475d75f2f0ac6 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 21 Aug 2019 09:21:51 -0400 Subject: [PATCH 06/17] Add deterministic ID tests --- stix2/test/v21/test_observed_data.py | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 90a494a4..370d1ccc 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1,5 +1,6 @@ import datetime as dt import re +import uuid import pytest import pytz @@ -1408,3 +1409,51 @@ def test_deterministic_id_same_extra_prop_vals(): ) assert email_addr_1.id == email_addr_2.id + + +def test_deterministic_id_diff_extra_prop_vals(): + email_addr_1 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Johnny Doe", + ) + + email_addr_2 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Janey Doe", + ) + + assert email_addr_1.id == email_addr_2.id + + +def test_deterministic_id_diff_contributing_prop_vals(): + email_addr_1 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Johnny Doe", + ) + + email_addr_2 = stix2.v21.EmailAddress( + value="jane@example.com", + display_name="Janey Doe", + ) + + assert email_addr_1.id != email_addr_2.id + + +def test_deterministic_id_no_contributing_props(): + email_msg_1 = stix2.v21.EmailMessage( + is_multipart=False, + ) + + email_msg_2 = stix2.v21.EmailMessage( + is_multipart=False, + ) + + assert email_msg_1.id != email_msg_2.id + + uuid_obj_1 = uuid.UUID(email_msg_1.id) + assert uuid_obj_1.variant == uuid.RFC_4122 + assert uuid_obj_1.version == 5 + + uuid_obj_2 = uuid.UUID(email_msg_2.id) + assert uuid_obj_2.variant == uuid.RFC_4122 + assert uuid_obj_2.version == 5 From 7c9fc3fd08468bdc64cc8d83e7e78a168222fc7d Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 21 Aug 2019 09:33:42 -0400 Subject: [PATCH 07/17] Fix deterministic ID tests --- stix2/test/v21/test_observed_data.py | 32 ++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 370d1ccc..dc27058e 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1410,6 +1410,14 @@ def test_deterministic_id_same_extra_prop_vals(): assert email_addr_1.id == email_addr_2.id + uuid_obj_1 = uuid.UUID(email_addr_1.id[-36:]) + assert uuid_obj_1.variant == uuid.RFC_4122 + assert uuid_obj_1.version == 5 + + uuid_obj_2 = uuid.UUID(email_addr_2.id[-36:]) + assert uuid_obj_2.variant == uuid.RFC_4122 + assert uuid_obj_2.version == 5 + def test_deterministic_id_diff_extra_prop_vals(): email_addr_1 = stix2.v21.EmailAddress( @@ -1424,6 +1432,14 @@ def test_deterministic_id_diff_extra_prop_vals(): assert email_addr_1.id == email_addr_2.id + uuid_obj_1 = uuid.UUID(email_addr_1.id[-36:]) + assert uuid_obj_1.variant == uuid.RFC_4122 + assert uuid_obj_1.version == 5 + + uuid_obj_2 = uuid.UUID(email_addr_2.id[-36:]) + assert uuid_obj_2.variant == uuid.RFC_4122 + assert uuid_obj_2.version == 5 + def test_deterministic_id_diff_contributing_prop_vals(): email_addr_1 = stix2.v21.EmailAddress( @@ -1438,6 +1454,14 @@ def test_deterministic_id_diff_contributing_prop_vals(): assert email_addr_1.id != email_addr_2.id + uuid_obj_1 = uuid.UUID(email_addr_1.id[-36:]) + assert uuid_obj_1.variant == uuid.RFC_4122 + assert uuid_obj_1.version == 5 + + uuid_obj_2 = uuid.UUID(email_addr_2.id[-36:]) + assert uuid_obj_2.variant == uuid.RFC_4122 + assert uuid_obj_2.version == 5 + def test_deterministic_id_no_contributing_props(): email_msg_1 = stix2.v21.EmailMessage( @@ -1450,10 +1474,10 @@ def test_deterministic_id_no_contributing_props(): assert email_msg_1.id != email_msg_2.id - uuid_obj_1 = uuid.UUID(email_msg_1.id) + uuid_obj_1 = uuid.UUID(email_msg_1.id[-36:]) assert uuid_obj_1.variant == uuid.RFC_4122 - assert uuid_obj_1.version == 5 + assert uuid_obj_1.version == 4 - uuid_obj_2 = uuid.UUID(email_msg_2.id) + uuid_obj_2 = uuid.UUID(email_msg_2.id[-36:]) assert uuid_obj_2.variant == uuid.RFC_4122 - assert uuid_obj_2.version == 5 + assert uuid_obj_2.version == 4 From 49077352d7db2f8f132c256c63fe1f6be6a53260 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Tue, 27 Aug 2019 17:36:45 -0400 Subject: [PATCH 08/17] Updates and corrections for SCO WD 05 updates. Temp backup; testing and more fixes coming soon --- stix2/base.py | 109 +++++++++--------- stix2/properties.py | 13 ++- stix2/test/v20/test_properties.py | 4 +- stix2/test/v21/test_properties.py | 4 +- stix2/v20/common.py | 6 +- stix2/v20/sdo.py | 54 ++++----- stix2/v20/sro.py | 18 +-- stix2/v21/common.py | 12 +- stix2/v21/observables.py | 178 ++++++++++++++++++++++++------ stix2/v21/sdo.py | 88 +++++++-------- stix2/v21/sro.py | 18 +-- 11 files changed, 317 insertions(+), 187 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 03d808e1..7eda12cd 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -317,10 +317,17 @@ def __init__(self, **kwargs): self.__allow_custom = kwargs.get('allow_custom', False) self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False) - if 'id' not in kwargs: - possible_id = self._generate_id(kwargs) - if possible_id is not None: - kwargs['id'] = possible_id + try: + # Since `spec_version` is optional, this is how we check for a 2.1 SCO + self._id_contributing_properties + + if 'id' not in kwargs: + possible_id = self._generate_id(kwargs) + if possible_id is not None: + kwargs['id'] = possible_id + except AttributeError: + # End up here if handling a 2.0 SCO, and don't need to do anything further + pass super(_Observable, self).__init__(**kwargs) @@ -364,37 +371,38 @@ def _generate_id(self, kwargs): required_prefix = self._type + "--" namespace = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") - try: - properties_to_use = self._id_contributing_properties - if properties_to_use: - streamlined_object = {} - if "hashes" in kwargs and "hashes" in properties_to_use: - possible_hash = self._choose_one_hash(kwargs["hashes"]) - if possible_hash: - streamlined_object["hashes"] = possible_hash - for key in kwargs.keys(): - if key in properties_to_use and key != "hashes": - if type(kwargs[key]) is dict: - for otherKey in kwargs[key]: - if isinstance(kwargs[key][otherKey], _STIXBase): - streamlined_object[key] = self._embed_obj_to_json(kwargs[key][otherKey]) - else: - streamlined_object[key] = kwargs[key] - else: - if isinstance(kwargs[key], _STIXBase): - streamlined_object[key] = self._embed_obj_to_json(kwargs[key]) - else: - streamlined_object[key] = kwargs[key] - - if streamlined_object: - data = canonicalize(str(streamlined_object), utf8=False) - return required_prefix + str(uuid.uuid5(namespace, str(data))) - return None - except AttributeError: - # We ideally end up here if handling a 2.0 SCO - return None + properties_to_use = self._id_contributing_properties + if properties_to_use: + streamlined_obj_vals = [] + if "hashes" in kwargs and "hashes" in properties_to_use: + possible_hash = _choose_one_hash(kwargs["hashes"]) + if possible_hash: + streamlined_obj_vals.append(possible_hash) + for key in properties_to_use: + if key != "hashes" and key in kwargs: + if isinstance(kwargs[key], dict) or isinstance(kwargs[key], _STIXBase): + temp_deep_copy = copy.deepcopy(dict(kwargs[key])) + _recursive_stix_to_dict(temp_deep_copy) + streamlined_obj_vals.append(temp_deep_copy) + else: + streamlined_obj_vals.append(kwargs[key]) + + if streamlined_obj_vals: + data = canonicalize(streamlined_obj_vals, utf8=False) + return required_prefix + six.text_type(uuid.uuid5(namespace, data)) + + # We return None if there are no values specified for any of the id-contributing-properties + return None + + +class _Extension(_STIXBase): + + def _check_object_constraints(self): + super(_Extension, self)._check_object_constraints() + self._check_at_least_one_property() + - def _choose_one_hash(self, hash_dict): +def _choose_one_hash(hash_dict): if "MD5" in hash_dict: return {"MD5": hash_dict["MD5"]} elif "SHA-1" in hash_dict: @@ -404,25 +412,24 @@ def _choose_one_hash(self, hash_dict): elif "SHA-512" in hash_dict: return {"SHA-512": hash_dict["SHA-512"]} else: - # Cheesy way to pick the first item in the dictionary, since its not indexable - for (k, v) in hash_dict.items(): - break - return {k: v} - - def _embed_obj_to_json(self, obj): - tmp_obj = dict(copy.deepcopy(obj)) - for prop_name in obj._defaulted_optional_properties: - del tmp_obj[prop_name] - return tmp_obj - - -class _Extension(_STIXBase): - - def _check_object_constraints(self): - super(_Extension, self)._check_object_constraints() - self._check_at_least_one_property() + k = next(iter(hash_dict), None) + if k is not None: + return {k: hash_dict[k]} def _cls_init(cls, obj, kwargs): if getattr(cls, '__init__', object.__init__) is not object.__init__: cls.__init__(obj, **kwargs) + + +def _recursive_stix_to_dict(input_dict): + for key in input_dict: + if isinstance(input_dict[key], dict): + _recursive_stix_to_dict(input_dict[key]) + elif isinstance(input_dict[key], _STIXBase): + input_dict[key] = dict(input_dict[key]) + + # There may stil be nested _STIXBase objects + _recursive_stix_to_dict(input_dict[key]) + else: + return diff --git a/stix2/properties.py b/stix2/properties.py index b9a5aff5..ed73f3a8 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -417,12 +417,16 @@ def clean(self, value): class ReferenceProperty(Property): - def __init__(self, type=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): + def __init__(self, valid_types=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): """ references sometimes must be to a specific object type """ - self.required_prefix = type + "--" if type else None self.spec_version = spec_version + + if valid_types and type(valid_types) is not list: + valid_types = [valid_types] + self.valid_types = valid_types + super(ReferenceProperty, self).__init__(**kwargs) def clean(self, value): @@ -430,7 +434,10 @@ def clean(self, value): value = value.id value = str(value) - _validate_id(value, self.spec_version, self.required_prefix) + if self.valid_types and value[:value.index('--')] in self.valid_types: + required_prefix = value[:value.index('--') + 2] + + _validate_id(value, self.spec_version, required_prefix) return value diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 9952eac8..548e7fe0 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -276,7 +276,7 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty(spec_version="2.0") + ref_prop = ReferenceProperty(valid_types=None, spec_version="2.0") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): @@ -288,7 +288,7 @@ def test_reference_property(): def test_reference_property_specific_type(): - ref_prop = ReferenceProperty("my-type", spec_version="2.0") + ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.0") with pytest.raises(ValueError): ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index fde13d3c..41ca3e9b 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -271,7 +271,7 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty(spec_version="2.1") + ref_prop = ReferenceProperty(valid_types=None, spec_version="2.1") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): @@ -283,7 +283,7 @@ def test_reference_property(): def test_reference_property_specific_type(): - ref_prop = ReferenceProperty("my-type", spec_version="2.1") + ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.1") with pytest.raises(ValueError): ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") diff --git a/stix2/v20/common.py b/stix2/v20/common.py index c2dedfb9..377d9926 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -49,7 +49,7 @@ class GranularMarking(_STIXBase): """ _properties = OrderedDict([ - ('marking_ref', ReferenceProperty(required=True, spec_version='2.0', type='marking-definition')), + ('marking_ref', ReferenceProperty(valid_types='marking-definition', spec_version='2.0', required=True)), ('selectors', ListProperty(SelectorProperty, required=True)), ]) @@ -105,10 +105,10 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ('definition_type', StringProperty(required=True)), ('definition', MarkingProperty(required=True)), diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 3fd3a84d..92880037 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -23,7 +23,7 @@ class AttackPattern(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -32,7 +32,7 @@ class AttackPattern(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -46,7 +46,7 @@ class Campaign(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -58,7 +58,7 @@ class Campaign(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -72,7 +72,7 @@ class CourseOfAction(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -80,7 +80,7 @@ class CourseOfAction(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -94,7 +94,7 @@ class Identity(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -105,7 +105,7 @@ class Identity(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -119,7 +119,7 @@ class Indicator(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -131,7 +131,7 @@ class Indicator(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -145,7 +145,7 @@ class IntrusionSet(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -160,7 +160,7 @@ class IntrusionSet(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -174,7 +174,7 @@ class Malware(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -183,7 +183,7 @@ class Malware(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -197,7 +197,7 @@ class ObservedData(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_observed', TimestampProperty(required=True)), @@ -207,7 +207,7 @@ class ObservedData(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -227,17 +227,17 @@ class Report(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(spec_version='2.0'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -251,7 +251,7 @@ class ThreatActor(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -267,7 +267,7 @@ class ThreatActor(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -281,7 +281,7 @@ class Tool(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -291,7 +291,7 @@ class Tool(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -305,7 +305,7 @@ class Vulnerability(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -313,7 +313,7 @@ class Vulnerability(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -352,7 +352,7 @@ def wrapper(cls): [ ('type', TypeProperty(type)), ('id', IDProperty(type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ], @@ -361,7 +361,7 @@ def wrapper(cls): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index f4500035..a21c871f 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -20,17 +20,17 @@ class Relationship(STIXRelationshipObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(spec_version='2.0', required=True)), - ('target_ref', ReferenceProperty(spec_version='2.0', required=True)), + ('source_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), + ('target_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -59,20 +59,20 @@ class Sighting(STIXRelationshipObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(spec_version='2.0', required=True)), - ('observed_data_refs', ListProperty(ReferenceProperty(type='observed-data', spec_version='2.0'))), - ('where_sighted_refs', ListProperty(ReferenceProperty(type='identity', spec_version='2.0'))), + ('sighting_of_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), + ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.0'))), + ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.0'))), ('summary', BooleanProperty(default=lambda: False)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 70be5dc3..e0859852 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -54,7 +54,7 @@ class GranularMarking(_STIXBase): _properties = OrderedDict([ ('lang', StringProperty()), - ('marking_ref', ReferenceProperty(type='marking-definition', spec_version='2.1')), + ('marking_ref', ReferenceProperty(valid_types='marking-definition', spec_version='2.1')), ('selectors', ListProperty(SelectorProperty, required=True)), ]) @@ -74,10 +74,10 @@ class LanguageContent(_STIXBase): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('object_ref', ReferenceProperty(spec_version='2.1', required=True)), + ('object_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), # TODO: 'object_modified' it MUST be an exact match for the modified time of the STIX Object (SRO or SDO) being referenced. ('object_modified', TimestampProperty(precision='millisecond')), # TODO: 'contents' https://docs.google.com/document/d/1ShNq4c3e1CkfANmD9O--mdZ5H0O_GLnjN28a_yrEaco/edit#heading=h.cfz5hcantmvx @@ -86,7 +86,7 @@ class LanguageContent(_STIXBase): ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -145,10 +145,10 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('definition_type', StringProperty(required=True)), ('definition', MarkingProperty(required=True)), diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index db7a3c78..f19d8ad5 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -7,16 +7,21 @@ from collections import OrderedDict import itertools +import warnings from ..base import _Extension, _Observable, _STIXBase from ..custom import _custom_extension_builder, _custom_observable_builder -from ..exceptions import AtLeastOnePropertyError, DependentPropertiesError +from ..exceptions import ( + AtLeastOnePropertyError, DependentPropertiesError, STIXDeprecationWarning, +) from ..properties import ( BinaryProperty, BooleanProperty, CallableValues, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty, - ObjectReferenceProperty, StringProperty, TimestampProperty, TypeProperty, + ObjectReferenceProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty, ) +from .common import GranularMarking class Artifact(_Observable): @@ -36,6 +41,10 @@ class Artifact(_Observable): ('encryption_algorithm', StringProperty()), ('decryption_key', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["hashes", "payload_bin"] @@ -59,6 +68,10 @@ class AutonomousSystem(_Observable): ('name', StringProperty()), ('rir', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["number"] @@ -79,8 +92,12 @@ class Directory(_Observable): ('ctime', TimestampProperty()), ('mtime', TimestampProperty()), ('atime', TimestampProperty()), - ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), + ('contains_refs', ListProperty(ReferenceProperty(valid_types=['file', 'directory'], spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["path"] @@ -96,11 +113,23 @@ class DomainName(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name'], spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] + def _check_object_constraints(self): + if self.get('resolves_to_refs'): + warnings.warn( + "The 'resolves_to_refs' property of domain-name is deprecated in " + "STIX 2.1. Use the 'resolves-to' relationship type instead", + STIXDeprecationWarning, + ) + class EmailAddress(_Observable): # TODO: Add link @@ -114,8 +143,12 @@ class EmailAddress(_Observable): ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('display_name', StringProperty()), - ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), + ('belongs_to_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] @@ -151,18 +184,23 @@ class EmailMessage(_Observable): ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), ('content_type', StringProperty()), - ('from_ref', ObjectReferenceProperty(valid_types='email-addr')), - ('sender_ref', ObjectReferenceProperty(valid_types='email-addr')), - ('to_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), - ('cc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), - ('bcc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('from_ref', ReferenceProperty(valid_types='email-addr')), + ('sender_ref', ReferenceProperty(valid_types='email-addr')), + ('to_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), + ('cc_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), + ('bcc_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), + ('message_id', StringProperty()), ('subject', StringProperty()), ('received_lines', ListProperty(StringProperty)), ('additional_header_fields', DictionaryProperty(spec_version='2.1')), ('body', StringProperty()), ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), - ('raw_email_ref', ObjectReferenceProperty(valid_types='artifact')), + ('raw_email_ref', ReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["from_ref", "subject", "body"] @@ -346,10 +384,14 @@ class File(_Observable): ('ctime', TimestampProperty()), ('mtime', TimestampProperty()), ('atime', TimestampProperty()), - ('parent_directory_ref', ObjectReferenceProperty(valid_types='directory')), - ('contains_refs', ListProperty(ObjectReferenceProperty)), - ('content_ref', ObjectReferenceProperty(valid_types='artifact')), + ('parent_directory_ref', ReferenceProperty(valid_types='directory')), + ('contains_refs', ListProperty(ReferenceProperty())), + ('content_ref', ReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["hashes", "name", "extensions"] @@ -369,12 +411,31 @@ class IPv4Address(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), - ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] + def _check_object_constraints(self): + if self.get('resolves_to_refs'): + warnings.warn( + "The 'resolves_to_refs' property of ipv4-addr is deprecated in " + "STIX 2.1. Use the 'resolves-to' relationship type instead", + STIXDeprecationWarning, + ) + + if self.get('belongs_to_refs'): + warnings.warn( + "The 'belongs_to_refs' property of ipv4-addr is deprecated in " + "STIX 2.1. Use the 'belongs-to' relationship type instead", + STIXDeprecationWarning, + ) + class IPv6Address(_Observable): # TODO: Add link @@ -387,12 +448,31 @@ class IPv6Address(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), - ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] + def _check_object_constraints(self): + if self.get('resolves_to_refs'): + warnings.warn( + "The 'resolves_to_refs' property of ipv6-addr is deprecated in " + "STIX 2.1. Use the 'resolves-to' relationship type instead", + STIXDeprecationWarning, + ) + + if self.get('belongs_to_refs'): + warnings.warn( + "The 'belongs_to_refs' property of ipv6-addr is deprecated in " + "STIX 2.1. Use the 'belongs-to' relationship type instead", + STIXDeprecationWarning, + ) + class MACAddress(_Observable): # TODO: Add link @@ -406,6 +486,10 @@ class MACAddress(_Observable): ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] @@ -422,6 +506,10 @@ class Mutex(_Observable): ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["name"] @@ -531,8 +619,8 @@ class NetworkTraffic(_Observable): ('start', TimestampProperty()), ('end', TimestampProperty()), ('is_active', BooleanProperty()), - ('src_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), - ('dst_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), + ('src_ref', ReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'], spec_version='2.1')), + ('dst_ref', ReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'], spec_version='2.1')), ('src_port', IntegerProperty(min=0, max=65535)), ('dst_port', IntegerProperty(min=0, max=65535)), ('protocols', ListProperty(StringProperty, required=True)), @@ -541,11 +629,15 @@ class NetworkTraffic(_Observable): ('src_packets', IntegerProperty(min=0)), ('dst_packets', IntegerProperty(min=0)), ('ipfix', DictionaryProperty(spec_version='2.1')), - ('src_payload_ref', ObjectReferenceProperty(valid_types='artifact')), - ('dst_payload_ref', ObjectReferenceProperty(valid_types='artifact')), - ('encapsulates_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), - ('encapsulates_by_ref', ObjectReferenceProperty(valid_types='network-traffic')), + ('src_payload_ref', ReferenceProperty(valid_types='artifact')), + ('dst_payload_ref', ReferenceProperty(valid_types='artifact')), + ('encapsulates_refs', ListProperty(ReferenceProperty(valid_types='network-traffic'))), + ('encapsulated_by_ref', ReferenceProperty(valid_types='network-traffic')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["start", "src_ref", "dst_ref", "src_port", "dst_port", "protocols"] @@ -652,16 +744,20 @@ class Process(_Observable): ('is_hidden', BooleanProperty()), ('pid', IntegerProperty()), # this is not the created timestamps of the object itself - ('created', TimestampProperty()), + ('created_time', TimestampProperty()), ('cwd', StringProperty()), ('command_line', StringProperty()), ('environment_variables', DictionaryProperty(spec_version='2.1')), - ('opened_connection_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), - ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), - ('image_ref', ObjectReferenceProperty(valid_types='file')), - ('parent_ref', ObjectReferenceProperty(valid_types='process')), - ('child_refs', ListProperty(ObjectReferenceProperty('process'))), + ('opened_connection_refs', ListProperty(ReferenceProperty(valid_types='network-traffic'))), + ('creator_user_ref', ReferenceProperty(valid_types='user-account')), + ('image_ref', ReferenceProperty(valid_types='file')), + ('parent_ref', ReferenceProperty(valid_types='process')), + ('child_refs', ListProperty(ReferenceProperty(valid_types='process'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = [] @@ -696,6 +792,10 @@ class Software(_Observable): ('vendor', StringProperty()), ('version', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["name", "cpe", "vendor", "version"] @@ -712,6 +812,10 @@ class URL(_Observable): ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] @@ -756,6 +860,10 @@ class UserAccount(_Observable): ('account_first_login', TimestampProperty()), ('account_last_login', TimestampProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["account_type", "user_id", "account_login"] @@ -804,9 +912,13 @@ class WindowsRegistryKey(_Observable): ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself ('modified_time', TimestampProperty()), - ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), + ('creator_user_ref', ReferenceProperty(valid_types='user-account')), ('number_of_subkeys', IntegerProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["key", "values"] @@ -867,6 +979,10 @@ class X509Certificate(_Observable): ('subject_public_key_exponent', IntegerProperty()), ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["hashes", "serial_number"] diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 662007c4..e9714e7f 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -30,7 +30,7 @@ class AttackPattern(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -42,7 +42,7 @@ class AttackPattern(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -58,7 +58,7 @@ class Campaign(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -72,7 +72,7 @@ class Campaign(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -98,7 +98,7 @@ class CourseOfAction(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -112,7 +112,7 @@ class CourseOfAction(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -138,13 +138,13 @@ class Grouping(STIXDomainObject): ('id', IDProperty(_type, spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('name', StringProperty()), ('description', StringProperty()), @@ -164,7 +164,7 @@ class Identity(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -178,7 +178,7 @@ class Identity(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -194,7 +194,7 @@ class Indicator(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -211,7 +211,7 @@ class Indicator(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -237,7 +237,7 @@ class Infrastructure(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('revoked', BooleanProperty(default=lambda: False)), @@ -245,7 +245,7 @@ class Infrastructure(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('name', StringProperty(required=True)), ('description', StringProperty()), @@ -278,7 +278,7 @@ class IntrusionSet(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -295,7 +295,7 @@ class IntrusionSet(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -321,7 +321,7 @@ class Location(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -340,7 +340,7 @@ class Location(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -429,7 +429,7 @@ class Malware(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -444,13 +444,13 @@ class Malware(STIXDomainObject): ('architecture_execution_envs', ListProperty(StringProperty)), ('implementation_languages', ListProperty(StringProperty)), ('capabilities', ListProperty(StringProperty)), - ('sample_refs', ListProperty(ReferenceProperty(spec_version='2.1'))), + ('sample_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.1'))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -484,19 +484,19 @@ class MalwareAnalysis(STIXDomainObject): ('id', IDProperty(_type, spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('product', StringProperty(required=True)), ('version', StringProperty()), - ('host_vm_ref', ReferenceProperty(type='software', spec_version='2.1')), - ('operating_system_ref', ReferenceProperty(type='software', spec_version='2.1')), - ('installed_software_refs', ListProperty(ReferenceProperty(type='software', spec_version='2.1'))), + ('host_vm_ref', ReferenceProperty(valid_types='software', spec_version='2.1')), + ('operating_system_ref', ReferenceProperty(valid_types='software', spec_version='2.1')), + ('installed_software_refs', ListProperty(ReferenceProperty(valid_types='software', spec_version='2.1'))), ('configuration_version', StringProperty()), ('modules', ListProperty(StringProperty)), ('analysis_engine_version', StringProperty()), @@ -505,7 +505,7 @@ class MalwareAnalysis(STIXDomainObject): ('analysis_started', TimestampProperty()), ('analysis_ended', TimestampProperty()), ('av_result', StringProperty()), - ('analysis_sco_refs', ListProperty(ReferenceProperty(spec_version='2.1'))), + ('analysis_sco_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.1'))), ]) def _check_object_constraints(self): @@ -525,7 +525,7 @@ class Note(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('abstract', StringProperty()), @@ -537,7 +537,7 @@ class Note(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -553,20 +553,20 @@ class ObservedData(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_observed', TimestampProperty(required=True)), ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(min=1, max=999999999, required=True)), ('objects', ObservableProperty(spec_version='2.1')), - ('object_refs', ListProperty(ReferenceProperty(spec_version="2.1"))), + ('object_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version="2.1"))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -609,7 +609,7 @@ class Opinion(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('explanation', StringProperty()), @@ -631,7 +631,7 @@ class Opinion(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -647,7 +647,7 @@ class Report(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -660,7 +660,7 @@ class Report(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -676,7 +676,7 @@ class ThreatActor(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -697,7 +697,7 @@ class ThreatActor(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -723,7 +723,7 @@ class Tool(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -737,7 +737,7 @@ class Tool(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -753,7 +753,7 @@ class Vulnerability(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -763,7 +763,7 @@ class Vulnerability(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -803,7 +803,7 @@ def wrapper(cls): ('type', TypeProperty(type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ], @@ -814,7 +814,7 @@ def wrapper(cls): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 149094ec..37f0024d 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -22,13 +22,13 @@ class Relationship(STIXRelationshipObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(spec_version='2.1', required=True)), - ('target_ref', ReferenceProperty(spec_version='2.1', required=True)), + ('source_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('target_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), ('start_time', TimestampProperty()), ('stop_time', TimestampProperty()), ('revoked', BooleanProperty(default=lambda: False)), @@ -36,7 +36,7 @@ class Relationship(STIXRelationshipObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -77,23 +77,23 @@ class Sighting(STIXRelationshipObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('description', StringProperty()), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(spec_version='2.1', required=True)), - ('observed_data_refs', ListProperty(ReferenceProperty(type='observed-data', spec_version='2.1'))), - ('where_sighted_refs', ListProperty(ReferenceProperty(type='identity', spec_version='2.1'))), + ('sighting_of_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.1'))), + ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.1'))), ('summary', BooleanProperty()), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) From f69b13a006e63c0cc36fb82a3ecb8af9c1e79f52 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Thu, 29 Aug 2019 17:15:51 -0400 Subject: [PATCH 09/17] Some more updates, primarily to ReferenceProperty (and related code) --- stix2/properties.py | 33 +++++++++++++++++++++++++++---- stix2/test/v20/test_properties.py | 2 +- stix2/test/v21/test_properties.py | 2 +- stix2/v20/sdo.py | 2 +- stix2/v20/sro.py | 8 +++++--- stix2/v21/common.py | 2 +- stix2/v21/observables.py | 2 +- stix2/v21/sdo.py | 2 +- stix2/v21/sro.py | 8 +++++--- 9 files changed, 45 insertions(+), 16 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 4eb5014c..a217950b 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -15,7 +15,10 @@ from .base import _Observable, _STIXBase from .core import STIX2_OBJ_MAPS, parse, parse_observable -from .exceptions import CustomContentError, DictionaryKeyError +from .exceptions import ( + CustomContentError, DictionaryKeyError, MissingPropertiesError, + MutuallyExclusivePropertiesError, +) from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime ERROR_INVALID_ID = ( @@ -417,15 +420,27 @@ def clean(self, value): class ReferenceProperty(Property): - def __init__(self, valid_types=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): + def __init__(self, valid_types=None, invalid_types=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): """ references sometimes must be to a specific object type """ self.spec_version = spec_version + # These checks need to be done prior to the STIX object finishing construction + # and thus we can't use base.py's _check_mutually_exclusive_properties() + # in the typical location of _check_object_constraints() in sdo.py + if valid_types and invalid_types: + raise MutuallyExclusivePropertiesError(self.__class__, ['invalid_types', 'valid_types']) + elif valid_types is None and invalid_types is None: + raise MissingPropertiesError(self.__class__, ['invalid_types', 'valid_types']) + if valid_types and type(valid_types) is not list: valid_types = [valid_types] + elif invalid_types and type(invalid_types) is not list: + invalid_types = [invalid_types] + self.valid_types = valid_types + self.invalid_types = invalid_types super(ReferenceProperty, self).__init__(**kwargs) @@ -434,8 +449,18 @@ def clean(self, value): value = value.id value = str(value) - if self.valid_types and value[:value.index('--')] in self.valid_types: - required_prefix = value[:value.index('--') + 2] + possible_prefix = value[:value.index('--') + 2] + + if self.valid_types: + if possible_prefix in self.valid_types: + required_prefix = possible_prefix + else: + raise ValueError("The type-specifying prefix for this identifier is invalid") + elif self.invalid_types: + if possible_prefix not in self.invalid_types: + required_prefix = possible_prefix + else: + raise ValueError("The type-specifying prefix for this identifier is invalid") _validate_id(value, self.spec_version, required_prefix) diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 548e7fe0..f71d8296 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -276,7 +276,7 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty(valid_types=None, spec_version="2.0") + ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.0") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 41ca3e9b..1fb3cc41 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -271,7 +271,7 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty(valid_types=None, spec_version="2.1") + ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.1") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 92880037..899ccd80 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -233,7 +233,7 @@ class Report(STIXDomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.0'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types="", spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index a21c871f..3b1e582e 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -16,6 +16,8 @@ class Relationship(STIXRelationshipObject): `the STIX 2.0 specification `__. """ + _invalid_source_target_types = ['bundle', 'language-content', 'marking-definition', 'relationship', 'sighting'] + _type = 'relationship' _properties = OrderedDict([ ('type', TypeProperty(_type)), @@ -25,8 +27,8 @@ class Relationship(STIXRelationshipObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), - ('target_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), + ('source_ref', ReferenceProperty(invalid_types=_invalid_source_target_types, spec_version='2.0', required=True)), + ('target_ref', ReferenceProperty(invalid_types=_invalid_source_target_types, spec_version='2.0', required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), @@ -65,7 +67,7 @@ class Sighting(STIXRelationshipObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), + ('sighting_of_ref', ReferenceProperty(valid_types="Left to user", spec_version='2.0', required=True)), ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.0'))), ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.0'))), ('summary', BooleanProperty(default=lambda: False)), diff --git a/stix2/v21/common.py b/stix2/v21/common.py index e0859852..44d36753 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -77,7 +77,7 @@ class LanguageContent(_STIXBase): ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('object_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('object_ref', ReferenceProperty(invalid_types=[""], spec_version='2.1', required=True)), # TODO: 'object_modified' it MUST be an exact match for the modified time of the STIX Object (SRO or SDO) being referenced. ('object_modified', TimestampProperty(precision='millisecond')), # TODO: 'contents' https://docs.google.com/document/d/1ShNq4c3e1CkfANmD9O--mdZ5H0O_GLnjN28a_yrEaco/edit#heading=h.cfz5hcantmvx diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index f19d8ad5..fbce8fd6 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -385,7 +385,7 @@ class File(_Observable): ('mtime', TimestampProperty()), ('atime', TimestampProperty()), ('parent_directory_ref', ReferenceProperty(valid_types='directory')), - ('contains_refs', ListProperty(ReferenceProperty())), + ('contains_refs', ListProperty(ReferenceProperty(invalid_types=[""]))), ('content_ref', ReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index e9714e7f..cb7eff3e 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -444,7 +444,7 @@ class Malware(STIXDomainObject): ('architecture_execution_envs', ListProperty(StringProperty)), ('implementation_languages', ListProperty(StringProperty)), ('capabilities', ListProperty(StringProperty)), - ('sample_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.1'))), + ('sample_refs', ListProperty(ReferenceProperty(valid_types=['artifact', 'file'], spec_version='2.1'))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 37f0024d..31f9c4c0 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -17,6 +17,8 @@ class Relationship(STIXRelationshipObject): `the STIX 2.1 specification `__. """ + _invalid_source_target_types = ['bundle', 'language-content', 'marking-definition', 'relationship', 'sighting'] + _type = 'relationship' _properties = OrderedDict([ ('type', TypeProperty(_type)), @@ -27,8 +29,8 @@ class Relationship(STIXRelationshipObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), - ('target_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('source_ref', ReferenceProperty(valid_types=_invalid_source_target_types, spec_version='2.1', required=True)), + ('target_ref', ReferenceProperty(valid_types=_invalid_source_target_types, spec_version='2.1', required=True)), ('start_time', TimestampProperty()), ('stop_time', TimestampProperty()), ('revoked', BooleanProperty(default=lambda: False)), @@ -84,7 +86,7 @@ class Sighting(STIXRelationshipObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('sighting_of_ref', ReferenceProperty(valid_types="Left to user", spec_version='2.1', required=True)), ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.1'))), ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.1'))), ('summary', BooleanProperty()), From 44ebd64a1626674c1f677eac3f62d20742f69470 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 30 Aug 2019 03:47:47 -0400 Subject: [PATCH 10/17] Some test fixes. More coming soon --- stix2/base.py | 3 ++- stix2/properties.py | 15 +++++++++++---- stix2/test/v21/test_utils.py | 6 ++++-- stix2/v20/sro.py | 2 +- stix2/v21/sdo.py | 26 ++++++++++++++++++-------- stix2/v21/sro.py | 6 +++--- 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 229f78f7..bae1f2c9 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -327,7 +327,8 @@ def _check_ref(self, ref, prop, prop_name): return # don't check if refs are valid if ref not in self._STIXBase__valid_refs: - raise InvalidObjRefError(self.__class__, prop_name, "'%s' is not a valid object in local scope" % ref) + if ref[:ref.index('--') + 2] not in self._STIXBase__valid_refs: + raise InvalidObjRefError(self.__class__, prop_name, "'%s' is not a valid object in local scope" % ref) try: allowed_types = prop.contained.valid_types diff --git a/stix2/properties.py b/stix2/properties.py index a217950b..e8c84422 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -452,15 +452,22 @@ def clean(self, value): possible_prefix = value[:value.index('--') + 2] if self.valid_types: - if possible_prefix in self.valid_types: + if self.valid_types == ["only_SDO"]: + self.valid_types = STIX2_OBJ_MAPS['v21']['objects'].keys() + elif self.valid_types == ["only_SCO"]: + self.valid_types = STIX2_OBJ_MAPS['v21']['observables'].keys() + elif self.valid_types == ["only_SCO_&_SRO"]: + self.valid_types = STIX2_OBJ_MAPS['v21']['observables'].keys() + ['relationship', 'sighting'] + + if possible_prefix[:-2] in self.valid_types: required_prefix = possible_prefix else: - raise ValueError("The type-specifying prefix for this identifier is invalid") + raise ValueError("The type-specifying prefix for this identifier is not valid") elif self.invalid_types: - if possible_prefix not in self.invalid_types: + if possible_prefix[:-2] not in self.invalid_types: required_prefix = possible_prefix else: - raise ValueError("The type-specifying prefix for this identifier is invalid") + raise ValueError("An invalid type-specifying prefix was specified for this identifier") _validate_id(value, self.spec_version, required_prefix) diff --git a/stix2/test/v21/test_utils.py b/stix2/test/v21/test_utils.py index dec32943..5cf88e4b 100644 --- a/stix2/test/v21/test_utils.py +++ b/stix2/test/v21/test_utils.py @@ -135,14 +135,16 @@ def test_deduplicate(stix_objs1): "0": { "name": "foo.exe", "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", }, "1": { "type": "ipv4-addr", "value": "198.51.100.3", + "id": "ipv4-addr--1f8f4d63-9f33-5353-a3e3-e1b84c83a7b5", }, "2": { "type": "network-traffic", - "src_ref": "1", + "src_ref": "ipv4-addr--1f8f4d63-9f33-5353-a3e3-e1b84c83a7b5", "protocols": [ "tcp", "http", @@ -161,7 +163,7 @@ def test_deduplicate(stix_objs1): }, }, }, - ), ('1', {"type": "ipv4-addr", "value": "198.51.100.3"}), 1, + ), ('1', {"type": "ipv4-addr", "value": "198.51.100.3", "id": "ipv4-addr--1f8f4d63-9f33-5353-a3e3-e1b84c83a7b5"}), 1, ), ( { diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index 3b1e582e..3f561b86 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -67,7 +67,7 @@ class Sighting(STIXRelationshipObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(valid_types="Left to user", spec_version='2.0', required=True)), + ('sighting_of_ref', ReferenceProperty(valid_types="only_SDO", spec_version='2.0', required=True)), ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.0'))), ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.0'))), ('summary', BooleanProperty(default=lambda: False)), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index cb7eff3e..30626592 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -6,9 +6,11 @@ from six.moves.urllib.parse import quote_plus -from ..core import STIXDomainObject +from ..core import STIX2_OBJ_MAPS, STIXDomainObject from ..custom import _custom_object_builder -from ..exceptions import PropertyPresenceError, STIXDeprecationWarning +from ..exceptions import ( + InvalidValueError, PropertyPresenceError, STIXDeprecationWarning, +) from ..properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, ListProperty, @@ -149,7 +151,7 @@ class Grouping(STIXDomainObject): ('name', StringProperty()), ('description', StringProperty()), ('context', StringProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), ]) @@ -505,7 +507,7 @@ class MalwareAnalysis(STIXDomainObject): ('analysis_started', TimestampProperty()), ('analysis_ended', TimestampProperty()), ('av_result', StringProperty()), - ('analysis_sco_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.1'))), + ('analysis_sco_refs', ListProperty(ReferenceProperty(valid_types="only_SCO", spec_version='2.1'))), ]) def _check_object_constraints(self): @@ -531,7 +533,7 @@ class Note(STIXDomainObject): ('abstract', StringProperty()), ('content', StringProperty(required=True)), ('authors', ListProperty(StringProperty)), - ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -560,7 +562,7 @@ class ObservedData(STIXDomainObject): ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(min=1, max=999999999, required=True)), ('objects', ObservableProperty(spec_version='2.1')), - ('object_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version="2.1"))), + ('object_refs', ListProperty(ReferenceProperty(valid_types="only_SCO_&_SRO", spec_version="2.1"))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -597,6 +599,14 @@ def _check_object_constraints(self): ["objects", "object_refs"], ) + if self.get('object_refs'): + for identifier in self.get('object_refs'): + identifier_prefix = identifier[:identifier.index('--') + 2] + if identifier_prefix in STIX2_OBJ_MAPS['v21']['observables'].keys(): + break + else: + raise InvalidValueError(self.__class__, 'object_refs', "At least one identifier must be of a SCO type if this property specified") + class Opinion(STIXDomainObject): # TODO: Add link @@ -625,7 +635,7 @@ class Opinion(STIXDomainObject): ], required=True, ), ), - ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -654,7 +664,7 @@ class Report(STIXDomainObject): ('description', StringProperty()), ('report_types', ListProperty(StringProperty, required=True)), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 31f9c4c0..57c7719d 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -29,8 +29,8 @@ class Relationship(STIXRelationshipObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(valid_types=_invalid_source_target_types, spec_version='2.1', required=True)), - ('target_ref', ReferenceProperty(valid_types=_invalid_source_target_types, spec_version='2.1', required=True)), + ('source_ref', ReferenceProperty(invalid_types=_invalid_source_target_types, spec_version='2.1', required=True)), + ('target_ref', ReferenceProperty(invalid_types=_invalid_source_target_types, spec_version='2.1', required=True)), ('start_time', TimestampProperty()), ('stop_time', TimestampProperty()), ('revoked', BooleanProperty(default=lambda: False)), @@ -86,7 +86,7 @@ class Sighting(STIXRelationshipObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(valid_types="Left to user", spec_version='2.1', required=True)), + ('sighting_of_ref', ReferenceProperty(valid_types="only_SDO", spec_version='2.1', required=True)), ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.1'))), ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.1'))), ('summary', BooleanProperty()), From abf29803367b5727149c7f37c4661ceea86d45f7 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 4 Sep 2019 19:08:34 -0400 Subject: [PATCH 11/17] Fix tests and ReferenceProperty --- stix2/base.py | 32 +++-- stix2/properties.py | 4 +- stix2/test/v20/test_utils.py | 4 +- stix2/test/v21/test_core.py | 4 +- stix2/test/v21/test_datastore_filters.py | 6 +- stix2/test/v21/test_observed_data.py | 154 ++++++++++++----------- stix2/v20/sdo.py | 2 +- stix2/v21/observables.py | 46 +++---- stix2/v21/sdo.py | 10 +- 9 files changed, 137 insertions(+), 125 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index bae1f2c9..5bc3d107 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -115,15 +115,15 @@ def _check_mutually_exclusive_properties(self, list_of_properties, at_least_one= def _check_at_least_one_property(self, list_of_properties=None): if not list_of_properties: list_of_properties = sorted(list(self.__class__._properties.keys())) - if 'type' in list_of_properties: - list_of_properties.remove('type') + if isinstance(self, _Observable): + props_to_remove = ["type", "id", "defanged", "spec_version"] + else: + props_to_remove = ["type"] + + list_of_properties = [prop for prop in list_of_properties if prop not in props_to_remove] current_properties = self.properties_populated() list_of_properties_populated = set(list_of_properties).intersection(current_properties) - if list_of_properties_populated == set(['id']) and isinstance(self, _Observable): - # Do not count the auto-generated id as a user-specified property - list_of_properties_populated = None - if list_of_properties and (not list_of_properties_populated or list_of_properties_populated == set(['extensions'])): raise AtLeastOnePropertyError(self.__class__, list_of_properties) @@ -327,8 +327,7 @@ def _check_ref(self, ref, prop, prop_name): return # don't check if refs are valid if ref not in self._STIXBase__valid_refs: - if ref[:ref.index('--') + 2] not in self._STIXBase__valid_refs: - raise InvalidObjRefError(self.__class__, prop_name, "'%s' is not a valid object in local scope" % ref) + raise InvalidObjRefError(self.__class__, prop_name, "'%s' is not a valid object in local scope" % ref) try: allowed_types = prop.contained.valid_types @@ -352,12 +351,14 @@ def _check_property(self, prop_name, prop, kwargs): if prop_name not in kwargs: return - if prop_name.endswith('_ref'): - ref = kwargs[prop_name] - self._check_ref(ref, prop, prop_name) - elif prop_name.endswith('_refs'): - for ref in kwargs[prop_name]: + from .properties import ObjectReferenceProperty + if isinstance(prop, ObjectReferenceProperty): + if prop_name.endswith('_ref'): + ref = kwargs[prop_name] self._check_ref(ref, prop, prop_name) + elif prop_name.endswith('_refs'): + for ref in kwargs[prop_name]: + self._check_ref(ref, prop, prop_name) def _generate_id(self, kwargs): required_prefix = self._type + "--" @@ -376,6 +377,11 @@ def _generate_id(self, kwargs): temp_deep_copy = copy.deepcopy(dict(kwargs[key])) _recursive_stix_to_dict(temp_deep_copy) streamlined_obj_vals.append(temp_deep_copy) + elif isinstance(kwargs[key], list) and isinstance(kwargs[key][0], _STIXBase): + for obj in kwargs[key]: + temp_deep_copy = copy.deepcopy(dict(obj)) + _recursive_stix_to_dict(temp_deep_copy) + streamlined_obj_vals.append(temp_deep_copy) else: streamlined_obj_vals.append(kwargs[key]) diff --git a/stix2/properties.py b/stix2/properties.py index e8c84422..99dc0a1b 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -462,12 +462,12 @@ def clean(self, value): if possible_prefix[:-2] in self.valid_types: required_prefix = possible_prefix else: - raise ValueError("The type-specifying prefix for this identifier is not valid") + raise ValueError("The type-specifying prefix '%s' for the identifier '%s' is not valid" % (possible_prefix, value)) elif self.invalid_types: if possible_prefix[:-2] not in self.invalid_types: required_prefix = possible_prefix else: - raise ValueError("An invalid type-specifying prefix was specified for this identifier") + raise ValueError("An invalid type-specifying prefix '%s' was specified for the identifier '%s'" % (possible_prefix, value)) _validate_id(value, self.spec_version, required_prefix) diff --git a/stix2/test/v20/test_utils.py b/stix2/test/v20/test_utils.py index ee011c1a..0433fd56 100644 --- a/stix2/test/v20/test_utils.py +++ b/stix2/test/v20/test_utils.py @@ -144,8 +144,8 @@ def test_deduplicate(stix_objs1): "type": "network-traffic", "src_ref": "1", "protocols": [ - "tcp", - "http", + "tcp", + "http", ], "extensions": { "http-request-ext": { diff --git a/stix2/test/v21/test_core.py b/stix2/test/v21/test_core.py index 25348cd9..2018395e 100644 --- a/stix2/test/v21/test_core.py +++ b/stix2/test/v21/test_core.py @@ -127,7 +127,7 @@ def test_register_observable_with_default_version(): "1": { "type": "directory", "path": "/usr/home", - "contains_refs": ["0"], + "contains_refs": ["file--420bc087-8b53-5ae9-8210-20d27d5e96c8"], }, }, ) @@ -165,7 +165,7 @@ def test_register_observable_extension_with_default_version(): "1": { "type": "directory", "path": "/usr/home", - "contains_refs": ["0"], + "contains_refs": ["file--420bc087-8b53-5ae9-8210-20d27d5e96c8"], }, }, ) diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index 92d035ba..9908fb3b 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -86,7 +86,7 @@ "objects": { "0": { "type": "file", - "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec", + "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", }, }, @@ -110,8 +110,8 @@ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe", "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec"}}), - Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe", "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec"}), + Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876"}}), + Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876"}), Filter("labels", "contains", "heartbleed"), ] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index dc27058e..951943aa 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -26,7 +26,7 @@ "objects": { "0": { "type": "file", - "id": "file--500d9a03-9d03-5c31-82b2-2be8aacec481", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "name": "foo.exe" } } @@ -45,6 +45,7 @@ def test_observed_data_example(): objects={ "0": { "name": "foo.exe", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "type": "file", }, }, @@ -66,15 +67,15 @@ def test_observed_data_example(): "objects": { "0": { "type": "file", - "id": "file--500d9a03-9d03-5c31-82b2-2be8aacec481", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "name": "foo.exe" }, "1": { "type": "directory", - "id": "directory--ed959127-2df3-5999-99b6-df7614398c1c", + "id": "directory--536a61a4-0934-516b-9aad-fcbb75e0583a", "path": "/usr/home", "contains_refs": [ - "0" + "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f" ] } } @@ -93,12 +94,14 @@ def test_observed_data_example_with_refs(): objects={ "0": { "name": "foo.exe", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "type": "file", }, "1": { "type": "directory", + "id": "directory--536a61a4-0934-516b-9aad-fcbb75e0583a", "path": "/usr/home", - "contains_refs": ["0"], + "contains_refs": ["file--5956efbb-a7b0-566d-a7f9-a202eb05c70f"], }, }, ) @@ -117,9 +120,9 @@ def test_observed_data_example_with_refs(): "last_observed": "2015-12-21T19:00:00Z", "number_observed": 50, "object_refs": [ - "foo--758bf2c0-a6f1-56d1-872e-6b727467739a", - "bar--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", - "baz--eca0b3ba-8d76-11e9-a1fd-34415dabec0c" + "file--758bf2c0-a6f1-56d1-872e-6b727467739a", + "url--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", + "mutex--eca0b3ba-8d76-11e9-a1fd-34415dabec0c" ] }""" @@ -134,9 +137,9 @@ def test_observed_data_example_with_object_refs(): last_observed="2015-12-21T19:00:00Z", number_observed=50, object_refs=[ - "foo--758bf2c0-a6f1-56d1-872e-6b727467739a", - "bar--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", - "baz--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", + "file--758bf2c0-a6f1-56d1-872e-6b727467739a", + "url--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", + "mutex--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", ], ) @@ -160,15 +163,15 @@ def test_observed_data_object_constraint(): }, }, object_refs=[ - "foo--758bf2c0-a6f1-56d1-872e-6b727467739a", - "bar--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", - "baz--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", + "file--758bf2c0-a6f1-56d1-872e-6b727467739a", + "url--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", + "mutex--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", ], ) def test_observed_data_example_with_bad_refs(): - with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + with pytest.raises(ValueError) as excinfo: stix2.v21.ObservedData( id=OBSERVED_DATA_ID, created_by_ref=IDENTITY_ID, @@ -180,19 +183,20 @@ def test_observed_data_example_with_bad_refs(): objects={ "0": { "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "name": "foo.exe", }, "1": { "type": "directory", "path": "/usr/home", - "contains_refs": ["2"], + "contains_refs": ["monkey--5956efbb-a7b0-566d-a7f9-a202eb05c70f"], }, }, ) - assert excinfo.value.cls == stix2.v21.ObservedData - assert excinfo.value.prop_name == "objects" - assert excinfo.value.reason == "Invalid object reference for 'Directory:contains_refs': '2' is not a valid object in local scope" + assert excinfo.value.cls == stix2.v21.Directory + assert excinfo.value.prop_name == "contains_refs" + assert "The type-specifying prefix 'monkey--' for the identifier" in excinfo.value.reason def test_observed_data_example_with_non_dictionary(): @@ -248,6 +252,7 @@ def test_observed_data_example_with_empty_dictionary(): "0": { "name": "foo.exe", "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", }, }, }, @@ -640,16 +645,18 @@ def test_observed_data_with_process_example(): objects={ "0": { "type": "file", + "id": "file--0d16c8d3-c177-5f5d-a022-b1bdac329bea", "hashes": { "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f", }, }, "1": { "type": "process", + "id": "process--f6c4a02c-23e1-4a6d-a0d7-d862e893817a", "pid": 1221, - "created": "2016-01-20T14:11:25.55Z", + "created_time": "2016-01-20T14:11:25.55Z", "command_line": "./gedit-bin --new-window", - "image_ref": "0", + "image_ref": "file--0d16c8d3-c177-5f5d-a022-b1bdac329bea", }, }, ) @@ -693,31 +700,33 @@ def test_artifact_mutual_exclusion_error(): def test_directory_example(): - dir = stix2.v21.Directory( - _valid_refs={"1": "file"}, + f = stix2.v21.File( + name="penguin.exe", + ) + + dir1 = stix2.v21.Directory( path='/usr/lib', ctime="2015-12-21T19:00:00Z", mtime="2015-12-24T19:00:00Z", atime="2015-12-21T20:00:00Z", - contains_refs=["1"], + contains_refs=[str(f.id)], ) - assert dir.path == '/usr/lib' - assert dir.ctime == dt.datetime(2015, 12, 21, 19, 0, 0, tzinfo=pytz.utc) - assert dir.mtime == dt.datetime(2015, 12, 24, 19, 0, 0, tzinfo=pytz.utc) - assert dir.atime == dt.datetime(2015, 12, 21, 20, 0, 0, tzinfo=pytz.utc) - assert dir.contains_refs == ["1"] + assert dir1.path == '/usr/lib' + assert dir1.ctime == dt.datetime(2015, 12, 21, 19, 0, 0, tzinfo=pytz.utc) + assert dir1.mtime == dt.datetime(2015, 12, 24, 19, 0, 0, tzinfo=pytz.utc) + assert dir1.atime == dt.datetime(2015, 12, 21, 20, 0, 0, tzinfo=pytz.utc) + assert dir1.contains_refs == ["file--9d050a3b-72cd-5b57-bf18-024e74e1e5eb"] def test_directory_example_ref_error(): - with pytest.raises(stix2.exceptions.InvalidObjRefError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.Directory( - _valid_refs=[], path='/usr/lib', ctime="2015-12-21T19:00:00Z", mtime="2015-12-24T19:00:00Z", atime="2015-12-21T20:00:00Z", - contains_refs=["1"], + contains_refs=["domain-name--02af94ea-7e38-5718-87c3-5cc023e3d49d"], ) assert excinfo.value.cls == stix2.v21.Directory @@ -725,22 +734,24 @@ def test_directory_example_ref_error(): def test_domain_name_example(): - dn = stix2.v21.DomainName( - _valid_refs={"1": 'domain-name'}, + dn1 = stix2.v21.DomainName( + value="mitre.org", + ) + + dn2 = stix2.v21.DomainName( value="example.com", - resolves_to_refs=["1"], + resolves_to_refs=[str(dn1.id)], ) - assert dn.value == "example.com" - assert dn.resolves_to_refs == ["1"] + assert dn2.value == "example.com" + assert dn2.resolves_to_refs == ["domain-name--02af94ea-7e38-5718-87c3-5cc023e3d49d"] def test_domain_name_example_invalid_ref_type(): - with pytest.raises(stix2.exceptions.InvalidObjRefError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.DomainName( - _valid_refs={"1": "file"}, value="example.com", - resolves_to_refs=["1"], + resolves_to_refs=["file--44a431e6-764b-5556-a3f5-bf655930a581"], ) assert excinfo.value.cls == stix2.v21.DomainName @@ -882,6 +893,7 @@ def test_file_example_with_RasterImageExt_Object(): "objects": { "0": { "type": "file", + "id": "file--44a431e6-764b-5556-a3f5-bf655930a581", "name": "picture.jpg", "hashes": { "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" @@ -993,18 +1005,17 @@ def test_file_example_encryption_error(): assert "At least one of the (hashes, name)" in str(excinfo.value) -def test_ip4_address_example(): +def test_ipv4_address_example(): ip4 = stix2.v21.IPv4Address( - _valid_refs={"4": "mac-addr", "5": "mac-addr"}, value="198.51.100.3", - resolves_to_refs=["4", "5"], + resolves_to_refs=["mac-addr--a85820f7-d9b7-567a-a3a6-dedc34139342", "mac-addr--9a59b496-fdeb-510f-97b5-7137210bc699"], ) assert ip4.value == "198.51.100.3" - assert ip4.resolves_to_refs == ["4", "5"] + assert ip4.resolves_to_refs == ["mac-addr--a85820f7-d9b7-567a-a3a6-dedc34139342", "mac-addr--9a59b496-fdeb-510f-97b5-7137210bc699"] -def test_ip4_address_valid_refs(): +def test_ipv4_address_valid_refs(): mac1 = stix2.v21.MACAddress( value="a1:b2:c3:d4:e5:f6", ) @@ -1013,22 +1024,21 @@ def test_ip4_address_valid_refs(): ) ip4 = stix2.v21.IPv4Address( - _valid_refs={"1": mac1, "2": mac2}, value="177.60.40.7", - resolves_to_refs=["1", "2"], + resolves_to_refs=[str(mac1.id), str(mac2.id)], ) assert ip4.value == "177.60.40.7" - assert ip4.resolves_to_refs == ["1", "2"] + assert ip4.resolves_to_refs == ["mac-addr--a85820f7-d9b7-567a-a3a6-dedc34139342", "mac-addr--9a59b496-fdeb-510f-97b5-7137210bc699"] -def test_ip4_address_example_cidr(): +def test_ipv4_address_example_cidr(): ip4 = stix2.v21.IPv4Address(value="198.51.100.0/24") assert ip4.value == "198.51.100.0/24" -def test_ip6_address_example(): +def test_ipv6_address_example(): ip6 = stix2.v21.IPv6Address(value="2001:0db8:85a3:0000:0000:8a2e:0370:7334") assert ip6.value == "2001:0db8:85a3:0000:0000:8a2e:0370:7334" @@ -1042,14 +1052,13 @@ def test_mac_address_example(): def test_network_traffic_example(): nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr", "1": "ipv4-addr"}, - protocols="tcp", - src_ref="0", - dst_ref="1", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", + dst_ref="ipv4-addr--6d39dd0b-1f74-5faf-8d76-d8762c2a57cb", ) assert nt.protocols == ["tcp"] - assert nt.src_ref == "0" - assert nt.dst_ref == "1" + assert nt.src_ref == "ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88" + assert nt.dst_ref == "ipv4-addr--6d39dd0b-1f74-5faf-8d76-d8762c2a57cb" def test_network_traffic_http_request_example(): @@ -1064,9 +1073,8 @@ def test_network_traffic_http_request_example(): }, ) nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr"}, - protocols="tcp", - src_ref="0", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", extensions={'http-request-ext': h}, ) assert nt.extensions['http-request-ext'].request_method == "get" @@ -1080,9 +1088,8 @@ def test_network_traffic_http_request_example(): def test_network_traffic_icmp_example(): h = stix2.v21.ICMPExt(icmp_type_hex="08", icmp_code_hex="00") nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr"}, - protocols="tcp", - src_ref="0", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", extensions={'icmp-ext': h}, ) assert nt.extensions['icmp-ext'].icmp_type_hex == "08" @@ -1097,9 +1104,8 @@ def test_network_traffic_socket_example(): socket_type="SOCK_STREAM", ) nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr"}, - protocols="tcp", - src_ref="0", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", extensions={'socket-ext': h}, ) assert nt.extensions['socket-ext'].is_listening @@ -1111,9 +1117,8 @@ def test_network_traffic_socket_example(): def test_network_traffic_tcp_example(): h = stix2.v21.TCPExt(src_flags_hex="00000002") nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr"}, - protocols="tcp", - src_ref="0", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", extensions={'tcp-ext': h}, ) assert nt.extensions['tcp-ext'].src_flags_hex == "00000002" @@ -1127,11 +1132,10 @@ def test_mutex_example(): def test_process_example(): p = stix2.v21.Process( - _valid_refs={"0": "file"}, pid=1221, - created="2016-01-20T14:11:25.55Z", + created_time="2016-01-20T14:11:25.55Z", command_line="./gedit-bin --new-window", - image_ref="0", + image_ref="file--ea587d87-5ed2-5625-a9ac-01fd64161fd8", ) assert p.command_line == "./gedit-bin --new-window" @@ -1143,7 +1147,7 @@ def test_process_example_empty_error(): assert excinfo.value.cls == stix2.v21.Process properties_of_process = list(stix2.v21.Process._properties.keys()) - properties_of_process.remove("type") + properties_of_process = [prop for prop in properties_of_process if prop not in ["type", "id", "defanged", "spec_version"]] assert excinfo.value.properties == sorted(properties_of_process) msg = "At least one of the ({1}) properties for {0} must be populated." msg = msg.format( @@ -1367,18 +1371,20 @@ def test_new_version_with_related_objects(): objects={ 'src_ip': { 'type': 'ipv4-addr', + 'id': 'ipv4-addr--2b94bc65-17d4-54f6-9ffe-7d103551bb9f', 'value': '127.0.0.1/32', }, 'domain': { 'type': 'domain-name', + 'id': 'domain-name--220a2699-5ebf-5b57-bf02-424964bb19c0', 'value': 'example.com', - 'resolves_to_refs': ['src_ip'], + 'resolves_to_refs': ['ipv4-addr--2b94bc65-17d4-54f6-9ffe-7d103551bb9f'], }, }, ) new_version = data.new_version(last_observed="2017-12-12T12:00:00Z") assert new_version.last_observed.year == 2017 - assert new_version.objects['domain'].resolves_to_refs[0] == 'src_ip' + assert new_version.objects['domain'].resolves_to_refs[0] == 'ipv4-addr--2b94bc65-17d4-54f6-9ffe-7d103551bb9f' def test_objects_deprecation(): diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 899ccd80..19c53d85 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -233,7 +233,7 @@ class Report(STIXDomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types="", spec_version='2.0'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index fbce8fd6..5719e427 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -184,18 +184,18 @@ class EmailMessage(_Observable): ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), ('content_type', StringProperty()), - ('from_ref', ReferenceProperty(valid_types='email-addr')), - ('sender_ref', ReferenceProperty(valid_types='email-addr')), - ('to_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), - ('cc_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), - ('bcc_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), + ('from_ref', ReferenceProperty(valid_types='email-addr', spec_version='2.1')), + ('sender_ref', ReferenceProperty(valid_types='email-addr', spec_version='2.1')), + ('to_refs', ListProperty(ReferenceProperty(valid_types='email-addr', spec_version='2.1'))), + ('cc_refs', ListProperty(ReferenceProperty(valid_types='email-addr', spec_version='2.1'))), + ('bcc_refs', ListProperty(ReferenceProperty(valid_types='email-addr', spec_version='2.1'))), ('message_id', StringProperty()), ('subject', StringProperty()), ('received_lines', ListProperty(StringProperty)), ('additional_header_fields', DictionaryProperty(spec_version='2.1')), ('body', StringProperty()), ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), - ('raw_email_ref', ReferenceProperty(valid_types='artifact')), + ('raw_email_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -384,9 +384,9 @@ class File(_Observable): ('ctime', TimestampProperty()), ('mtime', TimestampProperty()), ('atime', TimestampProperty()), - ('parent_directory_ref', ReferenceProperty(valid_types='directory')), - ('contains_refs', ListProperty(ReferenceProperty(invalid_types=[""]))), - ('content_ref', ReferenceProperty(valid_types='artifact')), + ('parent_directory_ref', ReferenceProperty(valid_types='directory', spec_version='2.1')), + ('contains_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'))), + ('content_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -411,8 +411,8 @@ class IPv4Address(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr'))), - ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system'))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr', spec_version='2.1'))), + ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system', spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -448,8 +448,8 @@ class IPv6Address(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr'))), - ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system'))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr', spec_version='2.1'))), + ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system', spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -629,10 +629,10 @@ class NetworkTraffic(_Observable): ('src_packets', IntegerProperty(min=0)), ('dst_packets', IntegerProperty(min=0)), ('ipfix', DictionaryProperty(spec_version='2.1')), - ('src_payload_ref', ReferenceProperty(valid_types='artifact')), - ('dst_payload_ref', ReferenceProperty(valid_types='artifact')), - ('encapsulates_refs', ListProperty(ReferenceProperty(valid_types='network-traffic'))), - ('encapsulated_by_ref', ReferenceProperty(valid_types='network-traffic')), + ('src_payload_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), + ('dst_payload_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), + ('encapsulates_refs', ListProperty(ReferenceProperty(valid_types='network-traffic', spec_version='2.1'))), + ('encapsulated_by_ref', ReferenceProperty(valid_types='network-traffic', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -748,11 +748,11 @@ class Process(_Observable): ('cwd', StringProperty()), ('command_line', StringProperty()), ('environment_variables', DictionaryProperty(spec_version='2.1')), - ('opened_connection_refs', ListProperty(ReferenceProperty(valid_types='network-traffic'))), - ('creator_user_ref', ReferenceProperty(valid_types='user-account')), - ('image_ref', ReferenceProperty(valid_types='file')), - ('parent_ref', ReferenceProperty(valid_types='process')), - ('child_refs', ListProperty(ReferenceProperty(valid_types='process'))), + ('opened_connection_refs', ListProperty(ReferenceProperty(valid_types='network-traffic', spec_version='2.1'))), + ('creator_user_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')), + ('image_ref', ReferenceProperty(valid_types='file', spec_version='2.1')), + ('parent_ref', ReferenceProperty(valid_types='process', spec_version='2.1')), + ('child_refs', ListProperty(ReferenceProperty(valid_types='process', spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -912,7 +912,7 @@ class WindowsRegistryKey(_Observable): ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself ('modified_time', TimestampProperty()), - ('creator_user_ref', ReferenceProperty(valid_types='user-account')), + ('creator_user_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')), ('number_of_subkeys', IntegerProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 30626592..4cae0e01 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -151,7 +151,7 @@ class Grouping(STIXDomainObject): ('name', StringProperty()), ('description', StringProperty()), ('context', StringProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'), required=True)), ]) @@ -533,7 +533,7 @@ class Note(STIXDomainObject): ('abstract', StringProperty()), ('content', StringProperty(required=True)), ('authors', ListProperty(StringProperty)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -601,7 +601,7 @@ def _check_object_constraints(self): if self.get('object_refs'): for identifier in self.get('object_refs'): - identifier_prefix = identifier[:identifier.index('--') + 2] + identifier_prefix = identifier[:identifier.index('--')] if identifier_prefix in STIX2_OBJ_MAPS['v21']['observables'].keys(): break else: @@ -635,7 +635,7 @@ class Opinion(STIXDomainObject): ], required=True, ), ), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -664,7 +664,7 @@ class Report(STIXDomainObject): ('description', StringProperty()), ('report_types', ListProperty(StringProperty, required=True)), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), From 8f773fd556d08b07dfa896f0d0a0ea6bd2b384a6 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 6 Sep 2019 00:25:42 -0400 Subject: [PATCH 12/17] Temp backup of some code changes. More coming soon --- stix2/base.py | 5 ++++ stix2/datastore/filters.py | 16 ++++++++++++ stix2/properties.py | 4 +-- stix2/test/v21/test_datastore_filters.py | 5 ++-- stix2/test/v21/test_observed_data.py | 31 +++++++++++++++++------- 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 5bc3d107..09ad8844 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -323,6 +323,11 @@ def __init__(self, **kwargs): super(_Observable, self).__init__(**kwargs) def _check_ref(self, ref, prop, prop_name): + """ + Only for checking `*_ref` or `*_refs` properties in spec_version 2.0 + STIX Cyber Observables (SCOs) + """ + if '*' in self._STIXBase__valid_refs: return # don't check if refs are valid diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py index 4f72b827..af21b10e 100644 --- a/stix2/datastore/filters.py +++ b/stix2/datastore/filters.py @@ -90,6 +90,14 @@ def _check_property(self, stix_obj_property): filter_value = self.value if self.op == "=": + boolA = stix_obj_property == filter_value + if boolA is False: + print ('$$$$$') + print (stix_obj_property) + print ('\n') + print (filter_value) + print ('\n') + print ('$$$$$') return stix_obj_property == filter_value elif self.op == "!=": return stix_obj_property != filter_value @@ -97,6 +105,14 @@ def _check_property(self, stix_obj_property): return stix_obj_property in filter_value elif self.op == "contains": if isinstance(filter_value, dict): + boolB = filter_value in stix_obj_property.values() + if boolB is False: + print ('$$$$$') + print (filter_value) + print ('\n') + print (stix_obj_property.values()) + print ('\n') + print ('$$$$$') return filter_value in stix_obj_property.values() else: return filter_value in stix_obj_property diff --git a/stix2/properties.py b/stix2/properties.py index 99dc0a1b..06618205 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -462,12 +462,12 @@ def clean(self, value): if possible_prefix[:-2] in self.valid_types: required_prefix = possible_prefix else: - raise ValueError("The type-specifying prefix '%s' for the identifier '%s' is not valid" % (possible_prefix, value)) + raise ValueError("The type-specifying prefix '%s' for this property is not valid" % (possible_prefix)) elif self.invalid_types: if possible_prefix[:-2] not in self.invalid_types: required_prefix = possible_prefix else: - raise ValueError("An invalid type-specifying prefix '%s' was specified for the identifier '%s'" % (possible_prefix, value)) + raise ValueError("An invalid type-specifying prefix '%s' was specified for this property" % (possible_prefix, value)) _validate_id(value, self.spec_version, required_prefix) diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index 9908fb3b..fd76ae43 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -86,6 +86,7 @@ "objects": { "0": { "type": "file", + "spec_version": "2.1", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", }, @@ -110,8 +111,8 @@ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876"}}), - Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876"}), + Filter("objects", "=", {"0": {"type": "file", "spec_version": "2.1", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe"}}), + Filter("objects", "contains", {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}), Filter("labels", "contains", "heartbleed"), ] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 951943aa..969f62dc 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -27,6 +27,7 @@ "0": { "type": "file", "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", + "spec_version": "2.1", "name": "foo.exe" } } @@ -44,14 +45,19 @@ def test_observed_data_example(): number_observed=50, objects={ "0": { - "name": "foo.exe", - "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", + "name": "foo.exe", }, }, ) - assert str(observed_data) == EXPECTED + assert observed_data.id == "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf" + assert observed_data.created_by_ref == "identity--311b2d2d-f010-4473-83ec-1edf84858f4c" + assert observed_data.created == observed_data.modified == dt.datetime(2016, 4, 6, 19, 58, 16, tzinfo=pytz.utc) + assert observed_data.first_observed == observed_data.last_observed == dt.datetime(2015, 12, 21, 19, 00, 00, tzinfo=pytz.utc) + assert observed_data.number_observed == 50 + assert observed_data.objects['0'] == stix2.v21.File(name="foo.exe") EXPECTED_WITH_REF = """{ @@ -68,11 +74,13 @@ def test_observed_data_example(): "0": { "type": "file", "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", + "spec_version": "2.1", "name": "foo.exe" }, "1": { "type": "directory", "id": "directory--536a61a4-0934-516b-9aad-fcbb75e0583a", + "spec_version": "2.1", "path": "/usr/home", "contains_refs": [ "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f" @@ -93,9 +101,9 @@ def test_observed_data_example_with_refs(): number_observed=50, objects={ "0": { - "name": "foo.exe", - "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", + "name": "foo.exe", }, "1": { "type": "directory", @@ -105,8 +113,13 @@ def test_observed_data_example_with_refs(): }, }, ) - - assert str(observed_data) == EXPECTED_WITH_REF + assert observed_data.id == "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf" + assert observed_data.created_by_ref == "identity--311b2d2d-f010-4473-83ec-1edf84858f4c" + assert observed_data.created == observed_data.modified == dt.datetime(2016, 4, 6, 19, 58, 16, tzinfo=pytz.utc) + assert observed_data.first_observed == observed_data.last_observed == dt.datetime(2015, 12, 21, 19, 00, 00, tzinfo=pytz.utc) + assert observed_data.number_observed == 50 + assert observed_data.objects['0'] == stix2.v21.File(name="foo.exe") + assert observed_data.objects['1'] == stix2.v21.Directory(path="/usr/home", contains_refs=["file--5956efbb-a7b0-566d-a7f9-a202eb05c70f"]) EXPECTED_OBJECT_REFS = """{ @@ -171,7 +184,7 @@ def test_observed_data_object_constraint(): def test_observed_data_example_with_bad_refs(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.ObservedData( id=OBSERVED_DATA_ID, created_by_ref=IDENTITY_ID, @@ -196,7 +209,7 @@ def test_observed_data_example_with_bad_refs(): assert excinfo.value.cls == stix2.v21.Directory assert excinfo.value.prop_name == "contains_refs" - assert "The type-specifying prefix 'monkey--' for the identifier" in excinfo.value.reason + assert "The type-specifying prefix 'monkey--' for this property is not valid" in excinfo.value.reason def test_observed_data_example_with_non_dictionary(): From 5b6592e2dc9ba212063fe39d237d78786354d72e Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 6 Sep 2019 18:08:27 -0400 Subject: [PATCH 13/17] Some changes. More fixes coming soon, hopefully --- stix2/base.py | 14 ++-- stix2/datastore/filters.py | 22 +++++- stix2/properties.py | 2 +- stix2/test/v21/test_datastore_filters.py | 2 +- stix2/test/v21/test_observed_data.py | 92 +++++++++++------------- 5 files changed, 73 insertions(+), 59 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 09ad8844..f933d3b1 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -357,11 +357,12 @@ def _check_property(self, prop_name, prop, kwargs): return from .properties import ObjectReferenceProperty - if isinstance(prop, ObjectReferenceProperty): - if prop_name.endswith('_ref'): + if prop_name.endswith('_ref'): + if isinstance(prop, ObjectReferenceProperty): ref = kwargs[prop_name] self._check_ref(ref, prop, prop_name) - elif prop_name.endswith('_refs'): + elif prop_name.endswith('_refs'): + if isinstance(prop.contained, ObjectReferenceProperty): for ref in kwargs[prop_name]: self._check_ref(ref, prop, prop_name) @@ -392,7 +393,12 @@ def _generate_id(self, kwargs): if streamlined_obj_vals: data = canonicalize(streamlined_obj_vals, utf8=False) - return required_prefix + six.text_type(uuid.uuid5(namespace, data)) + # print (str(type(data))) + try: + return required_prefix + six.text_type(uuid.uuid5(namespace, data)) + except UnicodeDecodeError: + return required_prefix + six.text_type(uuid.uuid5(namespace, six.binary_type(data))) + # return required_prefix + six.text_type(uuid.uuid5(namespace, data)) # We return None if there are no values specified for any of the id-contributing-properties return None diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py index af21b10e..806f6725 100644 --- a/stix2/datastore/filters.py +++ b/stix2/datastore/filters.py @@ -98,6 +98,15 @@ def _check_property(self, stix_obj_property): print (filter_value) print ('\n') print ('$$$$$') + pass + else: + # print ('222222222') + # print (stix_obj_property) + # print ('\n') + # print (filter_value) + # print ('\n') + # print ('222222222') + pass return stix_obj_property == filter_value elif self.op == "!=": return stix_obj_property != filter_value @@ -107,12 +116,21 @@ def _check_property(self, stix_obj_property): if isinstance(filter_value, dict): boolB = filter_value in stix_obj_property.values() if boolB is False: - print ('$$$$$') + print ('@@@@@@') print (filter_value) print ('\n') print (stix_obj_property.values()) print ('\n') - print ('$$$$$') + print ('@@@@@@@') + pass + else: + # print ('55555555555') + # print (filter_value) + # print ('\n') + # print (stix_obj_property.values()) + # print ('\n') + # print ('55555555555') + pass return filter_value in stix_obj_property.values() else: return filter_value in stix_obj_property diff --git a/stix2/properties.py b/stix2/properties.py index 06618205..c956a08b 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -457,7 +457,7 @@ def clean(self, value): elif self.valid_types == ["only_SCO"]: self.valid_types = STIX2_OBJ_MAPS['v21']['observables'].keys() elif self.valid_types == ["only_SCO_&_SRO"]: - self.valid_types = STIX2_OBJ_MAPS['v21']['observables'].keys() + ['relationship', 'sighting'] + self.valid_types = list(STIX2_OBJ_MAPS['v21']['observables'].keys()) + ['relationship', 'sighting'] if possible_prefix[:-2] in self.valid_types: required_prefix = possible_prefix diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index fd76ae43..96ac17fe 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -111,7 +111,7 @@ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "spec_version": "2.1", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe"}}), + Filter("objects", "=", {"0": {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}}), Filter("objects", "contains", {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}), Filter("labels", "contains", "heartbleed"), ] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 969f62dc..fbebdbdf 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -364,17 +364,21 @@ def test_parse_autonomous_system_valid(data): "type": "email-addr", "value": "john@example.com", "display_name": "John Doe", - "belongs_to_ref": "0" + "belongs_to_ref": "user-account--fc07c1af-6b11-41f8-97a4-47920d866a91" }""", ], ) def test_parse_email_address(data): - odata = stix2.parse_observable(data, {"0": "user-account"}, version='2.1') + odata = stix2.parse_observable(data, version='2.1') assert odata.type == "email-addr" - odata_str = re.compile('"belongs_to_ref": "0"', re.DOTALL).sub('"belongs_to_ref": "3"', data) - with pytest.raises(stix2.exceptions.InvalidObjRefError): - stix2.parse_observable(odata_str, {"0": "user-account"}, version='2.1') + odata_str = re.compile( + '"belongs_to_ref": "user-account--fc07c1af-6b11-41f8-97a4-47920d866a91"', re.DOTALL, + ).sub( + '"belongs_to_ref": "mutex--9be6365f-b89c-48c0-9340-6953f6595718"', data, + ) + with pytest.raises(stix2.exceptions.InvalidValueError): + stix2.parse_observable(odata_str, version='2.1') @pytest.mark.parametrize( @@ -385,12 +389,12 @@ def test_parse_email_address(data): "is_multipart": true, "content_type": "multipart/mixed", "date": "2016-06-19T14:20:40.000Z", - "from_ref": "1", + "from_ref": "email-addr--d4ef7e1f-086d-5ff4-bce4-312ddc3eae76", "to_refs": [ - "2" + "email-addr--8b0eb924-208c-5efd-80e5-84e2d610e54b" ], "cc_refs": [ - "3" + "email-addr--1766f860-5cf3-5697-8789-35f1242663d5" ], "subject": "Check out this picture of a cat!", "additional_header_fields": { @@ -407,12 +411,12 @@ def test_parse_email_address(data): { "content_type": "image/png", "content_disposition": "attachment; filename=\\"tabby.png\\"", - "body_raw_ref": "4" + "body_raw_ref": "artifact--80b04ad8-db52-464b-a85a-a44a5f3a60c5" }, { "content_type": "application/zip", "content_disposition": "attachment; filename=\\"tabby_pics.zip\\"", - "body_raw_ref": "5" + "body_raw_ref": "file--e63474fc-b386-5630-a003-1b555e22f99b" } ] } @@ -420,15 +424,7 @@ def test_parse_email_address(data): ], ) def test_parse_email_message(data): - valid_refs = { - "0": "email-message", - "1": "email-addr", - "2": "email-addr", - "3": "email-addr", - "4": "artifact", - "5": "file", - } - odata = stix2.parse_observable(data, valid_refs, version='2.1') + odata = stix2.parse_observable(data, version='2.1') assert odata.type == "email-message" assert odata.body_multipart[0].content_disposition == "inline" @@ -438,8 +434,8 @@ def test_parse_email_message(data): """ { "type": "email-message", - "from_ref": "0", - "to_refs": ["1"], + "from_ref": "email-addr--d4ef7e1f-086d-5ff4-bce4-312ddc3eae76", + "to_refs": ["email-addr--8b0eb924-208c-5efd-80e5-84e2d610e54b"], "is_multipart": true, "date": "1997-11-21T15:55:06.000Z", "subject": "Saying Hello", @@ -449,12 +445,8 @@ def test_parse_email_message(data): ], ) def test_parse_email_message_not_multipart(data): - valid_refs = { - "0": "email-addr", - "1": "email-addr", - } with pytest.raises(stix2.exceptions.DependentPropertiesError) as excinfo: - stix2.parse_observable(data, valid_refs, version='2.1') + stix2.parse_observable(data, version='2.1') assert excinfo.value.cls == stix2.v21.EmailMessage assert excinfo.value.dependencies == [("is_multipart", "body")] @@ -464,18 +456,21 @@ def test_parse_email_message_not_multipart(data): "data", [ """"0": { "type": "file", + "id": "file--ecd47d73-15e4-5250-afda-ef8897b22340", "hashes": { "SHA-256": "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a" } }, "1": { "type": "file", + "id": "file--65f2873d-38c2-56b4-bfa5-e3ef21e8a3c3", "hashes": { "SHA-256": "19c549ec2628b989382f6b280cbd7bb836a0b461332c0fe53511ce7d584b89d3" } }, "2": { "type": "file", + "id": "file--ef2d6dca-ec7d-5ab7-8dd9-ec9c0dee0eac", "hashes": { "SHA-256": "0969de02ecf8a5f003e3f6d063d848c8a193aada092623f8ce408c15bcb5f038" } @@ -490,9 +485,9 @@ def test_parse_email_message_not_multipart(data): "extensions": { "archive-ext": { "contains_refs": [ - "0", - "1", - "2" + "file--ecd47d73-15e4-5250-afda-ef8897b22340", + "file--65f2873d-38c2-56b4-bfa5-e3ef21e8a3c3", + "file--ef2d6dca-ec7d-5ab7-8dd9-ec9c0dee0eac" ] } } @@ -503,7 +498,11 @@ def test_parse_file_archive(data): odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str, version="2.1") assert all(x in odata.objects["3"].extensions['archive-ext'].contains_refs - for x in ["0", "1", "2"]) + for x in [ + "file--ecd47d73-15e4-5250-afda-ef8897b22340", + "file--65f2873d-38c2-56b4-bfa5-e3ef21e8a3c3", + "file--ef2d6dca-ec7d-5ab7-8dd9-ec9c0dee0eac", + ]) @pytest.mark.parametrize( @@ -514,12 +513,12 @@ def test_parse_file_archive(data): "is_multipart": true, "content_type": "multipart/mixed", "date": "2016-06-19T14:20:40.000Z", - "from_ref": "1", + "from_ref": "email-addr--d4ef7e1f-086d-5ff4-bce4-312ddc3eae76", "to_refs": [ - "2" + "email-addr--8b0eb924-208c-5efd-80e5-84e2d610e54b" ], "cc_refs": [ - "3" + "email-addr--1766f860-5cf3-5697-8789-35f1242663d5" ], "subject": "Check out this picture of a cat!", "additional_header_fields": { @@ -540,7 +539,7 @@ def test_parse_file_archive(data): { "content_type": "application/zip", "content_disposition": "attachment; filename=\\"tabby_pics.zip\\"", - "body_raw_ref": "5" + "body_raw_ref": "file--e63474fc-b386-5630-a003-1b555e22f99b" } ] } @@ -548,16 +547,8 @@ def test_parse_file_archive(data): ], ) def test_parse_email_message_with_at_least_one_error(data): - valid_refs = { - "0": "email-message", - "1": "email-addr", - "2": "email-addr", - "3": "email-addr", - "4": "artifact", - "5": "file", - } with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: - stix2.parse_observable(data, valid_refs, version='2.1') + stix2.parse_observable(data, version='2.1') assert excinfo.value.cls == stix2.v21.EmailMessage assert "At least one of the" in str(excinfo.value) @@ -569,8 +560,8 @@ def test_parse_email_message_with_at_least_one_error(data): """ { "type": "network-traffic", - "src_ref": "0", - "dst_ref": "1", + "src_ref": "ipv4-addr--e535b017-cc1c-566b-a3e2-f69f92ed9c4c", + "dst_ref": "ipv4-addr--78327430-9ad9-5632-ae3d-8e2fce8f5483", "protocols": [ "tcp" ] @@ -580,13 +571,12 @@ def test_parse_email_message_with_at_least_one_error(data): ) def test_parse_basic_tcp_traffic(data): odata = stix2.parse_observable( - data, {"0": "ipv4-addr", "1": "ipv4-addr"}, - version='2.1', + data, version='2.1', ) assert odata.type == "network-traffic" - assert odata.src_ref == "0" - assert odata.dst_ref == "1" + assert odata.src_ref == "ipv4-addr--e535b017-cc1c-566b-a3e2-f69f92ed9c4c" + assert odata.dst_ref == "ipv4-addr--78327430-9ad9-5632-ae3d-8e2fce8f5483" assert odata.protocols == ["tcp"] @@ -604,7 +594,7 @@ def test_parse_basic_tcp_traffic(data): "src_byte_count": 35779, "dst_byte_count": 935750, "encapsulates_refs": [ - "4" + "network-traffic--016914c3-b680-5df2-81c4-bb9ccf8dc8b0" ] } """, @@ -612,7 +602,7 @@ def test_parse_basic_tcp_traffic(data): ) def test_parse_basic_tcp_traffic_with_error(data): with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: - stix2.parse_observable(data, {"4": "network-traffic"}, version='2.1') + stix2.parse_observable(data, version='2.1') assert excinfo.value.cls == stix2.v21.NetworkTraffic assert excinfo.value.properties == ["dst_ref", "src_ref"] From d828e41c78932aaf290371d95281021af10bfdae Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 11 Sep 2019 10:44:14 -0400 Subject: [PATCH 14/17] End of changes --- stix2/base.py | 4 +-- stix2/canonicalization/Canonicalize.py | 8 ++++-- stix2/datastore/filters.py | 34 ------------------------ stix2/test/v21/test_datastore_filters.py | 11 ++++++-- 4 files changed, 17 insertions(+), 40 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index f933d3b1..4d4b080a 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -393,12 +393,12 @@ def _generate_id(self, kwargs): if streamlined_obj_vals: data = canonicalize(streamlined_obj_vals, utf8=False) - # print (str(type(data))) + + # try/except here to enable python 2 compatibility try: return required_prefix + six.text_type(uuid.uuid5(namespace, data)) except UnicodeDecodeError: return required_prefix + six.text_type(uuid.uuid5(namespace, six.binary_type(data))) - # return required_prefix + six.text_type(uuid.uuid5(namespace, data)) # We return None if there are no values specified for any of the id-contributing-properties return None diff --git a/stix2/canonicalization/Canonicalize.py b/stix2/canonicalization/Canonicalize.py index daf06165..78145be5 100644 --- a/stix2/canonicalization/Canonicalize.py +++ b/stix2/canonicalization/Canonicalize.py @@ -24,6 +24,8 @@ import re +import six + from stix2.canonicalization.NumberToJson import convert2Es6Format try: @@ -395,7 +397,8 @@ def _iterencode_dict(dct, _current_indent_level): else: items = dct.items() for key, value in items: - if isinstance(key, str): + # Replaced isinstance(key, str) with below to enable simultaneous python 2 & 3 compatibility + if isinstance(key, six.string_types) or isinstance(key, six.binary_type): pass # JavaScript is weakly typed for these, so it makes sense to # also allow them. Many encoders seem to do something like this. @@ -454,7 +457,8 @@ def _iterencode_dict(dct, _current_indent_level): del markers[markerid] def _iterencode(o, _current_indent_level): - if isinstance(o, str): + # Replaced isinstance(o, str) with below to enable simultaneous python 2 & 3 compatibility + if isinstance(o, six.string_types) or isinstance(o, six.binary_type): yield _encoder(o) elif o is None: yield 'null' diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py index 806f6725..4f72b827 100644 --- a/stix2/datastore/filters.py +++ b/stix2/datastore/filters.py @@ -90,23 +90,6 @@ def _check_property(self, stix_obj_property): filter_value = self.value if self.op == "=": - boolA = stix_obj_property == filter_value - if boolA is False: - print ('$$$$$') - print (stix_obj_property) - print ('\n') - print (filter_value) - print ('\n') - print ('$$$$$') - pass - else: - # print ('222222222') - # print (stix_obj_property) - # print ('\n') - # print (filter_value) - # print ('\n') - # print ('222222222') - pass return stix_obj_property == filter_value elif self.op == "!=": return stix_obj_property != filter_value @@ -114,23 +97,6 @@ def _check_property(self, stix_obj_property): return stix_obj_property in filter_value elif self.op == "contains": if isinstance(filter_value, dict): - boolB = filter_value in stix_obj_property.values() - if boolB is False: - print ('@@@@@@') - print (filter_value) - print ('\n') - print (stix_obj_property.values()) - print ('\n') - print ('@@@@@@@') - pass - else: - # print ('55555555555') - # print (filter_value) - # print ('\n') - # print (stix_obj_property.values()) - # print ('\n') - # print ('55555555555') - pass return filter_value in stix_obj_property.values() else: return filter_value in stix_obj_property diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index 96ac17fe..b7b41a00 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -89,6 +89,7 @@ "spec_version": "2.1", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", + "defanged": False, }, }, @@ -111,8 +112,14 @@ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}}), - Filter("objects", "contains", {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}), + Filter( + "objects", "=", + {"0": {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1", "defanged": False}}, + ), + Filter( + "objects", "contains", + {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1", "defanged": False}, + ), Filter("labels", "contains", "heartbleed"), ] From 9c7128d074a74f82421fd8f4e97b8c32c335a29d Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 11 Sep 2019 10:49:11 -0400 Subject: [PATCH 15/17] Fix indentation issue --- stix2/base.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 4d4b080a..dbe64610 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -412,18 +412,18 @@ def _check_object_constraints(self): def _choose_one_hash(hash_dict): - if "MD5" in hash_dict: - return {"MD5": hash_dict["MD5"]} - elif "SHA-1" in hash_dict: - return {"SHA-1": hash_dict["SHA-1"]} - elif "SHA-256" in hash_dict: - return {"SHA-256": hash_dict["SHA-256"]} - elif "SHA-512" in hash_dict: - return {"SHA-512": hash_dict["SHA-512"]} - else: - k = next(iter(hash_dict), None) - if k is not None: - return {k: hash_dict[k]} + if "MD5" in hash_dict: + return {"MD5": hash_dict["MD5"]} + elif "SHA-1" in hash_dict: + return {"SHA-1": hash_dict["SHA-1"]} + elif "SHA-256" in hash_dict: + return {"SHA-256": hash_dict["SHA-256"]} + elif "SHA-512" in hash_dict: + return {"SHA-512": hash_dict["SHA-512"]} + else: + k = next(iter(hash_dict), None) + if k is not None: + return {k: hash_dict[k]} def _cls_init(cls, obj, kwargs): From 8447c9fcd97eaf7a9a1151305b119fecdda00845 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 11 Sep 2019 14:21:41 -0400 Subject: [PATCH 16/17] Add few tests to improve some code coverage --- docs/guide/datastore.ipynb | 18 ++++++++---- stix2/test/v21/test_observed_data.py | 42 ++++++++++++++++++++++++++-- stix2/v21/observables.py | 2 +- stix2/v21/sdo.py | 15 ++-------- 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index 1c88cf12..1ea05ee3 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -450,6 +450,14 @@ "mem.source.filters.add([f1,f2])" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Note: The `defanged` property is now always included (implicitly) for STIX 2.1 Cyber Observable Objects (SCOs)\n", + "This is important to remember if you are writing a filter that involves checking the `objects` property of a STIX 2.1 `ObservedData` object. If any of the objects associated with the `objects` property are STIX 2.1 SCOs, then your filter must include the `defanged` property. For an example, refer to `filters[14]` & `filters[15]` in stix2/test/v21/test_datastore_filters.py " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -726,21 +734,21 @@ ], "metadata": { "kernelspec": { - "display_name": "cti-python-stix2", + "display_name": "Python 3", "language": "python", - "name": "cti-python-stix2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.6.7" } }, "nbformat": 4, diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index fbebdbdf..0074bf73 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -465,21 +465,21 @@ def test_parse_email_message_not_multipart(data): "type": "file", "id": "file--65f2873d-38c2-56b4-bfa5-e3ef21e8a3c3", "hashes": { - "SHA-256": "19c549ec2628b989382f6b280cbd7bb836a0b461332c0fe53511ce7d584b89d3" + "SHA-1": "6e71b3cac15d32fe2d36c270887df9479c25c640" } }, "2": { "type": "file", "id": "file--ef2d6dca-ec7d-5ab7-8dd9-ec9c0dee0eac", "hashes": { - "SHA-256": "0969de02ecf8a5f003e3f6d063d848c8a193aada092623f8ce408c15bcb5f038" + "SHA-512": "b7e98c78c24fb4c2c7b175e90474b21eae0ccf1b5ea4708b4e0f2d2940004419edc7161c18a1e71b2565df099ba017bcaa67a248e2989b6268ce078b88f2e210" } }, "3": { "type": "file", "name": "foo.zip", "hashes": { - "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" + "SHA3-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" }, "mime_type": "application/zip", "extensions": { @@ -1490,3 +1490,39 @@ def test_deterministic_id_no_contributing_props(): uuid_obj_2 = uuid.UUID(email_msg_2.id[-36:]) assert uuid_obj_2.variant == uuid.RFC_4122 assert uuid_obj_2.version == 4 + + +def test_ipv4_resolves_to_refs_deprecation(): + with pytest.warns(stix2.exceptions.STIXDeprecationWarning): + + stix2.v21.IPv4Address( + value="26.09.19.70", + resolves_to_refs=["mac-addr--08900593-0265-52fc-93c0-5b4a942f5887"], + ) + + +def test_ipv4_belongs_to_refs_deprecation(): + with pytest.warns(stix2.exceptions.STIXDeprecationWarning): + + stix2.v21.IPv4Address( + value="21.12.19.64", + belongs_to_refs=["autonomous-system--52e0a49d-d683-5801-a7b8-145765a1e116"], + ) + + +def test_ipv6_resolves_to_refs_deprecation(): + with pytest.warns(stix2.exceptions.STIXDeprecationWarning): + + stix2.v21.IPv6Address( + value="2001:0db8:85a3:0000:0000:8a2e:0370:7334", + resolves_to_refs=["mac-addr--08900593-0265-52fc-93c0-5b4a942f5887"], + ) + + +def test_ipv6_belongs_to_refs_deprecation(): + with pytest.warns(stix2.exceptions.STIXDeprecationWarning): + + stix2.v21.IPv6Address( + value="2001:0db8:85a3:0000:0000:8a2e:0370:7334", + belongs_to_refs=["autonomous-system--52e0a49d-d683-5801-a7b8-145765a1e116"], + ) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 5719e427..9fffeeda 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -385,7 +385,7 @@ class File(_Observable): ('mtime', TimestampProperty()), ('atime', TimestampProperty()), ('parent_directory_ref', ReferenceProperty(valid_types='directory', spec_version='2.1')), - ('contains_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'))), + ('contains_refs', ListProperty(ReferenceProperty(invalid_types="", spec_version='2.1'))), ('content_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 4cae0e01..8bf76b83 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -6,11 +6,9 @@ from six.moves.urllib.parse import quote_plus -from ..core import STIX2_OBJ_MAPS, STIXDomainObject +from ..core import STIXDomainObject from ..custom import _custom_object_builder -from ..exceptions import ( - InvalidValueError, PropertyPresenceError, STIXDeprecationWarning, -) +from ..exceptions import PropertyPresenceError, STIXDeprecationWarning from ..properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, ListProperty, @@ -597,16 +595,9 @@ def _check_object_constraints(self): self._check_mutually_exclusive_properties( ["objects", "object_refs"], + at_least_one=False, ) - if self.get('object_refs'): - for identifier in self.get('object_refs'): - identifier_prefix = identifier[:identifier.index('--')] - if identifier_prefix in STIX2_OBJ_MAPS['v21']['observables'].keys(): - break - else: - raise InvalidValueError(self.__class__, 'object_refs', "At least one identifier must be of a SCO type if this property specified") - class Opinion(STIXDomainObject): # TODO: Add link From 3b1c922ba63a1517b43dd4ed07a511d8e380c331 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 18 Sep 2019 10:29:07 -0400 Subject: [PATCH 17/17] Fix observed data property check for at least one property existing --- stix2/v21/sdo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 8bf76b83..7cd7891e 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -595,7 +595,6 @@ def _check_object_constraints(self): self._check_mutually_exclusive_properties( ["objects", "object_refs"], - at_least_one=False, )