use Text for timestamps, flaky

oasis-open · Dec 13, 2024 · aa41ecb · aa41ecb
1 parent 2b55b4d
commit aa41ecb
Show file tree

Hide file tree

Showing 12 changed files with 103 additions and 87 deletions.
diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py
@@ -1,13 +1,15 @@
 from typing import Any
 
-from sqlalchemy import Boolean, CheckConstraint, Float, Integer, String, Text, create_engine
+from sqlalchemy import (
+    Boolean, CheckConstraint, Float, Integer, String, Text, create_engine,
+)
 from sqlalchemy_utils import create_database, database_exists, drop_database
 
 from stix2.base import (
     _DomainObject, _MetaObject, _Observable, _RelationshipObject,
 )
-
 from stix2.properties import HexProperty
+from stix2.utils import STIXdatetime
 
 
 class DatabaseBackend:
@@ -126,13 +128,17 @@ def create_regex_constraint_expression(self, column_name, pattern):
         return CheckConstraint(self.create_regex_constraint_clause(column_name, pattern))
 
     def create_regex_constraint_and_expression(self, clause1, clause2):
-        return (CheckConstraint("((" + self.create_regex_constraint_clause(clause1[0], clause1[1]) + ") AND (" +
-                self.create_regex_constraint_clause(clause2[0], clause2[1]) + "))"))
+        return (
+            CheckConstraint(
+                "((" + self.create_regex_constraint_clause(clause1[0], clause1[1]) + ") AND (" +
+                self.create_regex_constraint_clause(clause2[0], clause2[1]) + "))",
+            )
+        )
 
     def process_value_for_insert(self, stix_type, value):
         sql_type = stix_type.determine_sql_type(self)
-        if sql_type == self.determine_sql_type_for_string_property():
-            return value
+        if sql_type == self.determine_sql_type_for_timestamp_property() and isinstance(value, STIXdatetime):
+            return value.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
         elif sql_type == self.determine_sql_type_for_hex_property() and isinstance(stix_type, HexProperty):
             return bytes.fromhex(value)
         else:

diff --git a/stix2/datastore/relational_db/database_backends/mariadb_backend.py b/stix2/datastore/relational_db/database_backends/mariadb_backend.py
@@ -1,7 +1,7 @@
 import os
 from typing import Any
 
-from sqlalchemy import Text, VARCHAR, CheckConstraint
+from sqlalchemy import VARCHAR, Text
 from sqlalchemy.schema import CreateSchema
 
 from stix2.base import (
@@ -80,5 +80,3 @@ def array_allowed():
 
     def create_regex_constraint_clause(self, column_name, pattern):
         return f"{column_name} REGEXP {pattern}"
-
-
diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py
@@ -1,7 +1,7 @@
 import os
 from typing import Any
 
-from sqlalchemy import TIMESTAMP, CheckConstraint, Text
+from sqlalchemy import Text
 from sqlalchemy.schema import CreateSchema
 
 from stix2.base import (
@@ -77,7 +77,7 @@ def determine_sql_type_for_key_as_id():  # noqa: F811
 
     @staticmethod
     def determine_sql_type_for_timestamp_property():  # noqa: F811
-        return TIMESTAMP(timezone=True)
+        return Text
 
     # =========================================================================
     # Other methods
@@ -87,4 +87,4 @@ def array_allowed():
         return True
 
     def create_regex_constraint_clause(self, column_name, pattern):
-        return f"{column_name} ~ {pattern}"
+        return f"{column_name} ~ {pattern}"
diff --git a/stix2/datastore/relational_db/database_backends/sqlite_backend.py b/stix2/datastore/relational_db/database_backends/sqlite_backend.py
@@ -1,19 +1,12 @@
-import os
 from typing import Any
 
-from sqlalchemy import TIMESTAMP, LargeBinary, Text
-from sqlalchemy import event
-
-from stix2.base import (
-    _DomainObject, _MetaObject, _Observable, _RelationshipObject,
-)
-from stix2.datastore.relational_db.utils import schema_for
+from sqlalchemy import Text, event
 
 from .database_backend_base import DatabaseBackend
 
 
 class SQLiteBackend(DatabaseBackend):
-    default_database_connection_url = f"sqlite:///stix-data-sink.db"
+    default_database_connection_url = "sqlite:///stix-data-sink.db"
 
     temp_sequence_count = 0
 
@@ -55,7 +48,7 @@ def determine_sql_type_for_key_as_id():  # noqa: F811
 
     @staticmethod
     def determine_sql_type_for_timestamp_property():  # noqa: F811
-        return TIMESTAMP(timezone=True)
+        return Text
 
     # =========================================================================
     # Other methods
@@ -72,5 +65,6 @@ def create_regex_constraint_and_expression(self, clause1, clause2):
 
     @staticmethod
     def next_id(data_sink):
+        # hack, which is nit reliable
         SQLiteBackend.temp_sequence_count += 1
         return SQLiteBackend.temp_sequence_count
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
@@ -30,6 +30,7 @@
 #   level:              what "level" of child table is involved
 #   foreign_key_value:
 
+
 @add_method(Property)
 def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
     pass
@@ -98,7 +99,8 @@ def generate_insert_information(self, dictionary_name, stix_object, **kwargs):
                 if not data_sink.db_backend.array_allowed():
                     next_id = data_sink.db_backend.next_id(data_sink)
                     table_child = data_sink.tables_dictionary[
-                        canonicalize_table_name(table_name + "_" + dictionary_name + "_" + "values", schema_name)]
+                        canonicalize_table_name(table_name + "_" + dictionary_name + "_" + "values", schema_name)
+                    ]
                     child_table_inserts = generate_insert_for_dictionary_list(table_child, next_id, value)
                     value = next_id
             else:
@@ -299,7 +301,7 @@ def generate_insert_information(   # noqa: F811
             for elem in stix_object[name]:
                 bindings = {
                     "id": stix_object["id"],
-                    name: db_backend.process_value_for_insert(self.contained, elem)
+                    name: db_backend.process_value_for_insert(self.contained, elem),
                 }
                 insert_statements.append(insert(table).values(bindings))
             return insert_statements
@@ -364,12 +366,14 @@ def generate_insert_for_external_references(data_sink, stix_object):
 
         if "hashes" in er:
             insert_statements.extend(
-                generate_insert_for_hashes(data_sink,
-                                           "hashes",
-                                           er,
-                                           "external_references",
-                                           schema_name,
-                                           foreign_key_value=next_id),
+                generate_insert_for_hashes(
+                    data_sink,
+                    "hashes",
+                    er,
+                    "external_references",
+                    schema_name,
+                    foreign_key_value=next_id,
+                ),
             )
 
     return insert_statements

diff --git a/stix2/datastore/relational_db/query.py b/stix2/datastore/relational_db/query.py
@@ -250,7 +250,7 @@ def _read_granular_markings(stix_id, stix_type_class, metadata, conn, db_backend
                 selector_id,
                 "selector",
                 selector_table,
-                conn
+                conn,
             )
             marking_dict["selectors"] = selectors
 
@@ -468,25 +468,27 @@ def _read_complex_property_value(obj_id, prop_name, prop_instance, obj_table, me
             ref_table = metadata.tables[ref_table_name]
             prop_value = _read_simple_array(obj_id, "ref_id", ref_table, conn)
 
-        elif isinstance(prop_instance.contained, (
-            # Most of these list-of-simple-type cases would occur when array
-            # columns are disabled.
-            stix2.properties.BinaryProperty,
-            stix2.properties.BooleanProperty,
-            stix2.properties.EnumProperty,
-            stix2.properties.HexProperty,
-            stix2.properties.IntegerProperty,
-            stix2.properties.FloatProperty,
-            stix2.properties.StringProperty,
-            stix2.properties.TimestampProperty,
-        )):
+        elif isinstance(
+            prop_instance.contained, (
+                # Most of these list-of-simple-type cases would occur when array
+                # columns are disabled.
+                stix2.properties.BinaryProperty,
+                stix2.properties.BooleanProperty,
+                stix2.properties.EnumProperty,
+                stix2.properties.HexProperty,
+                stix2.properties.IntegerProperty,
+                stix2.properties.FloatProperty,
+                stix2.properties.StringProperty,
+                stix2.properties.TimestampProperty,
+            ),
+        ):
             array_table_name = f"{obj_table.fullname}_{prop_name}"
             array_table = metadata.tables[array_table_name]
             prop_value = _read_simple_array(
                 obj_id,
                 prop_name,
                 array_table,
-                conn
+                conn,
             )
 
         elif isinstance(prop_instance.contained, stix2.properties.EmbeddedObjectProperty):
@@ -549,7 +551,7 @@ def _read_complex_top_level_property_value(
     type_table,
     metadata,
     conn,
-    db_backend
+    db_backend,
 ):
     """
     Read property values which require auxiliary tables to store.  These
@@ -581,7 +583,7 @@ def _read_complex_top_level_property_value(
             stix_id,
             stix_type_class,
             metadata,
-            conn
+            conn,
         )
 
     elif prop_name == "granular_markings":
@@ -590,7 +592,7 @@ def _read_complex_top_level_property_value(
             stix_type_class,
             metadata,
             conn,
-            db_backend
+            db_backend,
         )
 
     # Will apply when array columns are unsupported/disallowed by the backend
@@ -608,7 +610,7 @@ def _read_complex_top_level_property_value(
             prop_instance,
             type_table,
             metadata,
-            conn
+            conn,
         )
 
     return prop_value
@@ -663,7 +665,7 @@ def read_object(stix_id, metadata, conn, db_backend):
                 type_table,
                 metadata,
                 conn,
-                db_backend
+                db_backend,
             )
 
             if prop_value is not None:

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
@@ -11,6 +11,7 @@
 from stix2.datastore.relational_db.utils import canonicalize_table_name
 from stix2.parsing import parse
 
+
 def _add(store, stix_data, allow_custom=True, version="2.1"):
     """Add STIX objects to MemoryStore/Sink.
 

diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
@@ -1,10 +1,10 @@
 import datetime as dt
+import os  # noqa: F401
 
-from database_backends.postgres_backend import PostgresBackend
+from database_backends.mariadb_backend import MariaDBBackend  # noqa: F401
+from database_backends.postgres_backend import PostgresBackend  # noqa: F401
 from database_backends.sqlite_backend import SQLiteBackend
-from database_backends.mariadb_backend import MariaDBBackend
 import pytz
-import os
 
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
@@ -22,9 +22,9 @@
                     additional_header_fields={
                         "Reply-To": [
                             "[email protected]",
-                            "[email protected]"
-                        ]
-                    }
+                            "[email protected]",
+                        ],
+                    },
 )
 
 directory_stix_object = stix2.Directory(
@@ -284,14 +284,14 @@ class Test3Class:
 def test_dictionary():
     return Test3Class(
         prop_name={"a": 1, "b": 2.3, "c": "foo"},
-        list_of_timestamps=["2016-05-12T08:17:27.000Z", "2024-05-12T08:17:27.000Z"]
+        list_of_timestamps=["2016-05-12T08:17:27.000Z", "2024-05-12T08:17:27.000Z"],
     )
 
 
 def main():
     store = RelationalDBStore(
-        #MariaDBBackend(f"mariadb+pymysql://admin:[email protected]:3306/rdb", force_recreate=True),
-        #PostgresBackend("postgresql://localhost/stix-data-sink", force_recreate=True),
+        # MariaDBBackend(f"mariadb+pymysql://admin:[email protected]:3306/rdb", force_recreate=True),
+        # PostgresBackend("postgresql://localhost/stix-data-sink", force_recreate=True),
         SQLiteBackend("sqlite:///stix-data-sink.db", force_recreate=True),
 
         True,
@@ -305,9 +305,9 @@ def main():
         ap = kill_chain_test()
         store.add(ap)
 
-        # x=email_message
+        x = email_message
 
-        # store.add(x)
+        store.add(x)
 
         td = test_dictionary()
 
@@ -328,8 +328,6 @@ def main():
         pdf_file = file_example_with_PDFExt_Object()
         store.add(pdf_file)
 
-
-
         store.add(directory_stix_object)
 
         store.add(s)