Unstructured-IO · cragwolfe · Apr 24, 2024 · Mar 20, 2024 · Mar 21, 2024 · Mar 21, 2024
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,17 @@
+
+## 0.13.4-dev0
+
+### Enhancements
+* **Unique and deterministic hash IDs for elements** Element IDs produced by any partitioning function are now deterministic and unique at the document level by default. Before, hashes were based only on text; however, they now also take into account the element's sequence number on a page, the page's number in the document, and the document's file name.
+
+### Features
+
+### Fixes
+
 ## 0.13.3
 
 ### Enhancements
+* **Unique and deterministic hash IDs for elements** Element IDs produced by any partitioning function are now deterministic and unique at the document level by default. Before, hashes were based only on text; however, they now also take into account the element's sequence number on a page, the page's number in the document, and the document's file name.
 
 * **Remove duplicate image elements**. Remove image elements identified by PDFMiner that have similar bounding boxes and the same text.
 * **Add support for `start_index` in `html` links extraction**

diff --git a/docs/source/introduction/overview.rst b/docs/source/introduction/overview.rst
@@ -141,10 +141,8 @@ a list of elements from JSON, as seen in the snippet below
 Unique Element IDs
 ******************
 
-By default, the element ID is a SHA-256 hash of the element text. This is to ensure that
-the ID is deterministic. One downside is that the ID is not guaranteed to be unique.
-Different elements with the same text will have the same ID, and there could also
-be hash collisions. To use UUIDs in the output instead, you can pass
+By default, the element ID is a SHA-256 hash of the element's text, its position on the page, page number it's on, and the name of the document file - this is to ensure that the ID is deterministic and unique at the document level.
+To obtain globally unique IDs in the output (UUIDs), you can pass
 ``unique_element_ids=True`` into any of the partition functions. This can be helpful
 if you'd like to use the IDs as a primary key in a database, for example.
 
@@ -161,7 +159,7 @@ Element ID Design Principles
 #. A partitioning function can assign only one of two available ID types to a returned element: a hash or a UUID.
 #. All elements that are returned come with an ID, which is never None.
 #. No matter which type of ID is used, it will always be in string format.
-#. Partitioning a document returns elements with hashes as their default IDs.
+#. Partitioning a document returns elements with hashes as their default IDs, ensuring they are both deterministic and unique within a document.
 
 For creating elements independently of partitioning functions, refer to the `Element` class documentation in the source code (`unstructured/documents/elements.py`).
 

diff --git a/example-docs/duplicate-paragraphs.doc b/example-docs/duplicate-paragraphs.doc
diff --git a/example-docs/duplicate-paragraphs.docx b/example-docs/duplicate-paragraphs.docx
diff --git a/example-docs/fake-html-with-duplicate-elements.html b/example-docs/fake-html-with-duplicate-elements.html
@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <title>Simple Nested HTML</title>
+    </strong>
+
+<body>
+    <h1>Example heading.</h1>
+    <div>
+        <span>This is a span.</span>
+        <span>This is another span.</span>
+    </div>
+    <br>
+    <h1>Example heading.</h1>
+    <div>
+        <span>This is a span.</span>
+        <span>This is another span.</span>
+    </div>
+
+</body>
+
+</html>
diff --git a/example-docs/fake-memo-with-duplicate-page.pdf b/example-docs/fake-memo-with-duplicate-page.pdf
diff --git a/example-docs/spring-weather.html.json b/example-docs/spring-weather.html.json
@@ -223,4 +223,4 @@
       "page_number": 1
     }
   }
-]
+]
diff --git a/test_unstructured/documents/test_elements.py b/test_unstructured/documents/test_elements.py
@@ -4,6 +4,7 @@
 
 from __future__ import annotations
 
+import copy
 import json
 import pathlib
 from functools import partial
@@ -28,9 +29,29 @@
     RegexMetadata,
     Text,
     Title,
+    assign_and_map_hash_ids,
 )
 
 
+@pytest.mark.parametrize("element", [Element(), Text(text=""), CheckBox()])
+def test_Element_autoassigns_a_UUID_then_becomes_an_idempotent_and_deterministic_hash(
+    element: Element,
+):
+    # -- element self-assigns itself a UUID --
+    assert isinstance(element.id, str)
+    assert len(element.id) == 36
+    assert element.id.count("-") == 4
+
+    expected_hash = "5336294a19f32ff03ef80066fbc3e0f7"
+    # -- calling `.id_to_hash()` changes the element's id-type to hash --
+    assert element.id_to_hash(0) == expected_hash
+    assert element.id == expected_hash
+
+    # -- `.id_to_hash()` is idempotent --
+    assert element.id_to_hash(0) == expected_hash
+    assert element.id == expected_hash
+
+
 def test_Text_is_JSON_serializable():
     # -- This shold run without an error --
     json.dumps(Text(text="hello there!", element_id=None).to_dict())
@@ -45,25 +66,11 @@ def test_Text_is_JSON_serializable():
         CheckBox(),
     ],
 )
-def test_Element_autoassigns_a_UUID_then_becomes_an_idempotent_and_deterministic_hash(
-    element: Element,
-):
-    assert element._element_id is None, "Element should not have an ID yet"
-
-    # -- element self-assigns itself a UUID only when the ID is requested --
+def test_Element_self_assigns_itself_a_UUID_id(element: Element):
     assert isinstance(element.id, str)
     assert len(element.id) == 36
     assert element.id.count("-") == 4
 
-    expected_hash = "e3b0c44298fc1c149afbf4c8996fb924"
-    # -- calling `.id_to_hash()` changes the element's id-type to hash --
-    assert element.id_to_hash() == expected_hash
-    assert element.id == expected_hash
-
-    # -- `.id_to_hash()` is idempotent --
-    assert element.id_to_hash() == expected_hash
-    assert element.id == expected_hash
-
 
 def test_text_element_apply_cleaners():
     text_element = Text(text="[1] A Textbook on Crocodile Habitats")
@@ -408,9 +415,10 @@ def and_it_serializes_an_orig_elements_sub_object_to_base64_when_it_is_present(s
         assert meta.to_dict() == {
             "category_depth": 1,
             "orig_elements": (
-                "eJyFzcsKwjAQheFXKVm7yDS3xjcQXNaViKTJjBR6o46glr67zVI3Lmf4Dv95EdhhjwNf2yT2hYDGUaWt"
-                "JVm5WDoqNUL0UoJrqtLHJHaF6JFDChw2v6zbzfjkvD2OM/YZ8GvC/Khb7lBs5LcilUwRyCsblQYTiBQp"
-                "ZRxYZcCA/1spDtP98dU6DTEw3sa5fWOqs10vH0cLQn0="
+                "eJyFzcsKwjAQheFXKVm7MGkzbXwDocu6EpFcTqTQG3UEtfTdbZa"
+                "6cTnDd/jPi0CHHgNf2yAOmXCljjqXoErKoIw3hqJRXlPuyphrEr"
+                "tM9GAbLNvNL+t2M56ctvU4o0+AXxPSo2m5g9jIb6VwBE0VBSujp"
+                "1LJ6EiRLpwiSBf3fyvZcbo/vlqnwVvGbZzbN0KT7Hr5AG/eQyM="
             ),
             "page_number": 2,
         }
@@ -666,3 +674,73 @@ def it_can_find_the_consolidation_strategy_for_each_of_its_known_fields(self):
                 f"ElementMetadata field `.{field_name}` does not have a consolidation strategy."
                 f" Add one in `ConsolidationStrategy.field_consolidation_strategies()."
             )
+
+
+def test_hash_ids_are_unique_for_duplicate_elements():
+    # GIVEN
+    parent = Text(text="Parent", metadata=ElementMetadata(page_number=1))
+    elements = [
+        parent,
+        Text(text="Element", metadata=ElementMetadata(page_number=1, parent_id=parent.id)),
+        Text(text="Element", metadata=ElementMetadata(page_number=1, parent_id=parent.id)),
+    ]
+
+    # WHEN
+    updated_elements = assign_and_map_hash_ids(copy.deepcopy(elements))
+    ids = [element.id for element in updated_elements]
+
+    # THEN
+    assert len(ids) == len(set(ids)), "Recalculated IDs must be unique."
+    assert elements[1].metadata.parent_id == elements[2].metadata.parent_id
+
+    for idx, updated_element in enumerate(updated_elements):
+        assert updated_element.id != elements[idx].id, "IDs haven't changed after recalculation"
+        if updated_element.metadata.parent_id is not None:
+            assert updated_element.metadata.parent_id in ids, "Parent ID not in the list of IDs"
+            assert (
+                updated_element.metadata.parent_id != elements[idx].metadata.parent_id
+            ), "Parent ID hasn't changed after recalculation"
+
+
+def test_hash_ids_are_deterministic():
+    parent = Text(text="Parent", metadata=ElementMetadata(page_number=1))
+    elements = [
+        parent,
+        Text(text="Element", metadata=ElementMetadata(page_number=1, parent_id=parent.id)),
+        Text(text="Element", metadata=ElementMetadata(page_number=1, parent_id=parent.id)),
+    ]
+
+    updated_elements = assign_and_map_hash_ids(elements)
+    ids = [element.id for element in updated_elements]
+    parent_ids = [element.metadata.parent_id for element in updated_elements]
+
+    assert ids == [
+        "ea9eb7e80383c190f8cafce1ad666624",
+        "4112a8d24886276e18e759d06956021b",
+        "eba84bbe7f03e8b91a1527323040ee3d",
+    ]
+    assert parent_ids == [
+        None,
+        "ea9eb7e80383c190f8cafce1ad666624",
+        "ea9eb7e80383c190f8cafce1ad666624",
+    ]
+
+
+@pytest.mark.parametrize(
+    ("text", "sequence_number", "filename", "page_number", "expected_hash"),
+    [
+        # -- pdf files support page numbers --
+        ("foo", 1, "foo.pdf", 1, "4bb264eb23ceb44cd8fcc5af44f8dc71"),
+        ("foo", 2, "foo.pdf", 1, "75fc1de48cf724ec00aa8d1c5a0d3758"),
+        # -- txt files don't have a page number --
+        ("some text", 0, "some.txt", None, "1a2627b5760c06b1440102f11a1edb0f"),
+        ("some text", 1, "some.txt", None, "e3fd10d867c4a1c0264dde40e3d7e45a"),
+    ],
+)
+def test_id_to_hash_calculates(text, sequence_number, filename, page_number, expected_hash):
+    element = Text(
+        text=text,
+        metadata=ElementMetadata(filename=filename, page_number=page_number),
+    )
+    assert element.id_to_hash(sequence_number) == expected_hash, "Returned ID does not match"
+    assert element.id == expected_hash, "ID should be set"
diff --git a/test_unstructured/documents/test_email_elements.py b/test_unstructured/documents/test_email_elements.py
@@ -4,21 +4,41 @@
 
 from unstructured.cleaners.core import clean_prefix
 from unstructured.cleaners.translate import translate_text
-from unstructured.documents.email_elements import EmailElement, Name
+from unstructured.documents.email_elements import EmailElement, Name, Subject
+
+
+@pytest.mark.parametrize(
+    "element", [EmailElement(text=""), Name(text="", name=""), Subject(text="")]
+)
+def test_EmailElement_autoassigns_a_UUID_then_becomes_an_idempotent_and_deterministic_hash(
+    element: EmailElement,
+):
+    # -- element self-assigns itself a UUID --
+    assert isinstance(element.id, str)
+    assert len(element.id) == 36
+    assert element.id.count("-") == 4
+
+    expected_hash = "5336294a19f32ff03ef80066fbc3e0f7"
+    # -- calling `.id_to_hash()` changes the element's id-type to hash --
+    assert element.id_to_hash(0) == expected_hash
+    assert element.id == expected_hash
+
+    # -- `.id_to_hash()` is idempotent --
+    assert element.id_to_hash(0) == expected_hash
 
 
 def test_Name_should_assign_a_deterministic_and_an_idempotent_hash():
     element = Name(name="Example", text="hello there!")
-    expected_hash = "c69509590d81db2f37f9d75480c8efed"
+    expected_hash = "7d191bcecf80c122578c497de5f0dae7"
 
     assert element._element_id is None, "Element should not have an ID yet"
 
     # -- calculating hash for the first time --
-    assert element.id_to_hash() == expected_hash
+    assert element.id_to_hash(0) == expected_hash
     assert element.id == expected_hash
 
     # -- `.id_to_hash()` is idempotent --
-    assert element.id_to_hash() == expected_hash
+    assert element.id_to_hash(0) == expected_hash
     assert element.id == expected_hash
 
 
@@ -30,7 +50,7 @@ def test_Name_should_assign_a_deterministic_and_an_idempotent_hash():
         Name(name="Example", text="hello there!", element_id=None),
     ],
 )
-def test_EmailElement_should_assign_a_UUID_only_once_and_only_at_the_first_id_request(
+def test_EmailElement_assigns_a_UUID_only_once_and_only_at_the_first_id_request(
     element: EmailElement,
 ):
     assert element._element_id is None, "Element should not have an ID yet"

diff --git a/test_unstructured/partition/docx/test_doc.py b/test_unstructured/partition/docx/test_doc.py
@@ -19,6 +19,20 @@
 from unstructured.partition.docx import partition_docx
 
 
+def test_partition_doc_for_deterministic_and_unique_ids():
+    ids = [element.id for element in partition_doc("example-docs/duplicate-paragraphs.doc")]
+
+    assert ids == [
+        "ade273c622c48d67a7be7b3816d5b4d8",
+        "7d0b32fdf169f9578723486cb4bc1235",
+        "1feb6e8e9c1662cfaef75907aeeb0900",
+        "aa2a8ac10143b12f0fe2087837ea11d2",
+        "da31ba7ed3919067d2c6572dc1617271",
+        "1914359c179a160df921b769acf8c353",
+        "f9d0d379fc791bae487b7a45f65caa50",
+    ]
+
+
 @pytest.fixture()
 def mock_document():
     document = docx.Document()

diff --git a/test_unstructured/partition/docx/test_docx.py b/test_unstructured/partition/docx/test_docx.py
@@ -866,3 +866,18 @@ def mock_document_file_path(mock_document: Document, tmp_path: pathlib.Path) ->
     filename = str(tmp_path / "mock_document.docx")
     mock_document.save(filename)
     return filename
+
+
+def test_ids_are_unique_and_deterministic():
+    elements = partition_docx("example-docs/duplicate-paragraphs.docx")
+
+    ids = [e.id for e in elements]
+    assert ids == [
+        "2f22d82eea1faf5f40dac60cef52700e",
+        "ca9e1f448e531a5152d960e14eefc360",
+        "9ddeacb172ac17fb45e6f3f15f3c703d",
+        "a4fd85d3f4141acae38c8f9c936ed2f3",
+        "44ebaaf66640719c918246d4ccba1c45",
+        "f36e8ebcb3b6a051940a168fe73cbc44",
+        "532b395177652c7d61e1e4d855f1dc1d",
+    ], "IDs are not deterministic"
diff --git a/test_unstructured/partition/pdf_image/test_pdf.py b/test_unstructured/partition/pdf_image/test_pdf.py
@@ -1240,3 +1240,71 @@ def test_partition_pdf_always_keep_all_image_elements(
     )
     image_elements = [el for el in elements if el.category == ElementType.IMAGE]
     assert len(image_elements) == 3
+
+
+@pytest.fixture()
+def expected_element_ids_for_fast_strategy():
+    return [
+        "27a6cb3e5a4ad399b2f865729bbd3840",
+        "a90a54baba0093296a013d26b7acbc17",
+        "9be424e2d151dac4b5f36a85e9bbfe65",
+        "4631da875fb4996c63b2d80cea6b588e",
+        "6264f4eda97a049f4710f9bea0c01cbd",
+        "abded7b2ff3a5542c88b4a831755ec24",
+        "b781ea5123cb31e0571391b7b42cac75",
+        "033f27d2618ba4cda9068b267b5a731e",
+        "8982a12fcced30dd12ccbf61d14f30bf",
+        "41af2fd5df0cf47aa7e8ecca200d3ac6",
+    ]
+
+
+@pytest.fixture()
+def expected_element_ids_for_hi_res_strategy():
+    return [
+        "27a6cb3e5a4ad399b2f865729bbd3840",
+        "a90a54baba0093296a013d26b7acbc17",
+        "9be424e2d151dac4b5f36a85e9bbfe65",
+        "4631da875fb4996c63b2d80cea6b588e",
+        "6264f4eda97a049f4710f9bea0c01cbd",
+        "abded7b2ff3a5542c88b4a831755ec24",
+        "b781ea5123cb31e0571391b7b42cac75",
+        "033f27d2618ba4cda9068b267b5a731e",
+        "8982a12fcced30dd12ccbf61d14f30bf",
+        "41af2fd5df0cf47aa7e8ecca200d3ac6",
+    ]
+
+
+@pytest.fixture()
+def expected_element_ids_for_ocr_strategy():
+    return [
+        "272ab65cbe81795161128aea59599d83",
+        "b38affd7bbbb3dddf5c85ba8b14d380d",
+        "65903214d456b8b3cba6faa6714bd9ba",
+        "5b41ceae05dcfaeeac32ff8e82dc2ff1",
+        "6582fc6c6c595225feeddcc3263f0ae3",
+        "64b610c8f4274f1ce2175bf30814409d",
+        "8edde8bf2d3a68370dc4bd142c408ca4",
+        "a052bc17696043efce2e4f4f28393a83",
+    ]
+
+
+@pytest.fixture()
+def expected_ids(request):
+    return request.getfixturevalue(request.param)
+
+
+@pytest.mark.parametrize(
+    ("strategy", "expected_ids"),
+    [
+        (PartitionStrategy.FAST, "expected_element_ids_for_fast_strategy"),
+        (PartitionStrategy.HI_RES, "expected_element_ids_for_hi_res_strategy"),
+        (PartitionStrategy.OCR_ONLY, "expected_element_ids_for_ocr_strategy"),
+    ],
+    indirect=["expected_ids"],
+)
+def test_unique_and_deterministic_element_ids(strategy, expected_ids):
+    elements = pdf.partition_pdf(
+        "example-docs/fake-memo-with-duplicate-page.pdf", strategy=strategy, starting_page_number=2
+    )
+    ids = [element.id for element in elements]
+    assert ids == expected_ids, "Element IDs do not match expected IDs"
-Original file line number
+Diff line change
@@ Expand Up / @@ -223,4 +223,4 @@ @@
           "page_number": 1
         }
       }
-    ]
+    ]