feat: clean cid

Unstructured-IO · May 8, 2024 · 5fb5b4c · 5fb5b4c
1 parent 9a7e8aa
commit 5fb5b4c
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/unstructured/partition/pdf_image/pdf_image_utils.py b/unstructured/partition/pdf_image/pdf_image_utils.py
@@ -248,6 +248,12 @@ def is_cid_present(text: str) -> bool:
     return text.find("(cid:") != -1
 
 
+def clean_cid(text: str) -> str:
+    cid_pattern = r"\(cid\:(\d+)\)"
+    cleaned_text = re.sub(cid_pattern, "", text)
+    return cleaned_text
+
+
 def annotate_layout_elements_with_image(
     inferred_page_layout: "PageLayout",
     extracted_page_layout: Optional["PageLayout"],

diff --git a/unstructured/partition/pdf_image/pdfminer_processing.py b/unstructured/partition/pdf_image/pdfminer_processing.py
@@ -3,6 +3,7 @@
 from pdfminer.utils import open_filename
 
 from unstructured.documents.elements import ElementType
+from unstructured.partition.pdf_image.pdf_image_utils import clean_cid
 from unstructured.partition.pdf_image.pdfminer_utils import (
     get_images_from_pdf_element,
     open_pdfminer_pages_generator,
@@ -66,12 +67,14 @@ def process_data_with_pdfminer(
                 else:
                     continue
 
+            text = clean_cid(_text) if _text else _text
+
             text_region = element_class.from_coords(
                 x1 * coef,
                 y1 * coef,
                 x2 * coef,
                 y2 * coef,
-                text=_text,
+                text=text,
                 source=Source.PDFMINER,
             )