diff --git a/CHANGELOG.md b/CHANGELOG.md index d193864e..2bff3b80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,21 @@ +## 0.7.33-dev0 + +* deps: remove layoutparser optional dependencies + ## 0.7.32 -* refactor: remove all code related to filling inferred elements text from embedded text (pdfminer). +* refactor: remove all code related to filling inferred elements text from embedded text (pdfminer). * bug: set the Chipper max_length variable ## 0.7.31 -* refactor: remove all `cid` related code that was originally added to filter out invalid `pdfminer` text +* refactor: remove all `cid` related code that was originally added to filter out invalid `pdfminer` text * enhancement: Wrapped hf_hub_download with a function that checks for local file before checking HF ## 0.7.30 -* fix: table transformer doesn't return multiple cells with same coordinates -* +* fix: table transformer doesn't return multiple cells with same coordinates +* ## 0.7.29 * fix: table transformer predictions are now removed if confidence is below threshold @@ -458,4 +462,4 @@ we have the mapping from standard language code to paddle language code. ## 0.2.0 -* Initial release of unstructured-inference \ No newline at end of file +* Initial release of unstructured-inference diff --git a/requirements/base.in b/requirements/base.in index 9f321b26..55789e82 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,5 +1,5 @@ -c constraints.in -layoutparser[layoutmodels,tesseract] +layoutparser python-multipart huggingface-hub opencv-python!=4.7.0.68 diff --git a/requirements/base.txt b/requirements/base.txt index e4512b00..63dadcfb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,8 +4,6 @@ # # pip-compile requirements/base.in # -antlr4-python3-runtime==4.9.3 - # via omegaconf certifi==2024.2.2 # via requests cffi==1.16.0 @@ -16,73 +14,43 @@ charset-normalizer==3.3.2 # requests coloredlogs==15.0.1 # via onnxruntime -contourpy==1.2.1 - # via matplotlib -cryptography==42.0.5 +cryptography==42.0.7 # via pdfminer-six -cycler==0.12.1 - # via matplotlib -effdet==0.4.1 - # via layoutparser -filelock==3.13.4 +filelock==3.14.0 # via # huggingface-hub - # torch # transformers flatbuffers==24.3.25 # via onnxruntime -fonttools==4.51.0 - # via matplotlib -fsspec==2024.3.1 - # via - # huggingface-hub - # torch -huggingface-hub==0.22.2 +fsspec==2024.5.0 + # via huggingface-hub +huggingface-hub==0.23.1 # via # -r requirements/base.in - # timm # tokenizers # transformers humanfriendly==10.0 # via coloredlogs idna==3.7 # via requests -importlib-resources==6.4.0 - # via matplotlib iopath==0.1.10 # via layoutparser -jinja2==3.1.3 - # via torch -kiwisolver==1.4.5 - # via matplotlib -layoutparser[layoutmodels,tesseract]==0.3.4 +layoutparser==0.3.4 # via -r requirements/base.in -markupsafe==2.1.5 - # via jinja2 -matplotlib==3.8.4 - # via pycocotools mpmath==1.3.0 # via sympy -networkx==3.2.1 - # via torch numpy==1.26.4 # via - # contourpy # layoutparser - # matplotlib # onnx # onnxruntime # opencv-python # pandas - # pycocotools # scipy - # torchvision # transformers -omegaconf==2.3.0 - # via effdet onnx==1.16.0 # via -r requirements/base.in -onnxruntime==1.17.3 +onnxruntime==1.18.0 # via -r requirements/base.in opencv-python==4.9.0.80 # via @@ -91,9 +59,7 @@ opencv-python==4.9.0.80 packaging==24.0 # via # huggingface-hub - # matplotlib # onnxruntime - # pytesseract # transformers pandas==2.2.2 # via layoutparser @@ -106,31 +72,20 @@ pdfplumber==0.11.0 pillow==10.3.0 # via # layoutparser - # matplotlib # pdf2image # pdfplumber - # pytesseract - # torchvision portalocker==2.8.2 # via iopath protobuf==5.26.1 # via # onnx # onnxruntime -pycocotools==2.0.7 - # via effdet pycparser==2.22 # via cffi -pyparsing==3.1.2 - # via matplotlib -pypdfium2==4.29.0 +pypdfium2==4.30.0 # via pdfplumber -pytesseract==0.3.10 - # via layoutparser python-dateutil==2.9.0.post0 - # via - # matplotlib - # pandas + # via pandas python-multipart==0.0.9 # via -r requirements/base.in pytz==2024.1 @@ -139,59 +94,37 @@ pyyaml==6.0.1 # via # huggingface-hub # layoutparser - # omegaconf - # timm # transformers -rapidfuzz==3.8.1 +rapidfuzz==3.9.1 # via -r requirements/base.in -regex==2024.4.16 +regex==2024.5.15 # via transformers -requests==2.31.0 +requests==2.32.2 # via # huggingface-hub # transformers safetensors==0.4.3 - # via - # timm - # transformers + # via transformers scipy==1.13.0 # via layoutparser six==1.16.0 # via python-dateutil sympy==1.12 - # via - # onnxruntime - # torch -timm==0.9.16 - # via effdet + # via onnxruntime tokenizers==0.19.1 # via transformers -torch==2.2.2 - # via - # effdet - # layoutparser - # timm - # torchvision -torchvision==0.17.2 - # via - # effdet - # layoutparser - # timm -tqdm==4.66.2 +tqdm==4.66.4 # via # huggingface-hub # iopath # transformers -transformers==4.40.0 +transformers==4.41.0 # via -r requirements/base.in typing-extensions==4.11.0 # via # huggingface-hub # iopath - # torch tzdata==2024.1 # via pandas urllib3==2.2.1 # via requests -zipp==3.18.1 - # via importlib-resources diff --git a/requirements/dev.txt b/requirements/dev.txt index b065f8f4..b76c1905 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -25,7 +25,7 @@ attrs==23.2.0 # via # jsonschema # referencing -babel==2.14.0 +babel==2.15.0 # via jupyterlab-server beautifulsoup4==4.12.3 # via nbconvert @@ -58,13 +58,9 @@ comm==0.2.2 # ipykernel # ipywidgets contourpy==1.2.1 - # via - # -c requirements/base.txt - # matplotlib + # via matplotlib cycler==0.12.1 - # via - # -c requirements/base.txt - # matplotlib + # via matplotlib debugpy==1.8.1 # via ipykernel decorator==5.1.1 @@ -81,9 +77,7 @@ executing==2.0.1 fastjsonschema==2.19.1 # via nbformat fonttools==4.51.0 - # via - # -c requirements/base.txt - # matplotlib + # via matplotlib fqdn==1.5.1 # via jsonschema h11==0.14.0 @@ -115,9 +109,7 @@ importlib-metadata==7.1.0 # jupyterlab-server # nbconvert importlib-resources==6.4.0 - # via - # -c requirements/base.txt - # matplotlib + # via matplotlib ipykernel==6.29.4 # via # jupyter @@ -136,9 +128,8 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.1 # via ipython -jinja2==3.1.3 +jinja2==3.1.4 # via - # -c requirements/base.txt # jupyter-server # jupyterlab # jupyterlab-server @@ -147,7 +138,7 @@ json5==0.9.25 # via jupyterlab-server jsonpointer==2.4 # via jsonschema -jsonschema[format-nongpl]==4.21.1 +jsonschema[format-nongpl]==4.22.0 # via # jupyter-events # jupyterlab-server @@ -189,29 +180,24 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.6 +jupyterlab==4.2.0 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.26.0 +jupyterlab-server==2.27.1 # via # jupyterlab # notebook jupyterlab-widgets==3.0.10 # via ipywidgets kiwisolver==1.4.5 - # via - # -c requirements/base.txt - # matplotlib + # via matplotlib markupsafe==2.1.5 # via - # -c requirements/base.txt # jinja2 # nbconvert -matplotlib==3.8.4 - # via - # -c requirements/base.txt - # -r requirements/dev.in +matplotlib==3.9.0 + # via -r requirements/dev.in matplotlib-inline==0.1.7 # via # ipykernel @@ -220,7 +206,7 @@ mistune==3.0.2 # via nbconvert nbclient==0.10.0 # via nbconvert -nbconvert==7.16.3 +nbconvert==7.16.4 # via # jupyter # jupyter-server @@ -231,7 +217,7 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -notebook==7.1.3 +notebook==7.2.0 # via jupyter notebook-shim==0.2.4 # via @@ -270,7 +256,7 @@ pillow==10.3.0 # matplotlib pip-tools==7.4.1 # via -r requirements/dev.in -platformdirs==4.2.0 +platformdirs==4.2.2 # via # -c requirements/test.txt # jupyter-core @@ -292,17 +278,15 @@ pycparser==2.22 # via # -c requirements/base.txt # cffi -pygments==2.17.2 +pygments==2.18.0 # via # ipython # jupyter-console # nbconvert # qtconsole pyparsing==3.1.2 - # via - # -c requirements/base.txt - # matplotlib -pyproject-hooks==1.0.0 + # via matplotlib +pyproject-hooks==1.1.0 # via # build # pip-tools @@ -319,23 +303,23 @@ pyyaml==6.0.1 # -c requirements/base.txt # -c requirements/test.txt # jupyter-events -pyzmq==26.0.0 +pyzmq==26.0.3 # via # ipykernel # jupyter-client # jupyter-console # jupyter-server # qtconsole -qtconsole==5.5.1 +qtconsole==5.5.2 # via jupyter qtpy==2.4.1 # via qtconsole -referencing==0.34.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events -requests==2.31.0 +requests==2.32.2 # via # -c requirements/base.txt # -c requirements/test.txt @@ -348,7 +332,7 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rpds-py==0.18.0 +rpds-py==0.18.1 # via # jsonschema # referencing @@ -374,7 +358,7 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -tinycss2==1.2.1 +tinycss2==1.3.0 # via nbconvert tomli==2.0.1 # via @@ -382,7 +366,6 @@ tomli==2.0.1 # build # jupyterlab # pip-tools - # pyproject-hooks tornado==6.4 # via # ipykernel @@ -391,7 +374,7 @@ tornado==6.4 # jupyterlab # notebook # terminado -traitlets==5.14.2 +traitlets==5.14.3 # via # comm # ipykernel @@ -432,15 +415,14 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.7.0 +websocket-client==1.8.0 # via jupyter-server wheel==0.43.0 # via pip-tools widgetsnbextension==4.0.10 # via ipywidgets -zipp==3.18.1 +zipp==3.18.2 # via - # -c requirements/base.txt # importlib-metadata # importlib-resources diff --git a/requirements/test.txt b/requirements/test.txt index 864b5fe3..5f738730 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,7 +6,7 @@ # anyio==4.3.0 # via httpx -black==24.4.0 +black==24.4.2 # via -r requirements/test.in certifi==2024.2.2 # via @@ -22,7 +22,7 @@ click==8.1.7 # via # -r requirements/test.in # black -coverage[toml]==7.4.4 +coverage[toml]==7.5.1 # via # -r requirements/test.in # pytest-cov @@ -30,7 +30,7 @@ exceptiongroup==1.2.1 # via # anyio # pytest -filelock==3.13.4 +filelock==3.14.0 # via # -c requirements/base.txt # huggingface-hub @@ -40,7 +40,7 @@ flake8==7.0.0 # flake8-docstrings flake8-docstrings==1.7.0 # via -r requirements/test.in -fsspec==2024.3.1 +fsspec==2024.5.0 # via # -c requirements/base.txt # huggingface-hub @@ -50,7 +50,7 @@ httpcore==1.0.5 # via httpx httpx==0.27.0 # via -r requirements/test.in -huggingface-hub==0.22.2 +huggingface-hub==0.23.1 # via # -c requirements/base.txt # -r requirements/test.in @@ -64,7 +64,7 @@ iniconfig==2.0.0 # via pytest mccabe==0.7.0 # via flake8 -mypy==1.9.0 +mypy==1.10.0 # via -r requirements/test.in mypy-extensions==1.0.0 # via @@ -86,9 +86,9 @@ pillow==10.3.0 # via # -c requirements/base.txt # pdf2image -platformdirs==4.2.0 +platformdirs==4.2.2 # via black -pluggy==1.4.0 +pluggy==1.5.0 # via pytest pycodestyle==2.11.1 # via flake8 @@ -96,7 +96,7 @@ pydocstyle==6.3.0 # via flake8-docstrings pyflakes==3.2.0 # via flake8 -pytest==8.1.1 +pytest==8.2.1 # via # pytest-cov # pytest-mock @@ -108,11 +108,11 @@ pyyaml==6.0.1 # via # -c requirements/base.txt # huggingface-hub -requests==2.31.0 +requests==2.32.2 # via # -c requirements/base.txt # huggingface-hub -ruff==0.4.0 +ruff==0.4.4 # via -r requirements/test.in sniffio==1.3.1 # via @@ -126,7 +126,7 @@ tomli==2.0.1 # coverage # mypy # pytest -tqdm==4.66.2 +tqdm==4.66.4 # via # -c requirements/base.txt # huggingface-hub diff --git a/unstructured_inference/__version__.py b/unstructured_inference/__version__.py index e19533c3..2ee5708c 100644 --- a/unstructured_inference/__version__.py +++ b/unstructured_inference/__version__.py @@ -1 +1 @@ -__version__ = "0.7.32" # pragma: no cover +__version__ = "0.7.33" # pragma: no cover diff --git a/unstructured_inference/models/detectron2.py b/unstructured_inference/models/detectron2.py deleted file mode 100644 index 520298ce..00000000 --- a/unstructured_inference/models/detectron2.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Any, Dict, Final, List, Optional, Union - -from layoutparser.models.detectron2.layoutmodel import ( - Detectron2LayoutModel, - is_detectron2_available, -) -from layoutparser.models.model_config import LayoutModelConfig -from PIL import Image as PILImage - -from unstructured_inference.constants import ElementType -from unstructured_inference.inference.layoutelement import LayoutElement -from unstructured_inference.logger import logger -from unstructured_inference.models.unstructuredmodel import ( - UnstructuredObjectDetectionModel, -) -from unstructured_inference.utils import ( - LazyDict, - LazyEvaluateInfo, - download_if_needed_and_get_local_path, -) - -DETECTRON_CONFIG: Final = "lp://PubLayNet/faster_rcnn_R_50_FPN_3x/config" -DEFAULT_LABEL_MAP: Final[Dict[int, str]] = { - 0: ElementType.TEXT, - 1: ElementType.TITLE, - 2: ElementType.LIST, - 3: ElementType.TABLE, - 4: ElementType.FIGURE, -} -DEFAULT_EXTRA_CONFIG: Final[List[Any]] = ["MODEL.ROI_HEADS.SCORE_THRESH_TEST", 0.8] - - -# NOTE(alan): Entries are implemented as LazyDicts so that models aren't downloaded until they are -# needed. -MODEL_TYPES = { - "detectron2_lp": LazyDict( - model_path=LazyEvaluateInfo( - download_if_needed_and_get_local_path, - "layoutparser/detectron2", - "PubLayNet/faster_rcnn_R_50_FPN_3x/model_final.pth", - ), - config_path=LazyEvaluateInfo( - download_if_needed_and_get_local_path, - "layoutparser/detectron2", - "PubLayNet/faster_rcnn_R_50_FPN_3x/config.yml", - ), - label_map=DEFAULT_LABEL_MAP, - extra_config=DEFAULT_EXTRA_CONFIG, - ), - "checkbox": LazyDict( - model_path=LazyEvaluateInfo( - download_if_needed_and_get_local_path, - "unstructuredio/oer-checkbox", - "detectron2_finetuned_oer_checkbox.pth", - ), - config_path=LazyEvaluateInfo( - download_if_needed_and_get_local_path, - "unstructuredio/oer-checkbox", - "detectron2_oer_checkbox.json", - ), - label_map={0: "Unchecked", 1: "Checked"}, - extra_config=None, - ), -} - - -class UnstructuredDetectronModel(UnstructuredObjectDetectionModel): - """Unstructured model wrapper for Detectron2LayoutModel.""" - - def predict(self, x: PILImage.Image): - """Makes a prediction using detectron2 model.""" - super().predict(x) - prediction = self.model.detect(x) - return [LayoutElement.from_lp_textblock(block) for block in prediction] - - def initialize( - self, - config_path: Union[str, Path, LayoutModelConfig], - model_path: Optional[Union[str, Path]] = None, - label_map: Optional[Dict[int, str]] = None, - extra_config: Optional[list] = None, - device: Optional[str] = None, - ): - """Loads the detectron2 model using the specified parameters""" - - if not is_detectron2_available(): - raise ImportError( - "Failed to load the Detectron2 model. Ensure that the Detectron2 " - "module is correctly installed.", - ) - - config_path_str = str(config_path) - model_path_str: Optional[str] = None if model_path is None else str(model_path) - logger.info("Loading the Detectron2 layout model ...") - self.model = Detectron2LayoutModel( - config_path_str, - model_path=model_path_str, - label_map=label_map, - extra_config=extra_config, - device=device, - )