build(deps): add makefile to requirements (#1295)

This PR resolves #1294 by adding a Makefile to compile requirements.
This makefile respects the dependencies between file and will compile
them in order. E.g., extra-*.txt will be compiled __after__ base.txt is
updated.

Test locally by simply running `make pip-compile` or `cd requirements &&
make clean && make all`

---------

Co-authored-by: qued <[email protected]>

Makefile

@@ -232,8 +232,6 @@ install-paddleocr:
 pip-compile:
 	@scripts/pip-compile.sh
 
-
-
 ## install-project-local:   install unstructured into your local python environment
 .PHONY: install-project-local
 install-project-local: install

requirements/Makefile

@@ -0,0 +1,30 @@
+PIP = pip-compile --upgrade -o $@ $<
+SHELL := /bin/bash
+# we don't need to make constraints.txt from constraints.in
+REQUIREMENTS := $(filter-out constraints.in,$(shell find . -type f -name '*.in'))
+REQUIREMENTSTXT := $(patsubst %.in,%.txt,$(REQUIREMENTS))
+
+# those dependencies only needs constraint.in to and %.in to compile
+constraint_dependent = base.txt extra-pptx.txt
+
+.PHONY: all $(REQUIREMENTSTXT)
+
+all: $(REQUIREMENTSTXT)
+
+# a special case where the .txt depends on other .txt; here dev.txt depends on test.txt
+dev.txt: dev.in test.txt base.txt constraints.in
+	$(PIP)
+
+# the most common case: an extra (or test.txt) that depends on base.txt and %.in
+$(REQUIREMENTSTXT): %.txt: %.in base.txt constraints.in
+	$(PIP)
+
+# special case where its only extra dependency is constraints.in; here base.txt and extra-ppt.txt
+# only depends on their own .in and constraits.in
+$(constraint_dependent): %.txt: %.in constraints.in
+	$(PIP)
+
+.PHONY: clean
+
+clean:
+	rm $(REQUIREMENTSTXT)

scripts/pip-compile.sh

@@ -8,24 +8,9 @@ if ! python -c "import sys; assert sys.version_info.major == $major and sys.vers
   exit 1
 fi
 
-# NOTE(alan): Order matters in compiling. Start with base and test. Then do the rest in any order.
-echo "running: pip-compile --upgrade base.in"
-pip-compile --upgrade "requirements/base.in" -c requirements/constraints.in
-echo "running: pip-compile --upgrade test.in"
-pip-compile --upgrade "requirements/test.in" -c requirements/constraints.in
+pushd ./requirements || exit
+make clean
+make all
+popd || exit
 
-for file in requirements/*.in; do
-  if [[ "$file" =~ "constraints" ]] || [[ "$file" =~ "base" ]] || [[ "$file" =~ "test" ]]; then
-    continue;
-  fi;
-  echo "running: pip-compile --upgrade $file"
-  pip-compile --upgrade "$file" -c requirements/constraints.in
-done
-
-for file in requirements/**/*.in; do
-  if [[ "$file" =~ "constraints" ]] || [[ "$file" =~ "base" ]] || [[ "$file" =~ "test" ]]; then
-    continue;
-  fi;
-  echo "running: pip-compile --upgrade $file"
-  pip-compile --upgrade "$file" -c requirements/constraints.in
-done
+cp requirements/build.txt docs/requirements.txt