Numpy-based Object Detection for Bounding Boxes (#748)

.github/workflows/build-and-publish.yml

@@ -23,6 +23,23 @@ jobs:
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
           packages-dir: ./client/dist
+  build-and-publish-py-lite-package:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./lite
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Build wheel
+        run: pip install build && python -m build
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.VALOR_LITE_PYPI_API_TOKEN }}
+          packages-dir: ./lite/dist
   build-and-publish-ts-package:
     runs-on: ubuntu-latest
     defaults:

.github/workflows/lite-benchmark-evaluations.yml

@@ -0,0 +1,38 @@
+name: Run valor-lite benchmarks
+
+on:
+  push:
+    branches: "**"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  run-benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: install lite
+        run: pip install -e .
+        working-directory: ./lite
+      # - name: run classification benchmarks
+      #   run: python benchmark_script.py
+      #   working-directory: ./lite/benchmarks/classification
+      # - name: print classification results
+      #   run: |
+      #     export BENCHMARK_RESULTS=$(python -c "import os;import json;print(json.dumps(json.load(open('results.json', 'r')), indent=4));")
+      #     echo "$BENCHMARK_RESULTS"
+      #   working-directory: ./lite/benchmarks/classification
+      - name: run object detection benchmarks
+        run: python benchmark_objdet.py
+        working-directory: ./lite/benchmarks/
+      - name: print object detection results
+        run: |
+          export BENCHMARK_RESULTS=$(python -c "import os;import json;print(json.dumps(json.load(open('manager_results.json', 'r')), indent=4));")
+          echo "$BENCHMARK_RESULTS"
+        working-directory: ./lite/benchmarks/
+      - run: make stop-env

.github/workflows/lite-tests-and-coverage.yml

@@ -0,0 +1,35 @@
+name: Run valor-lite code coverage report
+
+on:
+  push:
+    branches: "**"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  lite-tests:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: run object detection tests and report coverage
+        run: |
+          pip install -e ".[test]"
+          COVERAGE_FILE=.coverage.functional python -m coverage run --omit "tests/*"  -m pytest -v tests/detection/
+          python -m coverage combine
+          python -m coverage report -m
+          python -m coverage json
+          export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
+          echo "total=$TOTAL" >> $GITHUB_ENV
+          if (( $TOTAL < 90 )); then
+            echo "Coverage is below 90%"
+            exit 1
+          fi
+        working-directory: ./lite

integration_tests/client/metrics/test_detection.py

@@ -2481,7 +2481,7 @@ def test_evaluate_detection_false_negatives_two_images_one_only_with_different_c
 ):
     """In this test we have
         1. An image with a matching groundtruth and prediction (same class, `"value"`, and high IOU)
-        2. A second image with a groundtruth annotation with clas `"other value"` and a prediction with higher confidence
+        2. A second image with a groundtruth annotation with class `"other value"` and a prediction with higher confidence
         then the prediction on the first image.
 
     In this case, the AP for class `"value"` should be 0.5 since the false positive has higher confidence than the true positive.

lite/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Striveworks
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

lite/README.md

@@ -0,0 +1 @@
+# valor-lite: Compute classification, object detection, and segmentation metrics locally.

lite/benchmarks/.gitignore

@@ -0,0 +1,2 @@
+*.json
+*.jsonl