WIP: update

LutingWang · Jul 13, 2024 · 974a01d · 974a01d
1 parent a5760e2
commit 974a01d
Show file tree

Hide file tree

Showing 60 changed files with 1,224 additions and 885 deletions.
diff --git a/.flake8 b/.flake8
@@ -1,5 +1,5 @@
 [flake8]
-ignore = E126,E201,E241,W503,W504
+ignore = D100,D101,D102,D103,D104,D105,D106,D107,E126,E201,E241,N805,N812,N817,W503,W504
 per-file-ignores =
     oadp/__init__.py:E402,F401,F403
     oadp/**/__init__.py:F401,F403
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.11"]
     steps:
       - uses: actions/checkout@master
       - uses: actions/setup-python@master

diff --git a/.gitignore b/.gitignore
@@ -161,6 +161,6 @@ cython_debug/
 #.idea/
 
 .DS_STORE
-data
-pretrained
-work_dirs
+/data
+/pretrained
+/work_dirs
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,59 +1,110 @@
 default_language_version:
-  python: python3.10
+  python: python3.11
 repos:
+  - repo: local
+    hooks:
+      - id: pylint
+        name: pylint
+        entry: pylint
+        language: system
+        types: [python]
+        args:
+          - -sn
+        exclude: ^configs/
+  - repo: meta
+    hooks:
+      - id: check-hooks-apply
+      - id: check-useless-excludes
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-ast
+      - id: check-case-conflict
+      - id: check-docstring-first
+      - id: check-json
       - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: check-toml
+      - id: check-vcs-permalinks
       - id: check-yaml
       - id: debug-statements
+      - id: destroyed-symlinks
+      - id: detect-private-key
       - id: end-of-file-fixer
       - id: fix-encoding-pragma
         args:
           - --remove
+      - id: forbid-submodules
       - id: mixed-line-ending
         args:
           - --fix=lf
-      - id: requirements-txt-fixer
-        files: requirements.txt
+      - id: pretty-format-json
+        args:
+          - --autofix
       - id: trailing-whitespace
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-  - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-  - repo: https://github.com/pycqa/docformatter
-    rev: v1.7.5
-    hooks:
-      - id: docformatter
-  - repo: https://github.com/google/yapf.git
-    rev: v0.40.2
-    hooks:
-      - id: yapf
-        additional_dependencies:
-          - toml
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.1.1
+    rev: v1.10.0
     hooks:
       - id: mypy
+        exclude: ^configs/
         additional_dependencies:
           - numpy
           - todd_ai
-        exclude: ^configs/
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.4
+    rev: v2.3.0
     hooks:
       - id: codespell
         args:
           - configs
           - oadp
           - tools
   - repo: https://github.com/commitizen-tools/commitizen
-    rev: v2.42.1
+    rev: v3.27.0
     hooks:
       - id: commitizen
+  - repo: https://github.com/google/yapf
+    rev: v0.40.2
+    hooks:
+      - id: yapf
+        require_serial: true
+  - repo: https://github.com/pycqa/autoflake
+    rev: v2.3.1
+    hooks:
+      - id: autoflake
+  - repo: https://github.com/pycqa/bandit
+    rev: 1.7.8
+    hooks:
+      - id: bandit
+        args:
+          - -c
+          - pyproject.toml
+        additional_dependencies:
+          - bandit[toml]
+  - repo: https://github.com/pycqa/doc8
+    rev: v1.1.1
+    hooks:
+      - id: doc8
+  - repo: https://github.com/pycqa/docformatter
+    rev: v1.7.5
+    hooks:
+      - id: docformatter
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-docstrings
+          - mccabe
+          - pep8-naming
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+  - repo: https://github.com/pycqa/pydocstyle
+    rev: 6.3.0
+    hooks:
+      - id: pydocstyle
+        additional_dependencies:
+          - pydocstyle[toml]
diff --git a/.todd_version b/.todd_version
@@ -0,0 +1 @@
+0ed76ff031b34506803acaa6ca85b54c8fb681bd
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,12 +1,11 @@
 {
-    "cSpell.words": [
-        "detpro",
-        "featmap",
-        "Netdisk",
-        "oadp",
-        "oake",
-        "segm",
-        "soco",
-        "vild"
-    ]
+  "cSpell.words": [
+    "CXCYWH",
+    "detpro",
+    "oadp",
+    "oake",
+    "soco",
+    "vild",
+    "XYXY"
+  ]
 }
diff --git a/Pipfile b/Pipfile
@@ -1,11 +1,12 @@
 [[source]]
-url = "https://pypi.org/simple"
+url = "https://bytedpypi.byted.org/simple"
 verify_ssl = true
-name = "pypi"
+name = "pip_conf_index_global"
 
 [packages]
 
 [dev-packages]
 
 [requires]
-python_version = "3.10"
+python_version = "3.11"
+python_full_version = "3.11.3"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/README.md b/README.md
@@ -233,21 +233,28 @@ Feature extraction can be very time consuming.
 Therefore, we provide archives of the extracted features on [Baidu Netdisk][].
 The extracted features are archived with the following command
 
-```bash
-cd data/coco/oake/
+```zsh
+split_zip() {
+    local split=$1
+    local prefix=objects_${split}2017
+    local directory=objects/${split}2017
+
+    ls ${directory} > ${prefix}.txt
+    split -l 3000 ${prefix}.txt ${prefix}.txt.
+
+    for f in ${prefix}.txt.*
+    do
+        echo $f
+        sed s:^:${directory}/: $f | zip -9q@ ${prefix}.${f:e}.zip
+    done
+
+    rm ${prefix}.txt*
+}
 
 tar -zcf globals.tar.gz globals
 tar -zcf blocks.tar.gz blocks
-tar -zcf objects.tar.gz objects/val2017
-
-cd objects/train2017
-ls > objects
-split -d -3000 - objects. < objects
-for i in objects.[0-9][0-9]; do
-    zip -q -9 "$i.zip" -@ < "$i"
-    mv "$i.zip" ../..
-done
-rm objects*
+split_zip val
+split_zip train
 ```
 
 The final directory for OAKE should look like
@@ -280,7 +287,7 @@ To conduct training for coco
 
 ```bash
 [DRY_RUN=True] [TRAIN_WITH_VAL_DATASET=True] (python|torchrun --nproc_per_node=${GPUS}) -m oadp.dp.train vild_ov_coco configs/dp/vild_ov_coco.py [--override .validator.dataloader.dataset.ann_file::data/coco/annotations/instances_val2017.48.json]
-[DRY_RUN=True] [TRAIN_WITH_VAL_DATASET=True] (python|torchrun --nproc_per_node=${GPUS}) -m oadp.dp.train oadp_ov_coco configs/dp/oadp_ov_coco.py [--override .validator.dataloader.dataset.ann_file::data/coco/annotations/instances_val2017.48.json]
+[DRY_RUN=True] [TRAIN_WITH_VAL_DATASET=True] (python|torchrun --nproc_per_node=${GPUS}) -m oadp.dp.train configs/dp/oadp_ov_coco.py --work-dir work_dirs/oadp_ov_coco [--override .validator.dataloader.dataset.ann_file::data/coco/annotations/instances_val2017.48.json]
 ```
 
 To conduct training for lvis

diff --git a/configs/dp/datasets/ov_coco.py b/configs/dp/datasets/ov_coco.py
@@ -4,7 +4,7 @@
 
 categories = 'coco'
 data_root = 'data/coco/'
-oake_root = data_root + 'oake/'
+oake_root = 'work_dirs/oake/'
 ann_file_prefix = 'annotations/instances_'
 
 train_pipeline = [
@@ -18,7 +18,7 @@
         ),
         globals_=dict(data_root=oake_root + 'globals'),
         blocks=dict(data_root=oake_root + 'blocks'),
-        objects=dict(data_root=oake_root + 'objects'),
+        objects=dict(data_root=oake_root + 'objects/coco/output'),
     ),
     dict(
         type='RandomResize', scale=[(1330, 640), (1333, 800)], keep_ratio=True

diff --git a/configs/dp/datasets/ov_lvis.py b/configs/dp/datasets/ov_lvis.py
@@ -5,7 +5,7 @@
 categories = 'lvis'
 dataset_type = 'LVISV1Dataset'
 data_root = 'data/lvis_v1/'
-oake_root = data_root + 'oake/'
+oake_root = 'work_dirs/oake/'
 
 train_pipeline = [
     dict(type='LoadImageFromFile', backend_args=None),
@@ -18,7 +18,7 @@
         ),
         globals_=dict(data_root=oake_root + 'globals'),
         blocks=dict(data_root=oake_root + 'blocks'),
-        objects=dict(data_root=oake_root + 'objects'),
+        objects=dict(data_root=oake_root + 'objects/lvis/output'),
     ),
     dict(
         type='RandomResize', scale=[(1330, 640), (1333, 800)], keep_ratio=True

diff --git a/configs/dp/models/block.py b/configs/dp/models/block.py
@@ -12,27 +12,33 @@
         ),
     ),
     distiller=dict(
-        student_hooks=dict(
+        student_hook_pipeline=dict(
             blocks=dict(
-                inputs=tuple(),
-                action=dict(
-                    type='StandardHook',
+                type='SingleOperator',
+                args=tuple(),
+                atom=dict(
+                    type=(
+                        'TaskRegistry.KDRegistry.KDDistillerRegistry.'
+                        'KDHookRegistry.Hook'
+                    ),
                     path='.roi_head._block_head.fc_cls._linear',
                 ),
             ),
         ),
-        losses=dict(
+        loss_pipeline=dict(
             loss_clip_blocks=dict(
-                inputs=('blocks', 'clip_blocks'),
-                action=dict(
-                    type='L1Loss',
+                type='SingleOperator',
+                args=('blocks', 'clip_blocks'),
+                atom=dict(
+                    type='ModelRegistry.LossRegistry.L1Loss',
                     weight=dict(type='WarmupScheduler', gain=128, end=200),
                 ),
             ),
             loss_clip_block_relations=dict(
-                inputs=('blocks', 'clip_blocks'),
-                action=dict(
-                    type='RKDLoss',
+                type='SingleOperator',
+                args=('blocks', 'clip_blocks'),
+                atom=dict(
+                    type='ModelRegistry.LossRegistry.RKDLoss',
                     weight=dict(type='WarmupScheduler', gain=8, end=200),
                 ),
             ),