From 1ee02311ff969d455a76b751cb95ad847bb49961 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:14:01 -0400 Subject: [PATCH 001/119] Create neuroconv_dockerfile --- dockerfiles/neuroconv_dockerfile | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 dockerfiles/neuroconv_dockerfile diff --git a/dockerfiles/neuroconv_dockerfile b/dockerfiles/neuroconv_dockerfile new file mode 100644 index 000000000..2cfe20ba1 --- /dev/null +++ b/dockerfiles/neuroconv_dockerfile @@ -0,0 +1,3 @@ +FROM python:3.10 +LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv +RUN pip install neuroconv[full] From ee8873b0cad2ccc00eb75228ef877914d06dc09e Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:38:38 -0400 Subject: [PATCH 002/119] Create docker_images.rst --- docs/developer_guide/docker_images.rst | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 docs/developer_guide/docker_images.rst diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst new file mode 100644 index 000000000..d6c2ab55d --- /dev/null +++ b/docs/developer_guide/docker_images.rst @@ -0,0 +1,30 @@ +Manually Build Docker Images +---------------------------- + +.. note: It is highly recommended to build the docker image on the same basic system or architecture type that you intend to run it on, *i.e.*, AWS Linux AMI 64-bit (x86), as it may experience difficulties running on other radically different systems (like an M1 Mac). + +.. note: The NeuroConv docker container comes prepackaged with all required installations, *i.e.*, equivalent to `pip install neuroconv[full]`. As such it is fairly heavy, so be sure that whatever system (or specifically CI environment) you build with has sufficient disk space. + +.. code: + + docker build -f neuroconv_dockerfile -t neuroconv . + + + +Publish Container to GitHub +--------------------------- + +The `LABEL` is the important item here. In each docker file we wish to publish on the GitHub Container Registry (GHCR), we will label them such as + +.. code: + + LABEL org.opencontainers.image.source=https://github.com/OWNER/REPO + +After building the image, publish the container with + +.. code: + + docker tag IMAGE_NAME ghcr.io/catalystneuro/neuroconv:TAG + export CR_PAT="" + echo $CR_PAT | docker login ghcr.io -u --password-stdin + docker push ghcr.io/catalystneuro/neuroconv:TAG From a0871e121624d9c0dba004d1ca0aff85abfe5400 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:39:18 -0400 Subject: [PATCH 003/119] Update developer_guide.rst --- docs/developer_guide.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/developer_guide.rst b/docs/developer_guide.rst index 409f4d765..04fc6b035 100644 --- a/docs/developer_guide.rst +++ b/docs/developer_guide.rst @@ -15,5 +15,6 @@ The most common contribution is for a user/developer to help us integrate a new Testing Suite Coding Style Building the Documentation + Building the Docker Image Or feel free to raise a `bug report `_ or `feature request `_ for our maintainers to prioritize! From 9c148a09a039712a933137b4854e325c2907b80e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 15:40:38 +0000 Subject: [PATCH 004/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/developer_guide/docker_images.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index d6c2ab55d..8cffd1b4b 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -19,7 +19,7 @@ The `LABEL` is the important item here. In each docker file we wish to publish o .. code: LABEL org.opencontainers.image.source=https://github.com/OWNER/REPO - + After building the image, publish the container with .. code: From 91bbd3698bf214bdaa1f25ba0a0ae09bb9346968 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 30 Mar 2023 13:20:46 -0400 Subject: [PATCH 005/119] add use instructions --- docs/developer_guide/docker_images.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index 8cffd1b4b..23a3f9d89 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -28,3 +28,20 @@ After building the image, publish the container with export CR_PAT="" echo $CR_PAT | docker login ghcr.io -u --password-stdin docker push ghcr.io/catalystneuro/neuroconv:TAG + + + +Running Docker Container (Linux) +-------------------------------- + +You can either perform a manual build locally following the instructions above, or pull the container from the GitHub Container Registry (GHCR) with + +.. code: + + docker pull ghcr.io/catalystneuro/neuroconv:latest + +and can then run the entrypoint (equivalent to the usual CLI usage) on a YAML specification file (named `your_specification_file.yml`) with + +.. code: + + docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /desired/alias/of/drive/your_specification_file.yml From 66a0affdd6dfbf0f63ac838f55a3632d91caa111 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 30 Mar 2023 17:20:56 +0000 Subject: [PATCH 006/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/developer_guide/docker_images.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index 23a3f9d89..6ed712805 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -39,7 +39,7 @@ You can either perform a manual build locally following the instructions above, .. code: docker pull ghcr.io/catalystneuro/neuroconv:latest - + and can then run the entrypoint (equivalent to the usual CLI usage) on a YAML specification file (named `your_specification_file.yml`) with .. code: From 7de44dcd4a8e53de4b7deb564053949036b3404c Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 30 Mar 2023 13:21:18 -0400 Subject: [PATCH 007/119] Update neuroconv_dockerfile --- dockerfiles/neuroconv_dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfiles/neuroconv_dockerfile b/dockerfiles/neuroconv_dockerfile index 2cfe20ba1..5555b9fbc 100644 --- a/dockerfiles/neuroconv_dockerfile +++ b/dockerfiles/neuroconv_dockerfile @@ -1,3 +1,4 @@ FROM python:3.10 LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN pip install neuroconv[full] +CMD ["python3 -m"] From dee4842b9605e2f34edc64b8625855046a658487 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 30 Mar 2023 13:22:13 -0400 Subject: [PATCH 008/119] Rename neuroconv_dockerfile to neuroconv_latest_release_dockerfile --- .../{neuroconv_dockerfile => neuroconv_latest_release_dockerfile} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dockerfiles/{neuroconv_dockerfile => neuroconv_latest_release_dockerfile} (100%) diff --git a/dockerfiles/neuroconv_dockerfile b/dockerfiles/neuroconv_latest_release_dockerfile similarity index 100% rename from dockerfiles/neuroconv_dockerfile rename to dockerfiles/neuroconv_latest_release_dockerfile From 17031c9276c777b69b821ab798b15639625d08c7 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 30 Mar 2023 13:23:36 -0400 Subject: [PATCH 009/119] added dev build --- dockerfiles/neuroconv_developer_build_dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 dockerfiles/neuroconv_developer_build_dockerfile diff --git a/dockerfiles/neuroconv_developer_build_dockerfile b/dockerfiles/neuroconv_developer_build_dockerfile new file mode 100644 index 000000000..5a5a4bea8 --- /dev/null +++ b/dockerfiles/neuroconv_developer_build_dockerfile @@ -0,0 +1,6 @@ +FROM python:3.10 +LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv +ADD /neuroconv /neuroconv # Current working directory must be just outside local clone of neuroconv +RUN cd neuroconv \ + && pip install -e .[full] +CMD ["python3 -m"] From de6b5f6a43d60d87b6e9edac749a5c7a9df7fbc4 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 30 Mar 2023 13:58:19 -0400 Subject: [PATCH 010/119] Update docker_images.rst --- docs/developer_guide/docker_images.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index 6ed712805..a8f8ac695 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -5,9 +5,23 @@ Manually Build Docker Images .. note: The NeuroConv docker container comes prepackaged with all required installations, *i.e.*, equivalent to `pip install neuroconv[full]`. As such it is fairly heavy, so be sure that whatever system (or specifically CI environment) you build with has sufficient disk space. + +Latest Release +~~~~~~~~~~~~~~ + +.. code: + + docker build -f neuroconv_latest_release_dockerfile -t neuroconv_latest_release . + + +Dev Branch +~~~~~~~~~~ + +Checkout to a specific branch on a local clone, then + .. code: - docker build -f neuroconv_dockerfile -t neuroconv . + docker build -f neuroconv_developer_build_dockerfile -t neuroconv_dev . From 37862f2abe0674b50a78768365f46e5e18bce1d6 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 30 Mar 2023 14:04:22 -0400 Subject: [PATCH 011/119] Update neuroconv_developer_build_dockerfile --- dockerfiles/neuroconv_developer_build_dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_developer_build_dockerfile b/dockerfiles/neuroconv_developer_build_dockerfile index 5a5a4bea8..d25bd5dbd 100644 --- a/dockerfiles/neuroconv_developer_build_dockerfile +++ b/dockerfiles/neuroconv_developer_build_dockerfile @@ -1,6 +1,6 @@ FROM python:3.10 LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -ADD /neuroconv /neuroconv # Current working directory must be just outside local clone of neuroconv +ADD /neuroconv /neuroconv RUN cd neuroconv \ && pip install -e .[full] CMD ["python3 -m"] From 45fe2669878d1b65491d686cb87c31e45aa235e6 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 31 Mar 2023 11:06:58 -0400 Subject: [PATCH 012/119] Update docker_images.rst --- docs/developer_guide/docker_images.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index a8f8ac695..3b715751b 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -28,20 +28,23 @@ Checkout to a specific branch on a local clone, then Publish Container to GitHub --------------------------- -The `LABEL` is the important item here. In each docker file we wish to publish on the GitHub Container Registry (GHCR), we will label them such as +The `LABEL` is the important item here; it determines the host repository on the GitHub Container Registry (GHCR). In each docker file we wish to publish on the GHCR, we will add this label right after the `FROM` clause .. code: - LABEL org.opencontainers.image.source=https://github.com/OWNER/REPO + FROM PARENT_IMAGE:TAG + LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -After building the image, publish the container with +After building the image itself, we can publish the container with .. code: - docker tag IMAGE_NAME ghcr.io/catalystneuro/neuroconv:TAG + docker tag IMAGE_NAME ghcr.io/catalystneuro/IMAGE_NAME:TAG export CR_PAT="" echo $CR_PAT | docker login ghcr.io -u --password-stdin - docker push ghcr.io/catalystneuro/neuroconv:TAG + docker push ghcr.io/catalystneuro/IMAGE_NAME:TAG + +.. note: Though it may appear confusing, the use of the `IMAGE_NAME` in these steps determines only the _name_ of the package as available from the 'packages' screen of the host repository; the `LABEL` itself ensured the upload and linkage to the NeuroConv GHCR. From c8d6eddd1c4808d1ac0e28216b1164d6168d33bc Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sun, 2 Apr 2023 16:25:56 -0400 Subject: [PATCH 013/119] Update docker_images.rst --- docs/developer_guide/docker_images.rst | 34 ++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index 3b715751b..f64fb901b 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -48,8 +48,8 @@ After building the image itself, we can publish the container with -Running Docker Container (Linux) --------------------------------- +Running Docker Container on local YAML (Linux) +---------------------------------------------- You can either perform a manual build locally following the instructions above, or pull the container from the GitHub Container Registry (GHCR) with @@ -62,3 +62,33 @@ and can then run the entrypoint (equivalent to the usual CLI usage) on a YAML sp .. code: docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /desired/alias/of/drive/your_specification_file.yml + + + +Running Docker Container with a YAML_STREAM (Linux) +--------------------------------------------------- + +An alternative approach that simplifies usage on AWS Batch is to specify the YAML contents as an environment variable. The file is constructed in the first step of the container launch. The only potential downside with this usage is maximum file size (~13,000 characters; current example files do not come remotely close to this). + +Otherwise, the YAML file transfer will have to managed separately, likely as a part of the data transfer or an entirely separate transfer step. + +To use manually outside of AWS Batch, + +.. code: + + export YAML_STREAM="" + docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:latest + +To use automatically via a Python helper function (coming in a separate PR) + +.. code: + import os + + with open(file="my_yaml_file.yml") as file: + yaml_stream = "".join(file.readlines()).replace("\"", "'") + + os.environ["YAML_STREAM"] = yaml_stream + +.. note: + + When using YAML files through the docker containers, always be sure that the NWB file paths are absolute paths stemming from the mounted volume; otherwise, the NWB file will indeed be written inside the container but will not be accessible outside of it. From c3940b759a3258132a24533a81e2269d27281883 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sun, 2 Apr 2023 16:26:37 -0400 Subject: [PATCH 014/119] Update docker_images.rst --- docs/developer_guide/docker_images.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index f64fb901b..459797d72 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -77,7 +77,7 @@ To use manually outside of AWS Batch, .. code: export YAML_STREAM="" - docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:latest + docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:dev_auto_yaml To use automatically via a Python helper function (coming in a separate PR) From c3984d357665afd9c917d6f8677a730d5be20efc Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sun, 2 Apr 2023 16:31:40 -0400 Subject: [PATCH 015/119] Create neuroconv_developer_build_auto_config --- dockerfiles/neuroconv_developer_build_auto_config | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 dockerfiles/neuroconv_developer_build_auto_config diff --git a/dockerfiles/neuroconv_developer_build_auto_config b/dockerfiles/neuroconv_developer_build_auto_config new file mode 100644 index 000000000..62722aa17 --- /dev/null +++ b/dockerfiles/neuroconv_developer_build_auto_config @@ -0,0 +1,8 @@ +FROM python:3.10 +LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv +ADD /neuroconv /neuroconv +RUN cd neuroconv \ + && pip install -e .[full] +CMD echo "$YAML_STREAM" > run.yml \ + && cat run.yml \ + && neuroconv run.yml From dd0f2a2577c2a77d9e3401957f2275580c22ad15 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sun, 2 Apr 2023 17:20:17 -0400 Subject: [PATCH 016/119] Create rclone_auto_config_dockerfile --- dockerfiles/rclone_auto_config_dockerfile | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 dockerfiles/rclone_auto_config_dockerfile diff --git a/dockerfiles/rclone_auto_config_dockerfile b/dockerfiles/rclone_auto_config_dockerfile new file mode 100644 index 000000000..a97a166ae --- /dev/null +++ b/dockerfiles/rclone_auto_config_dockerfile @@ -0,0 +1,5 @@ +FROM rclone/rclone:latest +CMD printf "[$DRIVE_NAME]\ntype = drive\nscope = drive\ntoken = {\"access_token\":\"$TOKEN\",\"token_type\":\"Bearer\",\"refresh_token\":\"$REFRESH_TOKEN\",\"expiry\":\"$EXPIRY\"}" > /mnt/data/rclone.conf \ + && cat /mnt/data/rclone.conf \ + && rclone $RCLONE_COMMAND +ENTRYPOINT [""] From c08ed01fcab656574548df1eb0911fa73c04d83b Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Jun 2023 11:40:46 -0400 Subject: [PATCH 017/119] add workflow for docker release --- .github/workflows/auto-release-docker.yml | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/auto-release-docker.yml diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/auto-release-docker.yml new file mode 100644 index 000000000..de2a45913 --- /dev/null +++ b/.github/workflows/auto-release-docker.yml @@ -0,0 +1,32 @@ +name: Upload Docker Image to GHCR + +on: + release: + types: [published] + +jobs: + release-image: + name: Release NeuroConv Image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.MY_GITHUB_USERNAME }} + password: ${{ secrets.MY_GITHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v3 + with: + push: true # Push is a shorthand for --output=type=registry + tags: + - ghcr.io/CatalystNeuro/NeuroConv:latest + - ghcr.io/CatalystNeuro/NeuroConv:0.3.1 # TODO, figure out how to auto-set this + context: docker + file: dockerfiles/neuroconv_developer_build_dockerfile From e7b5557b9cc71a7dba5eef820f1c45b50a9846a6 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Jun 2023 11:47:55 -0400 Subject: [PATCH 018/119] add docker publish for dev branch --- .../auto-publish-docker-dev-branch.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/auto-publish-docker-dev-branch.yml diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/auto-publish-docker-dev-branch.yml new file mode 100644 index 000000000..9df78959e --- /dev/null +++ b/.github/workflows/auto-publish-docker-dev-branch.yml @@ -0,0 +1,33 @@ +name: Upload Docker Image of Latest Developer Branch to GHCR + +on: + schedule: + - cron: "0 16 * * *" # Daily at noon EST + workflow_dispatch: + +jobs: + release-image: + name: Release NeuroConv Image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.MY_GITHUB_USERNAME }} + password: ${{ secrets.MY_GITHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v3 + with: + push: true # Push is a shorthand for --output=type=registry + tags: | + ghcr.io/CatalystNeuro/NeuroConv:latest + ghcr.io/CatalystNeuro/NeuroConv:0.3.1 # TODO, figure out how to auto-set this + context: docker + file: dockerfiles/neuroconv_developer_build_dockerfile From 23285edcd9bb83fa949bac3c1cf851ceb87143aa Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Jun 2023 11:50:05 -0400 Subject: [PATCH 019/119] initial trigger to show up on interface --- .github/workflows/auto-publish-docker-dev-branch.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/auto-publish-docker-dev-branch.yml index 9df78959e..9b09975b2 100644 --- a/.github/workflows/auto-publish-docker-dev-branch.yml +++ b/.github/workflows/auto-publish-docker-dev-branch.yml @@ -4,10 +4,11 @@ on: schedule: - cron: "0 16 * * *" # Daily at noon EST workflow_dispatch: + push: jobs: release-image: - name: Release NeuroConv Image + name: Release NeuroConv Image of Current Dev Branch runs-on: ubuntu-latest steps: - name: Checkout From 9a02034826f51b130b6215c38820faa3bd2892ce Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Jun 2023 11:57:21 -0400 Subject: [PATCH 020/119] trying to fix dev image publish --- .github/workflows/auto-publish-docker-dev-branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/auto-publish-docker-dev-branch.yml index 9b09975b2..450eca912 100644 --- a/.github/workflows/auto-publish-docker-dev-branch.yml +++ b/.github/workflows/auto-publish-docker-dev-branch.yml @@ -21,7 +21,7 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.MY_GITHUB_USERNAME }} + username: ${{ github.actor }} password: ${{ secrets.MY_GITHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v3 From 73ffcbabcc897df0987a5d96063325a8ae3b277a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Jun 2023 13:22:06 -0400 Subject: [PATCH 021/119] update username and ghcr name --- .github/workflows/auto-publish-docker-dev-branch.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/auto-publish-docker-dev-branch.yml index 450eca912..22feb8b4c 100644 --- a/.github/workflows/auto-publish-docker-dev-branch.yml +++ b/.github/workflows/auto-publish-docker-dev-branch.yml @@ -21,14 +21,12 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ secrets.MY_GITHUB_USERNAME }} password: ${{ secrets.MY_GITHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v3 with: push: true # Push is a shorthand for --output=type=registry - tags: | - ghcr.io/CatalystNeuro/NeuroConv:latest - ghcr.io/CatalystNeuro/NeuroConv:0.3.1 # TODO, figure out how to auto-set this + tags: ghcr.io/catalystneuro/neuroconv:dev context: docker file: dockerfiles/neuroconv_developer_build_dockerfile From e0c60463bb3bde589dd0b482b8c8ab6faf1faea7 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Jun 2023 13:24:26 -0400 Subject: [PATCH 022/119] try fixing action path --- .github/workflows/auto-publish-docker-dev-branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/auto-publish-docker-dev-branch.yml index 22feb8b4c..628add6b0 100644 --- a/.github/workflows/auto-publish-docker-dev-branch.yml +++ b/.github/workflows/auto-publish-docker-dev-branch.yml @@ -28,5 +28,5 @@ jobs: with: push: true # Push is a shorthand for --output=type=registry tags: ghcr.io/catalystneuro/neuroconv:dev - context: docker + context: dockerfiles file: dockerfiles/neuroconv_developer_build_dockerfile From 5419ce12e8fdf6adb9e30ea1c9cab1348e89c04f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Jun 2023 13:28:15 -0400 Subject: [PATCH 023/119] Update neuroconv_developer_build_dockerfile --- dockerfiles/neuroconv_developer_build_dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dockerfiles/neuroconv_developer_build_dockerfile b/dockerfiles/neuroconv_developer_build_dockerfile index d25bd5dbd..af4cc9126 100644 --- a/dockerfiles/neuroconv_developer_build_dockerfile +++ b/dockerfiles/neuroconv_developer_build_dockerfile @@ -1,6 +1,4 @@ FROM python:3.10 LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -ADD /neuroconv /neuroconv -RUN cd neuroconv \ - && pip install -e .[full] +RUN pip install git+https://github.com/catalystneuro/neuroconv.git@main CMD ["python3 -m"] From 05944f5b8c1681a9039f52a014730e68b304e071 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:46:48 -0400 Subject: [PATCH 024/119] retrigger with new credentials --- .github/workflows/auto-publish-docker-dev-branch.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/auto-publish-docker-dev-branch.yml index 628add6b0..47dd8380b 100644 --- a/.github/workflows/auto-publish-docker-dev-branch.yml +++ b/.github/workflows/auto-publish-docker-dev-branch.yml @@ -4,7 +4,10 @@ on: schedule: - cron: "0 16 * * *" # Daily at noon EST workflow_dispatch: - push: + +concurrency: # Cancel previous workflows on the same pull request + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: release-image: From a09904f7fdfe992f326305df7c1073d0fea68125 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:48:06 -0400 Subject: [PATCH 025/119] need to keep push until on main --- .github/workflows/auto-publish-docker-dev-branch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/auto-publish-docker-dev-branch.yml index 47dd8380b..50d6a9511 100644 --- a/.github/workflows/auto-publish-docker-dev-branch.yml +++ b/.github/workflows/auto-publish-docker-dev-branch.yml @@ -4,6 +4,7 @@ on: schedule: - cron: "0 16 * * *" # Daily at noon EST workflow_dispatch: + push: concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} From e8ed3ea0a3c130b359a64f4c495eb70d2c132ad2 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 22 Feb 2024 12:52:17 -0500 Subject: [PATCH 026/119] update credentials --- .github/workflows/auto-publish-docker-dev-branch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/auto-publish-docker-dev-branch.yml index 50d6a9511..fb4edfed2 100644 --- a/.github/workflows/auto-publish-docker-dev-branch.yml +++ b/.github/workflows/auto-publish-docker-dev-branch.yml @@ -25,8 +25,8 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ secrets.MY_GITHUB_USERNAME }} - password: ${{ secrets.MY_GITHUB_TOKEN }} + username: ${{ secrets.DOCKER_UPLOADER_USERNAME }} + password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} - name: Build and push uses: docker/build-push-action@v3 with: From 833062131e7960e0e97c8df59ff78d24e469ec12 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 22 Feb 2024 13:31:31 -0500 Subject: [PATCH 027/119] try to add tag --- .github/workflows/auto-release-docker.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/auto-release-docker.yml index de2a45913..9639a29db 100644 --- a/.github/workflows/auto-release-docker.yml +++ b/.github/workflows/auto-release-docker.yml @@ -1,8 +1,9 @@ name: Upload Docker Image to GHCR on: - release: - types: [published] + workflow_run: + workflows: [auto-publish] + types: [completed] jobs: release-image: @@ -10,7 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 + - name: Grab version from release tag + run: | + git fetch --prune --unshallow --tags + tags="$(git tag --list)" + latest_tag=${tags: -6 : 6} + echo "latest_tag=$latest_tag" >> "$GITHUB_ENV" - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -19,14 +26,12 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.MY_GITHUB_USERNAME }} - password: ${{ secrets.MY_GITHUB_TOKEN }} + username: ${{ secrets.DOCKER_UPLOADER_USERNAME }} + password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} - name: Build and push uses: docker/build-push-action@v3 with: push: true # Push is a shorthand for --output=type=registry - tags: - - ghcr.io/CatalystNeuro/NeuroConv:latest - - ghcr.io/CatalystNeuro/NeuroConv:0.3.1 # TODO, figure out how to auto-set this + tags: ghcr.io/CatalystNeuro/NeuroConv:$latest_tag context: docker file: dockerfiles/neuroconv_developer_build_dockerfile From 73a7e16367476403794e3b075c019f78be849bbd Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 22 Feb 2024 13:32:33 -0500 Subject: [PATCH 028/119] Update dockerfiles/neuroconv_developer_build_auto_config --- dockerfiles/neuroconv_developer_build_auto_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_developer_build_auto_config b/dockerfiles/neuroconv_developer_build_auto_config index 62722aa17..01b46d92b 100644 --- a/dockerfiles/neuroconv_developer_build_auto_config +++ b/dockerfiles/neuroconv_developer_build_auto_config @@ -1,4 +1,4 @@ -FROM python:3.10 +FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv ADD /neuroconv /neuroconv RUN cd neuroconv \ From 2479d053c0c52c12d2726607d4d9d23da32ec269 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 22 Feb 2024 13:33:06 -0500 Subject: [PATCH 029/119] Apply suggestions from code review --- dockerfiles/neuroconv_developer_build_dockerfile | 2 +- dockerfiles/neuroconv_latest_release_dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/neuroconv_developer_build_dockerfile b/dockerfiles/neuroconv_developer_build_dockerfile index af4cc9126..3ed66d8dc 100644 --- a/dockerfiles/neuroconv_developer_build_dockerfile +++ b/dockerfiles/neuroconv_developer_build_dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10 +FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN pip install git+https://github.com/catalystneuro/neuroconv.git@main CMD ["python3 -m"] diff --git a/dockerfiles/neuroconv_latest_release_dockerfile b/dockerfiles/neuroconv_latest_release_dockerfile index 5555b9fbc..88f72f73c 100644 --- a/dockerfiles/neuroconv_latest_release_dockerfile +++ b/dockerfiles/neuroconv_latest_release_dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10 +FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN pip install neuroconv[full] CMD ["python3 -m"] From 675abcfa42542051fe6c314b4560ad24f9f39334 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 22 Feb 2024 13:39:08 -0500 Subject: [PATCH 030/119] Update .github/workflows/auto-publish-docker-dev-branch.yml --- .github/workflows/auto-publish-docker-dev-branch.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/auto-publish-docker-dev-branch.yml index fb4edfed2..096fea4b0 100644 --- a/.github/workflows/auto-publish-docker-dev-branch.yml +++ b/.github/workflows/auto-publish-docker-dev-branch.yml @@ -4,7 +4,6 @@ on: schedule: - cron: "0 16 * * *" # Daily at noon EST workflow_dispatch: - push: concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} From 323e17f9fc81f1a34b423282277853c4a9dd4759 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 18 Mar 2024 14:50:21 -0400 Subject: [PATCH 031/119] add docker tests --- ..._yaml_conversion_specification_cli_test.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 tests/test_on_data/docker_yaml_conversion_specification_cli_test.py diff --git a/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py b/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py new file mode 100644 index 000000000..d2f12e724 --- /dev/null +++ b/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py @@ -0,0 +1,124 @@ +""" +This file is hidden from normal pytest globbing by appending the _test to the end instead of the beginning. + +Instead, the tests must be invoked directly from the file. This is designed mostly for use in the GitHub Actions. +""" +import os +import unittest +from datetime import datetime +from pathlib import Path + +from hdmf.testing import TestCase +from pynwb import NWBHDF5IO + +from neuroconv.tools import deploy_process + +from .setup_paths import ECEPHY_DATA_PATH as DATA_PATH +from .setup_paths import OUTPUT_PATH + + +class TestLatestDockerYAMLConversionSpecification(TestCase): + test_folder = OUTPUT_PATH + tag = os.getenv("NEUROCONV_DOCKER_TESTS_TAG", "latest") + source_volume = os.getenv("NEUROCONV_DOCKER_TESTS_SOURCE_VOLUME", "/home/runner/work/neuroconv") + + def test_run_conversion_from_yaml_cli(self): + path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" + yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" + deploy_process( + command=( + "docker run -it " + f"--volume {source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{tag}" + f"neuroconv {yaml_file_path} " + f"--data-folder-path {DATA_PATH} --output-folder-path {self.test_folder} --overwrite" + ) + ) + + nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") + assert nwbfile.subject.subject_id == "1" + assert "ElectricalSeriesAP" in nwbfile.acquisition + + nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") + assert nwbfile.subject.subject_id == "002" + + nwbfile_path = self.test_folder / "example_converter_spec_3.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Auto-generated by neuroconv" + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") + assert nwbfile.subject.subject_id == "Subject Name" + assert "spike_times" in nwbfile.units + + def test_run_conversion_from_yaml_default_nwbfile_name(self): + self.test_folder = self.test_folder / "test_organize" + self.test_folder.mkdir(exist_ok=True) + path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" + yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification_missing_nwbfile_names.yml" + deploy_process( + command=( + "docker run -it " + f"--volume {source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{tag}" + f"neuroconv {yaml_file_path} " + f"--data-folder-path {DATA_PATH} --output-folder-path {self.test_folder} --overwrite" + ) + ) + + nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201009T211909.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") + assert nwbfile.subject.subject_id == "Mouse 1" + assert "ElectricalSeriesAP" in nwbfile.acquisition + + nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00") + assert nwbfile.subject.subject_id == "Mouse 1" + assert "ElectricalSeriesAP" in nwbfile.acquisition + + nwbfile_path = self.test_folder / "example_defined_name.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") + assert nwbfile.subject.subject_id == "MyMouse002" + + nwbfile_path = self.test_folder / "sub-Subject-Name_ses-20201011T211909.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Auto-generated by neuroconv" + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") + assert nwbfile.subject.subject_id == "Subject Name" + assert "spike_times" in nwbfile.units From 5dd6ffa71630a8e055cce64715d671e1676bf1eb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 18:50:32 +0000 Subject: [PATCH 032/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../docker_yaml_conversion_specification_cli_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py b/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py index d2f12e724..5a73e3644 100644 --- a/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py +++ b/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py @@ -3,6 +3,7 @@ Instead, the tests must be invoked directly from the file. This is designed mostly for use in the GitHub Actions. """ + import os import unittest from datetime import datetime From 0d92791af7eadf2842785fc88fa1377fbd36adcb Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:07:09 -0400 Subject: [PATCH 033/119] add CI --- .github/workflows/docker_testing.yml | 89 ++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .github/workflows/docker_testing.yml diff --git a/.github/workflows/docker_testing.yml b/.github/workflows/docker_testing.yml new file mode 100644 index 000000000..b88ffe8f1 --- /dev/null +++ b/.github/workflows/docker_testing.yml @@ -0,0 +1,89 @@ +name: Docker CLI tests +on: + schedule: + - cron: "0 16 * * *" # Daily at noon EST + workflow_call: + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + S3_GIN_BUCKET: + required: true + CODECOV_TOKEN: + required: true + workflow_dispatch: + push: # on first commit + +jobs: + run: + name: ${{ matrix.os }} Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + os: [ubuntu-latest] #, macos-latest, windows-latest] # Seems docker might only be available for ubuntu on GitHub Actions + steps: + - uses: actions/checkout@v4 + - run: git fetch --prune --unshallow --tags + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Global Setup + run: python -m pip install -U pip # Official recommended way + + - name: Install pytest + run: pip install pytest + + - name: Pull docker image + run: docker pull ghcr.io/catalystneuro/neuroconv:latest + + - name: Get ephy_testing_data current head hash + id: ephys + run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" + - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + uses: actions/cache@v4 + id: cache-ephys-datasets + with: + path: ./ephy_testing_data + key: ephys-datasets-2023-06-26-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + - name: Get ophys_testing_data current head hash + id: ophys + run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + uses: actions/cache@v4 + id: cache-ophys-datasets + with: + path: ./ophys_testing_data + key: ophys-datasets-2022-08-18-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + - name: Get behavior_testing_data current head hash + id: behavior + run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)" + - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} + uses: actions/cache@v4 + id: cache-behavior-datasets + with: + path: ./behavior_testing_data + key: behavior-datasets-2023-07-26-${{ matrix.os }}-${{ steps.behavior.outputs.HASH_behavior_DATASET }} + + - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' || steps.cache-ophys-datasets.outputs.cache-hit != 'true' || steps.cache-behavior-datasets.outputs.cache-hit != 'true' + name: Install and configure AWS CLI + run: | + pip install awscli + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' + name: Download ephys dataset from S3 + run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ephy_testing_data ./ephy_testing_data + - if: steps.cache-ophys-datasets.outputs.cache-hit != 'true' + name: Download ophys dataset from S3 + run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ophys_testing_data ./ophys_testing_data + - if: steps.cache-behavior-datasets.outputs.cache-hit != 'true' + name: Download behavior dataset from S3 + run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/behavior_testing_data ./behavior_testing_data + + - name: Run docker tests + run: pytest tests/test_on_data/docker_yaml_conversion_specification_cli_test.py -vv -rsx From 99e673bedde922ec6d8d3fbe97a6c3683d6c7827 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:12:33 -0400 Subject: [PATCH 034/119] update installation for pytest checks --- .github/workflows/docker_testing.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker_testing.yml b/.github/workflows/docker_testing.yml index b88ffe8f1..8ea59d235 100644 --- a/.github/workflows/docker_testing.yml +++ b/.github/workflows/docker_testing.yml @@ -35,8 +35,10 @@ jobs: - name: Global Setup run: python -m pip install -U pip # Official recommended way - - name: Install pytest - run: pip install pytest + - name: Install pytest and neuroconv minimal + run: | + pip install pytest + pip install . - name: Pull docker image run: docker pull ghcr.io/catalystneuro/neuroconv:latest From 64f4feed166dd11e0099c70c4815d3ac98e886e2 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:18:42 -0400 Subject: [PATCH 035/119] debug attributes --- .../docker_yaml_conversion_specification_cli_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py b/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py index 5a73e3644..e0d16fa44 100644 --- a/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py +++ b/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py @@ -29,7 +29,7 @@ def test_run_conversion_from_yaml_cli(self): deploy_process( command=( "docker run -it " - f"--volume {source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{tag}" + f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag}" f"neuroconv {yaml_file_path} " f"--data-folder-path {DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ) @@ -75,7 +75,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): deploy_process( command=( "docker run -it " - f"--volume {source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{tag}" + f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag}" f"neuroconv {yaml_file_path} " f"--data-folder-path {DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ) From 729cc81a3965bed981c293a5eb15b52d212be780 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 12:15:10 -0400 Subject: [PATCH 036/119] fix docs rendering --- docs/developer_guide/docker_images.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index 459797d72..eae8efb69 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -1,15 +1,15 @@ Manually Build Docker Images ---------------------------- -.. note: It is highly recommended to build the docker image on the same basic system or architecture type that you intend to run it on, *i.e.*, AWS Linux AMI 64-bit (x86), as it may experience difficulties running on other radically different systems (like an M1 Mac). +.. note:: It is highly recommended to build the docker image on the same basic system or architecture type that you intend to run it on, *i.e.*, AWS Linux AMI 64-bit (x86), as it may experience difficulties running on other radically different systems (like an M1 Mac). -.. note: The NeuroConv docker container comes prepackaged with all required installations, *i.e.*, equivalent to `pip install neuroconv[full]`. As such it is fairly heavy, so be sure that whatever system (or specifically CI environment) you build with has sufficient disk space. +.. note:: The NeuroConv docker container comes prepackaged with all required installations, *i.e.*, equivalent to `pip install neuroconv[full]`. As such it is fairly heavy, so be sure that whatever system (or specifically CI environment) you build with has sufficient disk space. Latest Release ~~~~~~~~~~~~~~ -.. code: +.. code:: docker build -f neuroconv_latest_release_dockerfile -t neuroconv_latest_release . @@ -19,7 +19,7 @@ Dev Branch Checkout to a specific branch on a local clone, then -.. code: +.. code:: docker build -f neuroconv_developer_build_dockerfile -t neuroconv_dev . @@ -30,21 +30,21 @@ Publish Container to GitHub The `LABEL` is the important item here; it determines the host repository on the GitHub Container Registry (GHCR). In each docker file we wish to publish on the GHCR, we will add this label right after the `FROM` clause -.. code: +.. code:: FROM PARENT_IMAGE:TAG LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv After building the image itself, we can publish the container with -.. code: +.. code:: docker tag IMAGE_NAME ghcr.io/catalystneuro/IMAGE_NAME:TAG export CR_PAT="" echo $CR_PAT | docker login ghcr.io -u --password-stdin docker push ghcr.io/catalystneuro/IMAGE_NAME:TAG -.. note: Though it may appear confusing, the use of the `IMAGE_NAME` in these steps determines only the _name_ of the package as available from the 'packages' screen of the host repository; the `LABEL` itself ensured the upload and linkage to the NeuroConv GHCR. +.. note:: Though it may appear confusing, the use of the `IMAGE_NAME` in these steps determines only the _name_ of the package as available from the 'packages' screen of the host repository; the `LABEL` itself ensured the upload and linkage to the NeuroConv GHCR. @@ -53,13 +53,13 @@ Running Docker Container on local YAML (Linux) You can either perform a manual build locally following the instructions above, or pull the container from the GitHub Container Registry (GHCR) with -.. code: +.. code:: docker pull ghcr.io/catalystneuro/neuroconv:latest and can then run the entrypoint (equivalent to the usual CLI usage) on a YAML specification file (named `your_specification_file.yml`) with -.. code: +.. code:: docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /desired/alias/of/drive/your_specification_file.yml @@ -74,14 +74,15 @@ Otherwise, the YAML file transfer will have to managed separately, likely as a p To use manually outside of AWS Batch, -.. code: +.. code:: export YAML_STREAM="" docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:dev_auto_yaml To use automatically via a Python helper function (coming in a separate PR) -.. code: +.. code:: + import os with open(file="my_yaml_file.yml") as file: @@ -89,6 +90,6 @@ To use automatically via a Python helper function (coming in a separate PR) os.environ["YAML_STREAM"] = yaml_stream -.. note: +.. note:: When using YAML files through the docker containers, always be sure that the NWB file paths are absolute paths stemming from the mounted volume; otherwise, the NWB file will indeed be written inside the container but will not be accessible outside of it. From da3bd588a13518db74c62ff27091e76bb05ea268 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 12:20:20 -0400 Subject: [PATCH 037/119] improve docs --- docs/developer_guide/docker_images.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index eae8efb69..1b4b746b2 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -1,14 +1,16 @@ Manually Build Docker Images ---------------------------- -.. note:: It is highly recommended to build the docker image on the same basic system or architecture type that you intend to run it on, *i.e.*, AWS Linux AMI 64-bit (x86), as it may experience difficulties running on other radically different systems (like an M1 Mac). +.. note:: It is recommended to build the docker image on the same system architecture that you intend to run it on, *i.e.*, AWS Linux AMI 64-bit (x86), as it may experience difficulties running on other significantly different systems (like an M1 Mac). -.. note:: The NeuroConv docker container comes prepackaged with all required installations, *i.e.*, equivalent to `pip install neuroconv[full]`. As such it is fairly heavy, so be sure that whatever system (or specifically CI environment) you build with has sufficient disk space. +.. note:: The NeuroConv docker container comes prepackaged with all required installations, *i.e.*, equivalent to `pip install neuroconv[full]`. As such it is relatively heavy, so be sure that whatever environment you intend to use it in (such as in continuous integration) has sufficient disk space. Latest Release ~~~~~~~~~~~~~~ +To manually build the most recent release, navigate to the 'neuroconv/dockerfiles' folder and run... + .. code:: docker build -f neuroconv_latest_release_dockerfile -t neuroconv_latest_release . From 37291157ece1b655b0fd791c073d6921ab5de93d Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:56:57 -0400 Subject: [PATCH 038/119] add full demo --- docs/developer_guide/docker_images.rst | 98 +++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index 1b4b746b2..fe9d19908 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -72,7 +72,7 @@ Running Docker Container with a YAML_STREAM (Linux) An alternative approach that simplifies usage on AWS Batch is to specify the YAML contents as an environment variable. The file is constructed in the first step of the container launch. The only potential downside with this usage is maximum file size (~13,000 characters; current example files do not come remotely close to this). -Otherwise, the YAML file transfer will have to managed separately, likely as a part of the data transfer or an entirely separate transfer step. +Otherwise, the YAML file transfer will have to be managed separately, likely as a part of the data transfer or an entirely separate transfer step. To use manually outside of AWS Batch, @@ -95,3 +95,99 @@ To use automatically via a Python helper function (coming in a separate PR) .. note:: When using YAML files through the docker containers, always be sure that the NWB file paths are absolute paths stemming from the mounted volume; otherwise, the NWB file will indeed be written inside the container but will not be accessible outside of it. + + +Demo +---- + +The following is an explicit demonstration of how to use the Docker-based CLI for NeuroConv. + +It relies on some of the GIN data from the main testing suite, see :ref:`example_data` for more details. + +1. Make a new folder for the demo conversion named ``demo_neuroconv_docker``. + +2. Make a subfolder in ``demo_neuroconv_docker`` called ``demo_output``. + +3. Make a new file in this folder named ``demo_neuroconv_docker_yaml.yml``. + +4. Open this YAML file in a text editor, then copy and paste the following section into that file. + +.. code:: + +metadata: + NWBFile: + lab: My Lab + institution: My Institution + +data_interfaces: + ap: SpikeGLXRecordingInterface + phy: PhySortingInterface + +experiments: + my_experiment: + metadata: + NWBFile: + session_description: My session. + + sessions: + - nwbfile_name: spikeglx_from_docker_yaml.nwb + source_data: + ap: + file_path: /demo_neuroconv_docker/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin + metadata: + NWBFile: + session_start_time: "2020-10-10T21:19:09+00:00" + Subject: + subject_id: "1" + sex: F + age: P35D + species: Mus musculus + - nwbfile_name: phy_from_docker_yaml.nwb + metadata: + NWBFile: + session_start_time: "2020-10-10T21:19:09+00:00" + Subject: + subject_id: "002" + sex: F + age: P35D + species: Mus musculus + source_data: + phy: + folder_path: /demo_neuroconv_docker/phy/phy_example_0/ + + +5. To make things easier for volume mounting, copy and paste the ``Noise4Sam_g0`` and ``phy_example_0`` folders into this Docker demo folder so that you have the following folder structure... + +.. code:: + + | demo_neuroconv_docker/ + |-- demo_output/ + |-- demo_neuroconv_docker_yaml.yml + |-- spikeglx/ + |---- Noise4Sam_g0/ + |------ Noise4Sam_g0_imec0/ + |-------- Noise4Sam_g0_t0.imec0.ap.bin + |-------- Noise4Sam_g0_t0.imec0.ap.meta + |-- phy/ + |---- phy_example_0/ + |------ ... # The various file contents from the example Phy folder + +6. Pull the latest NeuroConv docker image from GitHub... + +.. code:: + + docker pull ghcr.io/catalystneuro/neuroconv:latest + +7. Run the command line interface on the YAML file using the docker container (instead of a local installation of the Python package)... + +.. code:: + + docker run -t --volume /demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv test_docker_yaml.yml + +.. note:: Docker relies heavily on absolute system paths, but these can vary depending on your system. For Windows, this might be something like ``C:/Users/MyUser/Downloads/``, for MacOSX it might be ``/Users/username/``. + +For example, assuming a MacOSX user with username 'MyUser', and assuming the ``demo_neuroconv_docker`` folder was created in the home directory, this would look like + +.. code:: + + docker run -t --volume /Users/MyUser/demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/test_docker_yaml.yml From 055f226430b35dae5921c683b8458ab10bcc482a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:57:52 -0400 Subject: [PATCH 039/119] move to user guide --- docs/user_guide/docker_demo.rst | 94 +++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 docs/user_guide/docker_demo.rst diff --git a/docs/user_guide/docker_demo.rst b/docs/user_guide/docker_demo.rst new file mode 100644 index 000000000..646e7b452 --- /dev/null +++ b/docs/user_guide/docker_demo.rst @@ -0,0 +1,94 @@ +Docker Demo +----------- + +The following is an explicit demonstration of how to use the Docker-based CLI for NeuroConv. + +It relies on some of the GIN data from the main testing suite, see :ref:`example_data` for more details. + +1. Make a new folder for the demo conversion named ``demo_neuroconv_docker``. + +2. Make a subfolder in ``demo_neuroconv_docker`` called ``demo_output``. + +3. Make a new file in this folder named ``demo_neuroconv_docker_yaml.yml``. + +4. Open this YAML file in a text editor, then copy and paste the following section into that file. + +.. code:: + +metadata: + NWBFile: + lab: My Lab + institution: My Institution + +data_interfaces: + ap: SpikeGLXRecordingInterface + phy: PhySortingInterface + +experiments: + my_experiment: + metadata: + NWBFile: + session_description: My session. + + sessions: + - nwbfile_name: spikeglx_from_docker_yaml.nwb + source_data: + ap: + file_path: /demo_neuroconv_docker/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin + metadata: + NWBFile: + session_start_time: "2020-10-10T21:19:09+00:00" + Subject: + subject_id: "1" + sex: F + age: P35D + species: Mus musculus + - nwbfile_name: phy_from_docker_yaml.nwb + metadata: + NWBFile: + session_start_time: "2020-10-10T21:19:09+00:00" + Subject: + subject_id: "002" + sex: F + age: P35D + species: Mus musculus + source_data: + phy: + folder_path: /demo_neuroconv_docker/phy/phy_example_0/ + + +5. To make things easier for volume mounting, copy and paste the ``Noise4Sam_g0`` and ``phy_example_0`` folders into this Docker demo folder so that you have the following folder structure... + +.. code:: + + | demo_neuroconv_docker/ + |-- demo_output/ + |-- demo_neuroconv_docker_yaml.yml + |-- spikeglx/ + |---- Noise4Sam_g0/ + |------ Noise4Sam_g0_imec0/ + |-------- Noise4Sam_g0_t0.imec0.ap.bin + |-------- Noise4Sam_g0_t0.imec0.ap.meta + |-- phy/ + |---- phy_example_0/ + |------ ... # The various file contents from the example Phy folder + +6. Pull the latest NeuroConv docker image from GitHub... + +.. code:: + + docker pull ghcr.io/catalystneuro/neuroconv:latest + +7. Run the command line interface on the YAML file using the docker container (instead of a local installation of the Python package)... + +.. code:: + + docker run -t --volume /demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv test_docker_yaml.yml + +.. note:: Docker relies heavily on absolute system paths, but these can vary depending on your system. For Windows, this might be something like ``C:/Users/MyUser/Downloads/``, for MacOSX it might be ``/Users/username/``. + +For example, assuming a MacOSX user with username 'MyUser', and assuming the ``demo_neuroconv_docker`` folder was created in the home directory, this would look like + +.. code:: + + docker run -t --volume /Users/MyUser/demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/test_docker_yaml.yml From 51e74f12b32d519ea75155e71bfdc9fee8623d4a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:58:32 -0400 Subject: [PATCH 040/119] add to index --- docs/user_guide/user_guide.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user_guide/user_guide.rst b/docs/user_guide/user_guide.rst index 8cf3336c9..b3928f6c6 100644 --- a/docs/user_guide/user_guide.rst +++ b/docs/user_guide/user_guide.rst @@ -23,3 +23,4 @@ and synchronize data across multiple sources. schemas csvs expand_path + docker_demo From 18d518f55508609bb705511cb03f86b6e3d2efe4 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:58:57 -0400 Subject: [PATCH 041/119] remove from dev guide --- docs/developer_guide/docker_images.rst | 96 -------------------------- 1 file changed, 96 deletions(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index fe9d19908..579128967 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -95,99 +95,3 @@ To use automatically via a Python helper function (coming in a separate PR) .. note:: When using YAML files through the docker containers, always be sure that the NWB file paths are absolute paths stemming from the mounted volume; otherwise, the NWB file will indeed be written inside the container but will not be accessible outside of it. - - -Demo ----- - -The following is an explicit demonstration of how to use the Docker-based CLI for NeuroConv. - -It relies on some of the GIN data from the main testing suite, see :ref:`example_data` for more details. - -1. Make a new folder for the demo conversion named ``demo_neuroconv_docker``. - -2. Make a subfolder in ``demo_neuroconv_docker`` called ``demo_output``. - -3. Make a new file in this folder named ``demo_neuroconv_docker_yaml.yml``. - -4. Open this YAML file in a text editor, then copy and paste the following section into that file. - -.. code:: - -metadata: - NWBFile: - lab: My Lab - institution: My Institution - -data_interfaces: - ap: SpikeGLXRecordingInterface - phy: PhySortingInterface - -experiments: - my_experiment: - metadata: - NWBFile: - session_description: My session. - - sessions: - - nwbfile_name: spikeglx_from_docker_yaml.nwb - source_data: - ap: - file_path: /demo_neuroconv_docker/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin - metadata: - NWBFile: - session_start_time: "2020-10-10T21:19:09+00:00" - Subject: - subject_id: "1" - sex: F - age: P35D - species: Mus musculus - - nwbfile_name: phy_from_docker_yaml.nwb - metadata: - NWBFile: - session_start_time: "2020-10-10T21:19:09+00:00" - Subject: - subject_id: "002" - sex: F - age: P35D - species: Mus musculus - source_data: - phy: - folder_path: /demo_neuroconv_docker/phy/phy_example_0/ - - -5. To make things easier for volume mounting, copy and paste the ``Noise4Sam_g0`` and ``phy_example_0`` folders into this Docker demo folder so that you have the following folder structure... - -.. code:: - - | demo_neuroconv_docker/ - |-- demo_output/ - |-- demo_neuroconv_docker_yaml.yml - |-- spikeglx/ - |---- Noise4Sam_g0/ - |------ Noise4Sam_g0_imec0/ - |-------- Noise4Sam_g0_t0.imec0.ap.bin - |-------- Noise4Sam_g0_t0.imec0.ap.meta - |-- phy/ - |---- phy_example_0/ - |------ ... # The various file contents from the example Phy folder - -6. Pull the latest NeuroConv docker image from GitHub... - -.. code:: - - docker pull ghcr.io/catalystneuro/neuroconv:latest - -7. Run the command line interface on the YAML file using the docker container (instead of a local installation of the Python package)... - -.. code:: - - docker run -t --volume /demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv test_docker_yaml.yml - -.. note:: Docker relies heavily on absolute system paths, but these can vary depending on your system. For Windows, this might be something like ``C:/Users/MyUser/Downloads/``, for MacOSX it might be ``/Users/username/``. - -For example, assuming a MacOSX user with username 'MyUser', and assuming the ``demo_neuroconv_docker`` folder was created in the home directory, this would look like - -.. code:: - - docker run -t --volume /Users/MyUser/demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/test_docker_yaml.yml From 778ede9b171625b03d115349504e04e2464797c4 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:59:41 -0400 Subject: [PATCH 042/119] try non-slim and normal python path --- dockerfiles/neuroconv_latest_release_dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/neuroconv_latest_release_dockerfile b/dockerfiles/neuroconv_latest_release_dockerfile index 88f72f73c..392b9c122 100644 --- a/dockerfiles/neuroconv_latest_release_dockerfile +++ b/dockerfiles/neuroconv_latest_release_dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.7-slim +FROM python:3.11.7 LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN pip install neuroconv[full] -CMD ["python3 -m"] +CMD ["python -m"] From 2b973246a21b84056a0e166866b5007879521712 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:01:28 -0400 Subject: [PATCH 043/119] push once and add manual --- .github/workflows/auto-release-docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/auto-release-docker.yml index 9639a29db..af98e853b 100644 --- a/.github/workflows/auto-release-docker.yml +++ b/.github/workflows/auto-release-docker.yml @@ -4,6 +4,8 @@ on: workflow_run: workflows: [auto-publish] types: [completed] + workflow_dispatch: + push: jobs: release-image: From 1b4ad266a41bd69537557e04f54ddae2047b2178 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:01:45 -0400 Subject: [PATCH 044/119] remove push after first time --- .github/workflows/auto-release-docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/auto-release-docker.yml index af98e853b..b610ba242 100644 --- a/.github/workflows/auto-release-docker.yml +++ b/.github/workflows/auto-release-docker.yml @@ -5,7 +5,6 @@ on: workflows: [auto-publish] types: [completed] workflow_dispatch: - push: jobs: release-image: From f2ca5fbaefd2e6cd6974dd390cf66aa45e4677c6 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:07:47 -0400 Subject: [PATCH 045/119] printout tag in action --- .github/workflows/auto-release-docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/auto-release-docker.yml index b610ba242..e5c6af31e 100644 --- a/.github/workflows/auto-release-docker.yml +++ b/.github/workflows/auto-release-docker.yml @@ -19,6 +19,8 @@ jobs: tags="$(git tag --list)" latest_tag=${tags: -6 : 6} echo "latest_tag=$latest_tag" >> "$GITHUB_ENV" + - name: Printout parsed version for GitHub Action log + run: echo "$latest_tag" - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx From 663ee2e0db1f9d03715cd0d051aeae3aa538ac63 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:08:19 -0400 Subject: [PATCH 046/119] clarify workflow name --- .github/workflows/auto-release-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/auto-release-docker.yml index e5c6af31e..158aedbc6 100644 --- a/.github/workflows/auto-release-docker.yml +++ b/.github/workflows/auto-release-docker.yml @@ -1,4 +1,4 @@ -name: Upload Docker Image to GHCR +name: Upload Docker Image of Latest Release to GHCR on: workflow_run: From 9bc34b56b6c4316a3d4e3f1a1728d3608f1c595d Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:09:10 -0400 Subject: [PATCH 047/119] add doc ref --- docs/developer_guide/testing_suite.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/developer_guide/testing_suite.rst b/docs/developer_guide/testing_suite.rst index 73631da32..1b395c284 100644 --- a/docs/developer_guide/testing_suite.rst +++ b/docs/developer_guide/testing_suite.rst @@ -61,6 +61,7 @@ These can be run in isolation using ``pip install -e neuroconv[test,]` ``pytest tests/test_`` where ```` can be any of ``ophys``, ``ecephys``, ``text``, or ``behavior``. +.. _example_data: Testing On Example Data ----------------------- From 3fe08b15df2ddef986b6a0a09d81212adeba75ab Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:09:28 -0400 Subject: [PATCH 048/119] push again for new name --- .github/workflows/auto-release-docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/auto-release-docker.yml index 158aedbc6..82d1f7ace 100644 --- a/.github/workflows/auto-release-docker.yml +++ b/.github/workflows/auto-release-docker.yml @@ -5,6 +5,7 @@ on: workflows: [auto-publish] types: [completed] workflow_dispatch: + push: jobs: release-image: From 56bba2e1bb355944b5868f56291cbb626f1ae223 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:10:39 -0400 Subject: [PATCH 049/119] specify output directory --- docs/user_guide/docker_demo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/docker_demo.rst b/docs/user_guide/docker_demo.rst index 646e7b452..52870578d 100644 --- a/docs/user_guide/docker_demo.rst +++ b/docs/user_guide/docker_demo.rst @@ -91,4 +91,4 @@ For example, assuming a MacOSX user with username 'MyUser', and assuming the ``d .. code:: - docker run -t --volume /Users/MyUser/demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/test_docker_yaml.yml + docker run -t --volume /Users/MyUser/demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/test_docker_yaml.yml --output-folder demo_output From db81628a8cafc85d1d5202d754a3e848d3677e34 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:19:41 -0400 Subject: [PATCH 050/119] set version tag through explicit output --- .github/workflows/auto-release-docker.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/auto-release-docker.yml index 82d1f7ace..d1df25f99 100644 --- a/.github/workflows/auto-release-docker.yml +++ b/.github/workflows/auto-release-docker.yml @@ -14,28 +14,29 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Grab version from release tag + - name: Parse the version from the GitHub latest release tag + id: parsed_version run: | git fetch --prune --unshallow --tags tags="$(git tag --list)" - latest_tag=${tags: -6 : 6} - echo "latest_tag=$latest_tag" >> "$GITHUB_ENV" + version_tag=${tags: -6 : 6} + echo "version_tag=$version_tag" >> $GITHUB_OUTPUT - name: Printout parsed version for GitHub Action log run: echo "$latest_tag" - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ secrets.DOCKER_UPLOADER_USERNAME }} password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: push: true # Push is a shorthand for --output=type=registry - tags: ghcr.io/CatalystNeuro/NeuroConv:$latest_tag + tags: ghcr.io/catalystneuro/neuroconv:latest,ghcr.io/catalystneuro/neuroconv:${{ steps.parsed_version.outputs.version_tag }} context: docker file: dockerfiles/neuroconv_developer_build_dockerfile From 09a3ccf463280e6ccfeee5ff41f0949ae4406b06 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:25:39 -0400 Subject: [PATCH 051/119] try dot context in v5 --- .github/workflows/auto-release-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/auto-release-docker.yml index d1df25f99..cd6f45cea 100644 --- a/.github/workflows/auto-release-docker.yml +++ b/.github/workflows/auto-release-docker.yml @@ -22,7 +22,7 @@ jobs: version_tag=${tags: -6 : 6} echo "version_tag=$version_tag" >> $GITHUB_OUTPUT - name: Printout parsed version for GitHub Action log - run: echo "$latest_tag" + run: echo ${{ steps.parsed_version.outputs.version_tag }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -38,5 +38,5 @@ jobs: with: push: true # Push is a shorthand for --output=type=registry tags: ghcr.io/catalystneuro/neuroconv:latest,ghcr.io/catalystneuro/neuroconv:${{ steps.parsed_version.outputs.version_tag }} - context: docker + context: . file: dockerfiles/neuroconv_developer_build_dockerfile From 7d2322bdb1c4ec3aef99183adc282cb50f85f690 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:29:57 -0400 Subject: [PATCH 052/119] correct names for latest release --- ... => build_and_upload_docker_image_of_latest_release.yml} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{auto-release-docker.yml => build_and_upload_docker_image_of_latest_release.yml} (87%) diff --git a/.github/workflows/auto-release-docker.yml b/.github/workflows/build_and_upload_docker_image_of_latest_release.yml similarity index 87% rename from .github/workflows/auto-release-docker.yml rename to .github/workflows/build_and_upload_docker_image_of_latest_release.yml index cd6f45cea..d1364e587 100644 --- a/.github/workflows/auto-release-docker.yml +++ b/.github/workflows/build_and_upload_docker_image_of_latest_release.yml @@ -1,4 +1,4 @@ -name: Upload Docker Image of Latest Release to GHCR +name: Build and Upload Docker Image of Latest Release to GHCR on: workflow_run: @@ -9,7 +9,7 @@ on: jobs: release-image: - name: Release NeuroConv Image + name: Build and Upload Docker Image of Latest Release to GHCR runs-on: ubuntu-latest steps: - name: Checkout @@ -39,4 +39,4 @@ jobs: push: true # Push is a shorthand for --output=type=registry tags: ghcr.io/catalystneuro/neuroconv:latest,ghcr.io/catalystneuro/neuroconv:${{ steps.parsed_version.outputs.version_tag }} context: . - file: dockerfiles/neuroconv_developer_build_dockerfile + file: dockerfiles/neuroconv_latest_release_dockerfile From 67b7ad5f9c76c3b159ebc5f7ef79bb35544056ed Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:39:32 -0400 Subject: [PATCH 053/119] undo push after success --- .../build_and_upload_docker_image_of_latest_release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_and_upload_docker_image_of_latest_release.yml b/.github/workflows/build_and_upload_docker_image_of_latest_release.yml index d1364e587..a9bdff091 100644 --- a/.github/workflows/build_and_upload_docker_image_of_latest_release.yml +++ b/.github/workflows/build_and_upload_docker_image_of_latest_release.yml @@ -5,7 +5,6 @@ on: workflows: [auto-publish] types: [completed] workflow_dispatch: - push: jobs: release-image: From 586828a7f82a32f65f5ef13446e31e46180f2c35 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:42:57 -0400 Subject: [PATCH 054/119] correct docs indent --- docs/user_guide/docker_demo.rst | 70 ++++++++++++++++----------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/user_guide/docker_demo.rst b/docs/user_guide/docker_demo.rst index 52870578d..4909c2507 100644 --- a/docs/user_guide/docker_demo.rst +++ b/docs/user_guide/docker_demo.rst @@ -15,46 +15,46 @@ It relies on some of the GIN data from the main testing suite, see :ref:`example .. code:: -metadata: - NWBFile: - lab: My Lab - institution: My Institution - -data_interfaces: - ap: SpikeGLXRecordingInterface - phy: PhySortingInterface - -experiments: - my_experiment: metadata: NWBFile: - session_description: My session. + lab: My Lab + institution: My Institution - sessions: - - nwbfile_name: spikeglx_from_docker_yaml.nwb - source_data: - ap: - file_path: /demo_neuroconv_docker/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin - metadata: - NWBFile: - session_start_time: "2020-10-10T21:19:09+00:00" - Subject: - subject_id: "1" - sex: F - age: P35D - species: Mus musculus - - nwbfile_name: phy_from_docker_yaml.nwb + data_interfaces: + ap: SpikeGLXRecordingInterface + phy: PhySortingInterface + + experiments: + my_experiment: metadata: NWBFile: - session_start_time: "2020-10-10T21:19:09+00:00" - Subject: - subject_id: "002" - sex: F - age: P35D - species: Mus musculus - source_data: - phy: - folder_path: /demo_neuroconv_docker/phy/phy_example_0/ + session_description: My session. + + sessions: + - nwbfile_name: spikeglx_from_docker_yaml.nwb + source_data: + ap: + file_path: /demo_neuroconv_docker/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin + metadata: + NWBFile: + session_start_time: "2020-10-10T21:19:09+00:00" + Subject: + subject_id: "1" + sex: F + age: P35D + species: Mus musculus + - nwbfile_name: phy_from_docker_yaml.nwb + metadata: + NWBFile: + session_start_time: "2020-10-10T21:19:09+00:00" + Subject: + subject_id: "002" + sex: F + age: P35D + species: Mus musculus + source_data: + phy: + folder_path: /demo_neuroconv_docker/phy/phy_example_0/ 5. To make things easier for volume mounting, copy and paste the ``Noise4Sam_g0`` and ``phy_example_0`` folders into this Docker demo folder so that you have the following folder structure... From da7835b13d306956a16da063186245d563e5422d Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:30:21 -0400 Subject: [PATCH 055/119] polish demo --- docs/user_guide/docker_demo.rst | 69 ++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/docs/user_guide/docker_demo.rst b/docs/user_guide/docker_demo.rst index 4909c2507..3b21138b5 100644 --- a/docs/user_guide/docker_demo.rst +++ b/docs/user_guide/docker_demo.rst @@ -5,13 +5,29 @@ The following is an explicit demonstration of how to use the Docker-based CLI fo It relies on some of the GIN data from the main testing suite, see :ref:`example_data` for more details. + +.. note:: + + Docker relies heavily on absolute system paths, but these can vary depending on your system. + + For Windows, this might be something like: ``C:/Users/MyUser/Downloads/``. + For MacOSX it might be: ``/Users/username/``. + + This demo will use home directory of an Ubuntu (Linux) system for a user named 'MyUser': ``/home/MyUser``. + + +.. note:: + + For Unix systems (MacOSX/Linux) you will likely require sudo access in order to run the ``docker`` based commands. + + If this is the case for your system, then any time you see the ``docker`` usage on the command like you will need to prepend as ``sudo docker``. + + 1. Make a new folder for the demo conversion named ``demo_neuroconv_docker``. 2. Make a subfolder in ``demo_neuroconv_docker`` called ``demo_output``. -3. Make a new file in this folder named ``demo_neuroconv_docker_yaml.yml``. - -4. Open this YAML file in a text editor, then copy and paste the following section into that file. +3. Create a file in this folder named ``demo_neuroconv_docker_yaml.yml`` with the following content... .. code:: @@ -57,38 +73,47 @@ It relies on some of the GIN data from the main testing suite, see :ref:`example folder_path: /demo_neuroconv_docker/phy/phy_example_0/ -5. To make things easier for volume mounting, copy and paste the ``Noise4Sam_g0`` and ``phy_example_0`` folders into this Docker demo folder so that you have the following folder structure... +4. To make things easier for volume mounting, copy and paste the ``Noise4Sam_g0`` and ``phy_example_0`` folders into this Docker demo folder so that you have the following folder structure... .. code:: - | demo_neuroconv_docker/ - |-- demo_output/ - |-- demo_neuroconv_docker_yaml.yml - |-- spikeglx/ - |---- Noise4Sam_g0/ - |------ Noise4Sam_g0_imec0/ - |-------- Noise4Sam_g0_t0.imec0.ap.bin - |-------- Noise4Sam_g0_t0.imec0.ap.meta - |-- phy/ - |---- phy_example_0/ - |------ ... # The various file contents from the example Phy folder - -6. Pull the latest NeuroConv docker image from GitHub... + demo_neuroconv_docker/ + ¦ demo_output/ + ¦ demo_neuroconv_docker_yaml.yml + ¦ spikeglx/ + ¦ +-- Noise4Sam_g0/ + ¦ +-- ... # .nidq streams + ¦ ¦ +-- Noise4Sam_g0_imec0/ + ¦ ¦ +-- Noise4Sam_g0_t0.imec0.ap.bin + ¦ ¦ +-- Noise4Sam_g0_t0.imec0.ap.meta + ¦ ¦ +-- ... # .lf streams + ¦ phy/ + ¦ +-- phy_example_0/ + ¦ ¦ +-- ... # The various file contents from the example Phy folder + +5. Pull the latest NeuroConv docker image from GitHub... .. code:: docker pull ghcr.io/catalystneuro/neuroconv:latest -7. Run the command line interface on the YAML file using the docker container (instead of a local installation of the Python package)... +6. Run the command line interface on the YAML file using the docker container (instead of a local installation of the Python package)... .. code:: - docker run -t --volume /demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv test_docker_yaml.yml + docker run -t --volume /home/user/demo_neuroconv_docker:/demo_neuroconv_docker ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/demo_neuroconv_docker_yaml.yml --output-folder-path /demo_neuroconv_docker/demo_output -.. note:: Docker relies heavily on absolute system paths, but these can vary depending on your system. For Windows, this might be something like ``C:/Users/MyUser/Downloads/``, for MacOSX it might be ``/Users/username/``. +docker run -t --volume /home/ubuntu/demo_neuroconv_docker:/demo_neuroconv_docker ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/demo_neuroconv_docker_yaml.yml --output-folder-path /demo_neuroconv_docker/demo_output -For example, assuming a MacOSX user with username 'MyUser', and assuming the ``demo_neuroconv_docker`` folder was created in the home directory, this would look like +Voilà! If everything occurred successfully, you should see... .. code:: - docker run -t --volume /Users/MyUser/demo_neuroconv_docker:/demo_neuroconv_docker/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/test_docker_yaml.yml --output-folder demo_output + Source data is valid! + Metadata is valid! + conversion_options is valid! + NWB file saved at /demo_neuroconv_docker/demo_output/spikeglx_from_docker_yaml.nwb! + Source data is valid! + Metadata is valid! + conversion_options is valid! + NWB file saved at /demo_neuroconv_docker/demo_output/phy_from_docker_yaml.nwb! From af9e86f0a56b7e6b07fc09c7d981a4543de08eed Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:33:28 -0400 Subject: [PATCH 056/119] try to determine if slim is the problem --- dockerfiles/neuroconv_latest_release_dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_latest_release_dockerfile b/dockerfiles/neuroconv_latest_release_dockerfile index 392b9c122..daad0394d 100644 --- a/dockerfiles/neuroconv_latest_release_dockerfile +++ b/dockerfiles/neuroconv_latest_release_dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.7 +FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN pip install neuroconv[full] CMD ["python -m"] From b76a18e961b32170fee414e6060990bff3f89b02 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:34:29 -0400 Subject: [PATCH 057/119] re-add push --- .../build_and_upload_docker_image_of_latest_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_upload_docker_image_of_latest_release.yml b/.github/workflows/build_and_upload_docker_image_of_latest_release.yml index a9bdff091..a6f723182 100644 --- a/.github/workflows/build_and_upload_docker_image_of_latest_release.yml +++ b/.github/workflows/build_and_upload_docker_image_of_latest_release.yml @@ -5,6 +5,7 @@ on: workflows: [auto-publish] types: [completed] workflow_dispatch: + push: # remove after merge; can't trigger manually until on main jobs: release-image: From 8cb41c7f07eb44f40ab475f4427894d59c8cb61f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:38:46 -0400 Subject: [PATCH 058/119] update dev version too --- ... => build_and_upload_docker_image_dev.yml} | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) rename .github/workflows/{auto-publish-docker-dev-branch.yml => build_and_upload_docker_image_dev.yml} (52%) diff --git a/.github/workflows/auto-publish-docker-dev-branch.yml b/.github/workflows/build_and_upload_docker_image_dev.yml similarity index 52% rename from .github/workflows/auto-publish-docker-dev-branch.yml rename to .github/workflows/build_and_upload_docker_image_dev.yml index 096fea4b0..601629f35 100644 --- a/.github/workflows/auto-publish-docker-dev-branch.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -1,35 +1,32 @@ -name: Upload Docker Image of Latest Developer Branch to GHCR +name: Build and Upload Docker Image of Current Dev Branch to GHCR on: schedule: - cron: "0 16 * * *" # Daily at noon EST workflow_dispatch: - -concurrency: # Cancel previous workflows on the same pull request - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + push: # remove after merge; can't trigger manually until on main jobs: release-image: - name: Release NeuroConv Image of Current Dev Branch + name: Build and Upload Docker Image of Current Dev Branch to GHCR runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ secrets.DOCKER_UPLOADER_USERNAME }} password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: push: true # Push is a shorthand for --output=type=registry tags: ghcr.io/catalystneuro/neuroconv:dev - context: dockerfiles - file: dockerfiles/neuroconv_developer_build_dockerfile + context: . + file: dockerfiles/neuroconv_dev_branch_dockerfile From 153f95c4f2869e06b058a7f2d7af72c6b51aa9b0 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:39:03 -0400 Subject: [PATCH 059/119] shorten --- ...lease.yml => build_and_upload_docker_image_latest_release.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{build_and_upload_docker_image_of_latest_release.yml => build_and_upload_docker_image_latest_release.yml} (100%) diff --git a/.github/workflows/build_and_upload_docker_image_of_latest_release.yml b/.github/workflows/build_and_upload_docker_image_latest_release.yml similarity index 100% rename from .github/workflows/build_and_upload_docker_image_of_latest_release.yml rename to .github/workflows/build_and_upload_docker_image_latest_release.yml From edc6e1911ffe709904d82d14e64e3b6b3afa5775 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:39:36 -0400 Subject: [PATCH 060/119] adjust dockerfile name --- ...conv_developer_build_dockerfile => neuroconv_dev_dockerfile} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename dockerfiles/{neuroconv_developer_build_dockerfile => neuroconv_dev_dockerfile} (90%) diff --git a/dockerfiles/neuroconv_developer_build_dockerfile b/dockerfiles/neuroconv_dev_dockerfile similarity index 90% rename from dockerfiles/neuroconv_developer_build_dockerfile rename to dockerfiles/neuroconv_dev_dockerfile index 3ed66d8dc..6cf2b161c 100644 --- a/dockerfiles/neuroconv_developer_build_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -1,4 +1,4 @@ FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN pip install git+https://github.com/catalystneuro/neuroconv.git@main -CMD ["python3 -m"] +CMD ["python -m"] From 1ad92c4c4b38ef531007a4fa29f559bba62bcb0d Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:42:08 -0400 Subject: [PATCH 061/119] correct file reference --- .github/workflows/build_and_upload_docker_image_dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index 601629f35..5eca20abd 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -29,4 +29,4 @@ jobs: push: true # Push is a shorthand for --output=type=registry tags: ghcr.io/catalystneuro/neuroconv:dev context: . - file: dockerfiles/neuroconv_dev_branch_dockerfile + file: dockerfiles/neuroconv_dev_dockerfile From 5e967fa773bd7e107e8a08e817111433567e168d Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:53:38 -0400 Subject: [PATCH 062/119] update demo --- docs/user_guide/docker_demo.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/user_guide/docker_demo.rst b/docs/user_guide/docker_demo.rst index 3b21138b5..92a1d5b2f 100644 --- a/docs/user_guide/docker_demo.rst +++ b/docs/user_guide/docker_demo.rst @@ -1,7 +1,7 @@ Docker Demo ----------- -The following is an explicit demonstration of how to use the Docker-based CLI for NeuroConv. +The following is an explicit demonstration of how to use the Docker-based NeuroConv YAML specification via the command line. It relies on some of the GIN data from the main testing suite, see :ref:`example_data` for more details. @@ -11,9 +11,10 @@ It relies on some of the GIN data from the main testing suite, see :ref:`example Docker relies heavily on absolute system paths, but these can vary depending on your system. For Windows, this might be something like: ``C:/Users/MyUser/Downloads/``. + For MacOSX it might be: ``/Users/username/``. - This demo will use home directory of an Ubuntu (Linux) system for a user named 'MyUser': ``/home/MyUser``. + For this demo, we will use the home directory of an Ubuntu (Linux) system for a user named 'MyUser' as our base: ``/home/MyUser``. .. note:: @@ -23,7 +24,7 @@ It relies on some of the GIN data from the main testing suite, see :ref:`example If this is the case for your system, then any time you see the ``docker`` usage on the command like you will need to prepend as ``sudo docker``. -1. Make a new folder for the demo conversion named ``demo_neuroconv_docker``. +1. In your base directory (which this demo will refer to as ``/home/MyUser/``), make a new folder for the demo conversion named ``demo_neuroconv_docker``. 2. Make a subfolder in ``demo_neuroconv_docker`` called ``demo_output``. @@ -103,8 +104,6 @@ It relies on some of the GIN data from the main testing suite, see :ref:`example docker run -t --volume /home/user/demo_neuroconv_docker:/demo_neuroconv_docker ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/demo_neuroconv_docker_yaml.yml --output-folder-path /demo_neuroconv_docker/demo_output -docker run -t --volume /home/ubuntu/demo_neuroconv_docker:/demo_neuroconv_docker ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/demo_neuroconv_docker_yaml.yml --output-folder-path /demo_neuroconv_docker/demo_output - Voilà! If everything occurred successfully, you should see... .. code:: From 714aa4b23d2c8387cc40ced9ecadd584563648bf Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:00:59 -0400 Subject: [PATCH 063/119] swap to conda for git in dev --- dockerfiles/neuroconv_dev_dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dockerfiles/neuroconv_dev_dockerfile b/dockerfiles/neuroconv_dev_dockerfile index 6cf2b161c..2a74c26f0 100644 --- a/dockerfiles/neuroconv_dev_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -1,4 +1,7 @@ -FROM python:3.11.7-slim +FROM continuumio/anaconda3:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -RUN pip install git+https://github.com/catalystneuro/neuroconv.git@main +RUN conda create -n neuroconv python=3.11.7 +RUN conda activate neuroconv \ + conda install git + python -m pip install git+https://github.com/catalystneuro/neuroconv.git@main CMD ["python -m"] From 2a0896d33874795b942633d9f6cf3bf21b99c001 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:12:59 -0400 Subject: [PATCH 064/119] debugging dev file --- dockerfiles/neuroconv_dev_dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_dev_dockerfile b/dockerfiles/neuroconv_dev_dockerfile index 2a74c26f0..590198540 100644 --- a/dockerfiles/neuroconv_dev_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -2,6 +2,6 @@ FROM continuumio/anaconda3:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN conda create -n neuroconv python=3.11.7 RUN conda activate neuroconv \ - conda install git + conda install git \ python -m pip install git+https://github.com/catalystneuro/neuroconv.git@main CMD ["python -m"] From 019cf5cad82b5a98a936ad032bb300f3b304a75a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:22:10 -0400 Subject: [PATCH 065/119] include szonja fix --- dockerfiles/neuroconv_latest_release_dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfiles/neuroconv_latest_release_dockerfile b/dockerfiles/neuroconv_latest_release_dockerfile index daad0394d..7a534b3ff 100644 --- a/dockerfiles/neuroconv_latest_release_dockerfile +++ b/dockerfiles/neuroconv_latest_release_dockerfile @@ -1,4 +1,5 @@ FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv +RUN apt update && apt install musl-dev python3-dev -y RUN pip install neuroconv[full] CMD ["python -m"] From 62d0b499f6b9f72f744f00fc2209040332ce2340 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:23:32 -0400 Subject: [PATCH 066/119] debug dev file --- dockerfiles/neuroconv_dev_dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/neuroconv_dev_dockerfile b/dockerfiles/neuroconv_dev_dockerfile index 590198540..5819318be 100644 --- a/dockerfiles/neuroconv_dev_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -1,7 +1,7 @@ FROM continuumio/anaconda3:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN conda create -n neuroconv python=3.11.7 -RUN conda activate neuroconv \ - conda install git \ +RUN conda activate neuroconv & + conda install git & python -m pip install git+https://github.com/catalystneuro/neuroconv.git@main CMD ["python -m"] From ab82d221c967aadb5492ae546365190666d4918f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:24:19 -0400 Subject: [PATCH 067/119] cancel past triggers --- .github/workflows/build_and_upload_docker_image_dev.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index 5eca20abd..531a236cd 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -6,6 +6,10 @@ on: workflow_dispatch: push: # remove after merge; can't trigger manually until on main +concurrency: # Cancel previous workflows on the same pull request + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: release-image: name: Build and Upload Docker Image of Current Dev Branch to GHCR From 25f080f16fe453a94682d84b9d8ac3971860090e Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:24:37 -0400 Subject: [PATCH 068/119] cancel past triggers --- .../build_and_upload_docker_image_latest_release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_and_upload_docker_image_latest_release.yml b/.github/workflows/build_and_upload_docker_image_latest_release.yml index a6f723182..47d138bb0 100644 --- a/.github/workflows/build_and_upload_docker_image_latest_release.yml +++ b/.github/workflows/build_and_upload_docker_image_latest_release.yml @@ -7,6 +7,10 @@ on: workflow_dispatch: push: # remove after merge; can't trigger manually until on main +concurrency: # Cancel previous workflows on the same pull request + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: release-image: name: Build and Upload Docker Image of Latest Release to GHCR From d07d75046bb843326f957b0b27ab16a7e4ac5a57 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:37:58 -0400 Subject: [PATCH 069/119] debug dev file --- dockerfiles/neuroconv_dev_dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dockerfiles/neuroconv_dev_dockerfile b/dockerfiles/neuroconv_dev_dockerfile index 5819318be..fa91c322f 100644 --- a/dockerfiles/neuroconv_dev_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -1,7 +1,5 @@ FROM continuumio/anaconda3:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN conda create -n neuroconv python=3.11.7 -RUN conda activate neuroconv & - conda install git & - python -m pip install git+https://github.com/catalystneuro/neuroconv.git@main +RUN conda activate neuroconv && conda install git && python -m pip install git+https://github.com/catalystneuro/neuroconv.git@main CMD ["python -m"] From e5a6e63fa08d898338ebf841563cd7b457801084 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:44:51 -0400 Subject: [PATCH 070/119] move docker tests out of main scopes --- ...est.py => docker_yaml_conversion_specification_cli.py} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename tests/{test_on_data/docker_yaml_conversion_specification_cli_test.py => docker_yaml_conversion_specification_cli.py} (95%) diff --git a/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py b/tests/docker_yaml_conversion_specification_cli.py similarity index 95% rename from tests/test_on_data/docker_yaml_conversion_specification_cli_test.py rename to tests/docker_yaml_conversion_specification_cli.py index e0d16fa44..026a36bf2 100644 --- a/tests/test_on_data/docker_yaml_conversion_specification_cli_test.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -14,8 +14,8 @@ from neuroconv.tools import deploy_process -from .setup_paths import ECEPHY_DATA_PATH as DATA_PATH -from .setup_paths import OUTPUT_PATH +from .test_on_data.setup_paths import ECEPHY_DATA_PATH as DATA_PATH +from .test_on_data.setup_paths import OUTPUT_PATH class TestLatestDockerYAMLConversionSpecification(TestCase): @@ -30,8 +30,8 @@ def test_run_conversion_from_yaml_cli(self): command=( "docker run -it " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag}" - f"neuroconv {yaml_file_path} " - f"--data-folder-path {DATA_PATH} --output-folder-path {self.test_folder} --overwrite" + f"neuroconv /neuroconv/tests/test_on_data/conversion_specifications/{yaml_file_path} " + f"--data-folder-path /neuroconv/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ) ) From 33b7b4c4c0d85a7f3262acaf7ce3b6c680876c00 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:45:36 -0400 Subject: [PATCH 071/119] clarify docs --- tests/docker_yaml_conversion_specification_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index 026a36bf2..4145a6888 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -1,5 +1,5 @@ """ -This file is hidden from normal pytest globbing by appending the _test to the end instead of the beginning. +This file is hidden from normal pytest globbing by not including 'test' in the filename. Instead, the tests must be invoked directly from the file. This is designed mostly for use in the GitHub Actions. """ From 22434b644b8c541d2bc1fd8426786c02805e3b2c Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:51:42 -0400 Subject: [PATCH 072/119] dev file debugging --- dockerfiles/neuroconv_dev_dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_dev_dockerfile b/dockerfiles/neuroconv_dev_dockerfile index fa91c322f..c92ded8b3 100644 --- a/dockerfiles/neuroconv_dev_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -1,5 +1,5 @@ FROM continuumio/anaconda3:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv RUN conda create -n neuroconv python=3.11.7 -RUN conda activate neuroconv && conda install git && python -m pip install git+https://github.com/catalystneuro/neuroconv.git@main +RUN conda activate neuroconv && /bin/sh -c conda install git && python -m pip install git+https://github.com/catalystneuro/neuroconv.git@main CMD ["python -m"] From eab9ab2162dc124fbd131e8734a26d70789b00d5 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:25:44 -0400 Subject: [PATCH 073/119] move rclone stuff to other branch --- dockerfiles/rclone_auto_config_dockerfile | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 dockerfiles/rclone_auto_config_dockerfile diff --git a/dockerfiles/rclone_auto_config_dockerfile b/dockerfiles/rclone_auto_config_dockerfile deleted file mode 100644 index a97a166ae..000000000 --- a/dockerfiles/rclone_auto_config_dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM rclone/rclone:latest -CMD printf "[$DRIVE_NAME]\ntype = drive\nscope = drive\ntoken = {\"access_token\":\"$TOKEN\",\"token_type\":\"Bearer\",\"refresh_token\":\"$REFRESH_TOKEN\",\"expiry\":\"$EXPIRY\"}" > /mnt/data/rclone.conf \ - && cat /mnt/data/rclone.conf \ - && rclone $RCLONE_COMMAND -ENTRYPOINT [""] From ff957953e11df30080b83e26935db3e27b46755b Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:27:55 -0400 Subject: [PATCH 074/119] re-order user guide --- docs/user_guide/user_guide.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user_guide/user_guide.rst b/docs/user_guide/user_guide.rst index b3928f6c6..0752b3bd6 100644 --- a/docs/user_guide/user_guide.rst +++ b/docs/user_guide/user_guide.rst @@ -2,7 +2,7 @@ User Guide ========== NeuroConv allows you to easily build programs to convert data from neurophysiology experiments -to NWB. The building-blocks of these conversions are ``DataInterface`` classes. Each +to NWB. The building blocks of these conversions are ``DataInterface`` classes. Each ``DataInterface`` is responsible for a specific format of data, and contains methods to read data and metadata from that format and write it to NWB. We have pre-built ``DataInterface`` classes for many common data formats available in our :ref:`Conversion Gallery `. @@ -18,9 +18,9 @@ and synchronize data across multiple sources. datainterfaces nwbconverter - yaml - temporal_alignment schemas + temporal_alignment csvs expand_path + yaml docker_demo From 088026a9f5bf287e9f2b06ea0aa01bcb249c166c Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:44:59 -0400 Subject: [PATCH 075/119] update dev docs --- docs/developer_guide/docker_images.rst | 57 ++++++++++++-------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index 579128967..018a1c272 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -1,15 +1,19 @@ Manually Build Docker Images ---------------------------- -.. note:: It is recommended to build the docker image on the same system architecture that you intend to run it on, *i.e.*, AWS Linux AMI 64-bit (x86), as it may experience difficulties running on other significantly different systems (like an M1 Mac). +.. note:: + + It is recommended to build the docker image on the same system architecture that you intend to run it on, *i.e.*, AWS Linux AMI 64-bit (x86), as it may experience difficulties running on other significantly different systems (like an M1 Mac). + +.. note:: -.. note:: The NeuroConv docker container comes prepackaged with all required installations, *i.e.*, equivalent to `pip install neuroconv[full]`. As such it is relatively heavy, so be sure that whatever environment you intend to use it in (such as in continuous integration) has sufficient disk space. + The NeuroConv docker container comes prepackaged with all required installations, equivalent to running ``pip install neuroconv[full]``. As such it is relatively heavy, so be sure that whatever environment you intend to use it in (such as in continuous integration) has sufficient disk space. Latest Release ~~~~~~~~~~~~~~ -To manually build the most recent release, navigate to the 'neuroconv/dockerfiles' folder and run... +To manually build the most recent release, navigate to the ``neuroconv/dockerfiles`` folder and run... .. code:: @@ -19,25 +23,25 @@ To manually build the most recent release, navigate to the 'neuroconv/dockerfile Dev Branch ~~~~~~~~~~ -Checkout to a specific branch on a local clone, then +Checkout to a specific branch on a local clone, then... .. code:: - docker build -f neuroconv_developer_build_dockerfile -t neuroconv_dev . + docker build -f neuroconv_dev_dockerfile -t neuroconv_dev . Publish Container to GitHub --------------------------- -The `LABEL` is the important item here; it determines the host repository on the GitHub Container Registry (GHCR). In each docker file we wish to publish on the GHCR, we will add this label right after the `FROM` clause +The ``LABEL`` is important to include as it determines the host repository on the GitHub Container Registry (GHCR). In each dockerfile we wish to publish on the GHCR, we will add this label right after the ``FROM`` clause... .. code:: FROM PARENT_IMAGE:TAG LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -After building the image itself, we can publish the container with +After building the image itself, we can publish the container with... .. code:: @@ -46,20 +50,22 @@ After building the image itself, we can publish the container with echo $CR_PAT | docker login ghcr.io -u --password-stdin docker push ghcr.io/catalystneuro/IMAGE_NAME:TAG -.. note:: Though it may appear confusing, the use of the `IMAGE_NAME` in these steps determines only the _name_ of the package as available from the 'packages' screen of the host repository; the `LABEL` itself ensured the upload and linkage to the NeuroConv GHCR. +.. note:: + + Though it may appear confusing, the use of the ``IMAGE_NAME`` in these steps determines only the _name_ of the package as available from the 'packages' screen of the host repository; the ``LABEL`` itself ensured the upload and linkage to the NeuroConv GHCR. -Running Docker Container on local YAML (Linux) ----------------------------------------------- +Run Docker container on local YAML conversion specification file +---------------------------------------------------------------- -You can either perform a manual build locally following the instructions above, or pull the container from the GitHub Container Registry (GHCR) with +You can either perform a manual build locally following the instructions above, or pull the container from the GitHub Container Registry (GHCR) with... .. code:: docker pull ghcr.io/catalystneuro/neuroconv:latest -and can then run the entrypoint (equivalent to the usual CLI usage) on a YAML specification file (named `your_specification_file.yml`) with +and can then run the entrypoint (equivalent to the usual command line usage) on a YAML specification file (named ``your_specification_file.yml``) with... .. code:: @@ -67,31 +73,22 @@ and can then run the entrypoint (equivalent to the usual CLI usage) on a YAML sp -Running Docker Container with a YAML_STREAM (Linux) ---------------------------------------------------- +Run Docker container on YAML conversion specification environment variable +-------------------------------------------------------------------------- -An alternative approach that simplifies usage on AWS Batch is to specify the YAML contents as an environment variable. The file is constructed in the first step of the container launch. The only potential downside with this usage is maximum file size (~13,000 characters; current example files do not come remotely close to this). +An alternative approach that simplifies usage on systems such as AWS Batch is to specify the YAML contents as an environment variable. The YAML file is constructed in the first step of the container launch. -Otherwise, the YAML file transfer will have to be managed separately, likely as a part of the data transfer or an entirely separate transfer step. +The only potential downside with this usage is the maximum size of an environment variable (~13,000 characters). Typical YAML specification files should not come remotely close to this limit. -To use manually outside of AWS Batch, +Otherwise, in any cloud deployment, the YAML file transfer will have to be managed separately, likely as a part of the data transfer or an entirely separate step. -.. code:: - - export YAML_STREAM="" - docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:dev_auto_yaml - -To use automatically via a Python helper function (coming in a separate PR) +To use this alternative image on a local environment, you no longer need to invoke the ``neuroconv`` entrypoint pointing to a file. Instead, just set the environment variable and run the docker container on the mounted volume... .. code:: - import os - - with open(file="my_yaml_file.yml") as file: - yaml_stream = "".join(file.readlines()).replace("\"", "'") - - os.environ["YAML_STREAM"] = yaml_stream + export YAML_STREAM="" # On Windows, use `set` instead of `export` + docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:yaml_variable .. note:: - When using YAML files through the docker containers, always be sure that the NWB file paths are absolute paths stemming from the mounted volume; otherwise, the NWB file will indeed be written inside the container but will not be accessible outside of it. + When using YAML files through the docker containers, always be sure that the file and folder paths are absolute stemming from the base volume as mounted inside the container; otherwise, the NWB file will be written inside pf the container but will then not be accessible outside of it. From 2bf30e3ba24368c736285eec9df60997d08e3b1f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:46:57 -0400 Subject: [PATCH 076/119] update yaml variable dockerfile --- dockerfiles/neuroconv_developer_build_auto_config | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dockerfiles/neuroconv_developer_build_auto_config b/dockerfiles/neuroconv_developer_build_auto_config index 01b46d92b..98688d586 100644 --- a/dockerfiles/neuroconv_developer_build_auto_config +++ b/dockerfiles/neuroconv_developer_build_auto_config @@ -1,8 +1,3 @@ -FROM python:3.11.7-slim +FROM https://github.com/catalystneuro/neuroconv:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -ADD /neuroconv /neuroconv -RUN cd neuroconv \ - && pip install -e .[full] -CMD echo "$YAML_STREAM" > run.yml \ - && cat run.yml \ - && neuroconv run.yml +CMD echo "$NEUROCONV_YAML" > run.yml && neuroconv run.yml From 47967b91f9961a45ebdc2ce5906631e3241dd3dc Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:48:32 -0400 Subject: [PATCH 077/119] adjust name --- ...developer_build_auto_config => neuroconv_latest_yaml_variable} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dockerfiles/{neuroconv_developer_build_auto_config => neuroconv_latest_yaml_variable} (100%) diff --git a/dockerfiles/neuroconv_developer_build_auto_config b/dockerfiles/neuroconv_latest_yaml_variable similarity index 100% rename from dockerfiles/neuroconv_developer_build_auto_config rename to dockerfiles/neuroconv_latest_yaml_variable From d590878e3feb10837850e46dcad6675fe8d62b78 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:53:17 -0400 Subject: [PATCH 078/119] add workflow for YAML alternative --- ..._and_upload_docker_image_yaml_variable.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/build_and_upload_docker_image_yaml_variable.yml diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml new file mode 100644 index 000000000..110c901a5 --- /dev/null +++ b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml @@ -0,0 +1,31 @@ +name: Build and Upload Docker Image of latest with YAML variable to GHCR + +on: + workflow_run: + workflows: [build_and_upload_docker_image_latest_release] + types: [completed] + workflow_dispatch: + push: # remove after merge; can't trigger manually until on main + +concurrency: # Cancel previous workflows on the same pull request + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + release-image: + name: Build and Upload Docker Image of latest with YAML variable to GHCR + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push YAML variable image based on latest + uses: docker/build-push-action@v5 + with: + push: true # Push is a shorthand for --output=type=registry + tags: ghcr.io/catalystneuro/neuroconv:yaml_variable + context: . + file: dockerfiles/neuroconv_latest_yaml_variable From b3423e87ac473d20a3ba388a405f4e2e62c8f59f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:55:02 -0400 Subject: [PATCH 079/119] adjust docker test --- .github/workflows/docker_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_testing.yml b/.github/workflows/docker_testing.yml index 8ea59d235..7bfc866fe 100644 --- a/.github/workflows/docker_testing.yml +++ b/.github/workflows/docker_testing.yml @@ -88,4 +88,4 @@ jobs: run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/behavior_testing_data ./behavior_testing_data - name: Run docker tests - run: pytest tests/test_on_data/docker_yaml_conversion_specification_cli_test.py -vv -rsx + run: pytest tests/docker_yaml_conversion_specification_cli.py -vv -rsx From 8ccace1fddf2610cefea93aa75a314e24ba40160 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:04:53 -0400 Subject: [PATCH 080/119] add test for yaml variable --- ...ocker_yaml_conversion_specification_cli.py | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index 4145a6888..fb2405758 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -28,7 +28,7 @@ def test_run_conversion_from_yaml_cli(self): yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" deploy_process( command=( - "docker run -it " + "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag}" f"neuroconv /neuroconv/tests/test_on_data/conversion_specifications/{yaml_file_path} " f"--data-folder-path /neuroconv/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" @@ -74,7 +74,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification_missing_nwbfile_names.yml" deploy_process( command=( - "docker run -it " + "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag}" f"neuroconv {yaml_file_path} " f"--data-folder-path {DATA_PATH} --output-folder-path {self.test_folder} --overwrite" @@ -123,3 +123,54 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") assert nwbfile.subject.subject_id == "Subject Name" assert "spike_times" in nwbfile.units + +def test_run_conversion_from_yaml_variable(self): + path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" + yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" + + with open(file=yaml_file_path, mode="r") as io: + yaml_lines = io.readlines() + + yaml_string = "".join(yaml_lines) + os.environ["NEUROCONV_YAML"] = yaml_string + os.environ["NEUROCONV_DATA_PATH"] = DATA_PATH + os.environ["NEUROCONV_OUTPUT_PATH"] = OUTPUT_PATH + + deploy_process( + command=( + "docker run -t " + f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" + ) + ) + + nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") + assert nwbfile.subject.subject_id == "1" + assert "ElectricalSeriesAP" in nwbfile.acquisition + + nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") + assert nwbfile.subject.subject_id == "002" + + nwbfile_path = self.test_folder / "example_converter_spec_3.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Auto-generated by neuroconv" + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") + assert nwbfile.subject.subject_id == "Subject Name" + assert "spike_times" in nwbfile.units From 60bf1d57d2ec064302ed34d872ac8a6475ae8725 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 22:05:04 +0000 Subject: [PATCH 081/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ...ocker_yaml_conversion_specification_cli.py | 92 +++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index fb2405758..b7d2ddb2b 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -124,53 +124,53 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.subject.subject_id == "Subject Name" assert "spike_times" in nwbfile.units -def test_run_conversion_from_yaml_variable(self): - path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" - yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" - with open(file=yaml_file_path, mode="r") as io: - yaml_lines = io.readlines() - - yaml_string = "".join(yaml_lines) - os.environ["NEUROCONV_YAML"] = yaml_string - os.environ["NEUROCONV_DATA_PATH"] = DATA_PATH - os.environ["NEUROCONV_OUTPUT_PATH"] = OUTPUT_PATH - - deploy_process( - command=( - "docker run -t " - f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" - ) - ) +def test_run_conversion_from_yaml_variable(self): + path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" + yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" - nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Subject navigating a Y-shaped maze." - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") - assert nwbfile.subject.subject_id == "1" - assert "ElectricalSeriesAP" in nwbfile.acquisition + with open(file=yaml_file_path, mode="r") as io: + yaml_lines = io.readlines() - nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Subject navigating a Y-shaped maze." - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") - assert nwbfile.subject.subject_id == "002" + yaml_string = "".join(yaml_lines) + os.environ["NEUROCONV_YAML"] = yaml_string + os.environ["NEUROCONV_DATA_PATH"] = DATA_PATH + os.environ["NEUROCONV_OUTPUT_PATH"] = OUTPUT_PATH - nwbfile_path = self.test_folder / "example_converter_spec_3.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Auto-generated by neuroconv" - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") - assert nwbfile.subject.subject_id == "Subject Name" - assert "spike_times" in nwbfile.units + deploy_process( + command=( + "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" + ) + ) + + nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") + assert nwbfile.subject.subject_id == "1" + assert "ElectricalSeriesAP" in nwbfile.acquisition + + nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") + assert nwbfile.subject.subject_id == "002" + + nwbfile_path = self.test_folder / "example_converter_spec_3.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Auto-generated by neuroconv" + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") + assert nwbfile.subject.subject_id == "Subject Name" + assert "spike_times" in nwbfile.units From 5589e2fc52e7101a81882d8f049ba1e4e661f0da Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:06:02 -0400 Subject: [PATCH 082/119] update dockerfile with flags --- dockerfiles/neuroconv_latest_yaml_variable | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_latest_yaml_variable b/dockerfiles/neuroconv_latest_yaml_variable index 98688d586..dffad95b4 100644 --- a/dockerfiles/neuroconv_latest_yaml_variable +++ b/dockerfiles/neuroconv_latest_yaml_variable @@ -1,3 +1,3 @@ FROM https://github.com/catalystneuro/neuroconv:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -CMD echo "$NEUROCONV_YAML" > run.yml && neuroconv run.yml +CMD echo "$NEUROCONV_YAML" > run.yml && neuroconv run.yml --data-folder-path $NEUROCONV_DATA_PATH --output-folder-path $NEUROCONV_OUTPUT_PATH From 32c18f80f4130e89381e17158812e9341d4ac49a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:09:14 -0400 Subject: [PATCH 083/119] update dev docs --- docs/developer_guide/docker_images.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index 018a1c272..d78b3cfd0 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -69,7 +69,7 @@ and can then run the entrypoint (equivalent to the usual command line usage) on .. code:: - docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /desired/alias/of/drive/your_specification_file.yml + docker run -it --volume /your/local/volume/:/desired/alias/of/volume/ ghcr.io/catalystneuro/neuroconv:latest neuroconv /desired/alias/of/drive/your_specification_file.yml @@ -82,13 +82,15 @@ The only potential downside with this usage is the maximum size of an environmen Otherwise, in any cloud deployment, the YAML file transfer will have to be managed separately, likely as a part of the data transfer or an entirely separate step. -To use this alternative image on a local environment, you no longer need to invoke the ``neuroconv`` entrypoint pointing to a file. Instead, just set the environment variable and run the docker container on the mounted volume... +To use this alternative image on a local environment, you no longer need to invoke the ``neuroconv`` entrypoint pointing to a file. Instead, just set the environment variables and run the docker container on the mounted volume... .. code:: - export YAML_STREAM="" # On Windows, use `set` instead of `export` - docker run -it --volume /your/local/drive/:/desired/alias/of/drive/ ghcr.io/catalystneuro/neuroconv:yaml_variable + export YAML_STREAM="" + export NEUROCONV_DATA_PATH="/desired/alias/of/volume/" + export NEUROCONV_OUTPUT_PATH="/desired/alias/of/volume/" + docker run -it --volume /your/local/volume/:/desired/alias/of/volume/ ghcr.io/catalystneuro/neuroconv:yaml_variable .. note:: - When using YAML files through the docker containers, always be sure that the file and folder paths are absolute stemming from the base volume as mounted inside the container; otherwise, the NWB file will be written inside pf the container but will then not be accessible outside of it. + On Windows, use ``set`` instead of ``export``. From 7f4f5d1cf8f8df0a9c8633e9dfde060072ea8ade Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:10:06 -0400 Subject: [PATCH 084/119] remove push --- .../workflows/build_and_upload_docker_image_yaml_variable.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml index 110c901a5..768dd0c30 100644 --- a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml +++ b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml @@ -5,7 +5,6 @@ on: workflows: [build_and_upload_docker_image_latest_release] types: [completed] workflow_dispatch: - push: # remove after merge; can't trigger manually until on main concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} From 1a3771d2cd2d3c0af24fb3b46b7c73067b3338be Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:12:13 -0400 Subject: [PATCH 085/119] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1720eb930..53e065be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Upcoming +### Features + +* Released the first official Docker images for the package on the GitHub Container Repository (GHCR). [PR #383](https://github.com/catalystneuro/neuroconv/pull/383) + + + # v0.4.8 (March 20, 2024) ### Bug fixes From 76017245ba29aa98a2ff753ba78bf154dcbf3cea Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:44:09 -0400 Subject: [PATCH 086/119] bypass git, just do direct local install --- dockerfiles/neuroconv_dev_dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dockerfiles/neuroconv_dev_dockerfile b/dockerfiles/neuroconv_dev_dockerfile index c92ded8b3..175c2e21a 100644 --- a/dockerfiles/neuroconv_dev_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -1,5 +1,5 @@ -FROM continuumio/anaconda3:latest +FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -RUN conda create -n neuroconv python=3.11.7 -RUN conda activate neuroconv && /bin/sh -c conda install git && python -m pip install git+https://github.com/catalystneuro/neuroconv.git@main +ADD /neuroconv /neuroconv +RUN cd neuroconv && pip install .[full] CMD ["python -m"] From f3dee1b3c0057082358b58ad9fdacdd4c1844f5c Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:45:19 -0400 Subject: [PATCH 087/119] fix scope --- ...ocker_yaml_conversion_specification_cli.py | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index b7d2ddb2b..9d2867be9 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -125,52 +125,52 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert "spike_times" in nwbfile.units -def test_run_conversion_from_yaml_variable(self): - path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" - yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" - - with open(file=yaml_file_path, mode="r") as io: - yaml_lines = io.readlines() - - yaml_string = "".join(yaml_lines) - os.environ["NEUROCONV_YAML"] = yaml_string - os.environ["NEUROCONV_DATA_PATH"] = DATA_PATH - os.environ["NEUROCONV_OUTPUT_PATH"] = OUTPUT_PATH - - deploy_process( - command=( - "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" + def test_run_conversion_from_yaml_variable(self): + path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" + yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" + + with open(file=yaml_file_path, mode="r") as io: + yaml_lines = io.readlines() + + yaml_string = "".join(yaml_lines) + os.environ["NEUROCONV_YAML"] = yaml_string + os.environ["NEUROCONV_DATA_PATH"] = DATA_PATH + os.environ["NEUROCONV_OUTPUT_PATH"] = OUTPUT_PATH + + deploy_process( + command=( + "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" + ) ) - ) - - nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Subject navigating a Y-shaped maze." - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") - assert nwbfile.subject.subject_id == "1" - assert "ElectricalSeriesAP" in nwbfile.acquisition - - nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Subject navigating a Y-shaped maze." - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") - assert nwbfile.subject.subject_id == "002" - - nwbfile_path = self.test_folder / "example_converter_spec_3.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Auto-generated by neuroconv" - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") - assert nwbfile.subject.subject_id == "Subject Name" - assert "spike_times" in nwbfile.units + + nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") + assert nwbfile.subject.subject_id == "1" + assert "ElectricalSeriesAP" in nwbfile.acquisition + + nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") + assert nwbfile.subject.subject_id == "002" + + nwbfile_path = self.test_folder / "example_converter_spec_3.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Auto-generated by neuroconv" + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") + assert nwbfile.subject.subject_id == "Subject Name" + assert "spike_times" in nwbfile.units From 8278e577dd143ae88ea206f353c4efbb11419b6a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 22:45:38 +0000 Subject: [PATCH 088/119] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../docker_yaml_conversion_specification_cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index 9d2867be9..fa68da000 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -124,25 +124,25 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.subject.subject_id == "Subject Name" assert "spike_times" in nwbfile.units - def test_run_conversion_from_yaml_variable(self): path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" - + with open(file=yaml_file_path, mode="r") as io: yaml_lines = io.readlines() - + yaml_string = "".join(yaml_lines) os.environ["NEUROCONV_YAML"] = yaml_string os.environ["NEUROCONV_DATA_PATH"] = DATA_PATH os.environ["NEUROCONV_OUTPUT_PATH"] = OUTPUT_PATH - + deploy_process( command=( - "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" + "docker run -t " + f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" ) ) - + nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " with NWBHDF5IO(path=nwbfile_path, mode="r") as io: @@ -153,7 +153,7 @@ def test_run_conversion_from_yaml_variable(self): assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "1" assert "ElectricalSeriesAP" in nwbfile.acquisition - + nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " with NWBHDF5IO(path=nwbfile_path, mode="r") as io: @@ -163,7 +163,7 @@ def test_run_conversion_from_yaml_variable(self): assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") assert nwbfile.subject.subject_id == "002" - + nwbfile_path = self.test_folder / "example_converter_spec_3.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " with NWBHDF5IO(path=nwbfile_path, mode="r") as io: From 3bc6b303fca6ee23b4acb9e241e1383a687ae875 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:46:32 -0400 Subject: [PATCH 089/119] Update docker_testing.yml --- .github/workflows/docker_testing.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/docker_testing.yml b/.github/workflows/docker_testing.yml index 7bfc866fe..075d1b81f 100644 --- a/.github/workflows/docker_testing.yml +++ b/.github/workflows/docker_testing.yml @@ -2,18 +2,8 @@ name: Docker CLI tests on: schedule: - cron: "0 16 * * *" # Daily at noon EST - workflow_call: - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - S3_GIN_BUCKET: - required: true - CODECOV_TOKEN: - required: true workflow_dispatch: - push: # on first commit + push: # remove once on main jobs: run: From 03e5fff16fd8809d05b8e653b2368357f47bb219 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:55:26 -0400 Subject: [PATCH 090/119] Update neuroconv_dev_dockerfile --- dockerfiles/neuroconv_dev_dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_dev_dockerfile b/dockerfiles/neuroconv_dev_dockerfile index 175c2e21a..0be5d7743 100644 --- a/dockerfiles/neuroconv_dev_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -1,5 +1,5 @@ FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -ADD /neuroconv /neuroconv +ADD neuroconv neuroconv RUN cd neuroconv && pip install .[full] CMD ["python -m"] From f20acba43ed0c06344bb3486f51725854145d0f9 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:01:38 -0400 Subject: [PATCH 091/119] Update build_and_upload_docker_image_dev.yml --- .github/workflows/build_and_upload_docker_image_dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index 531a236cd..f5b0966b1 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -27,6 +27,8 @@ jobs: registry: ghcr.io username: ${{ secrets.DOCKER_UPLOADER_USERNAME }} password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} + - name: test + run: pwd - name: Build and push uses: docker/build-push-action@v5 with: From 58990b7856ef139d6fe10618f287a5b0a156f309 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:03:15 -0400 Subject: [PATCH 092/119] Update neuroconv_dev_dockerfile --- dockerfiles/neuroconv_dev_dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_dev_dockerfile b/dockerfiles/neuroconv_dev_dockerfile index 0be5d7743..646ad2a78 100644 --- a/dockerfiles/neuroconv_dev_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -1,5 +1,5 @@ FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -ADD neuroconv neuroconv +ADD ./ neuroconv RUN cd neuroconv && pip install .[full] CMD ["python -m"] From 2423f28953498d733de1605fe7ca8ca82b22f5a5 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:06:56 -0400 Subject: [PATCH 093/119] debugs to tests --- tests/docker_yaml_conversion_specification_cli.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index fa68da000..e8b2c76ac 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -24,12 +24,13 @@ class TestLatestDockerYAMLConversionSpecification(TestCase): source_volume = os.getenv("NEUROCONV_DOCKER_TESTS_SOURCE_VOLUME", "/home/runner/work/neuroconv") def test_run_conversion_from_yaml_cli(self): - path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" + path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "conversion_specifications" yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" + deploy_process( command=( "docker run -t " - f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag}" + f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag} " f"neuroconv /neuroconv/tests/test_on_data/conversion_specifications/{yaml_file_path} " f"--data-folder-path /neuroconv/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ) @@ -70,12 +71,14 @@ def test_run_conversion_from_yaml_cli(self): def test_run_conversion_from_yaml_default_nwbfile_name(self): self.test_folder = self.test_folder / "test_organize" self.test_folder.mkdir(exist_ok=True) - path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" + + path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "conversion_specifications" yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification_missing_nwbfile_names.yml" + deploy_process( command=( "docker run -t " - f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag}" + f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag} " f"neuroconv {yaml_file_path} " f"--data-folder-path {DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ) @@ -125,7 +128,7 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert "spike_times" in nwbfile.units def test_run_conversion_from_yaml_variable(self): - path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" + path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "conversion_specifications" yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" with open(file=yaml_file_path, mode="r") as io: From 339ee36343995ef2089eee5b30c2687a62b94cf5 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:07:23 -0400 Subject: [PATCH 094/119] Update build_and_upload_docker_image_dev.yml --- .github/workflows/build_and_upload_docker_image_dev.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index f5b0966b1..531a236cd 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -27,8 +27,6 @@ jobs: registry: ghcr.io username: ${{ secrets.DOCKER_UPLOADER_USERNAME }} password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} - - name: test - run: pwd - name: Build and push uses: docker/build-push-action@v5 with: From b18465deac8e0d461ff2baee8e7a2a10b372e0ba Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:11:54 -0400 Subject: [PATCH 095/119] debug environment variable paths --- tests/docker_yaml_conversion_specification_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index e8b2c76ac..d9d85fd58 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -136,8 +136,8 @@ def test_run_conversion_from_yaml_variable(self): yaml_string = "".join(yaml_lines) os.environ["NEUROCONV_YAML"] = yaml_string - os.environ["NEUROCONV_DATA_PATH"] = DATA_PATH - os.environ["NEUROCONV_OUTPUT_PATH"] = OUTPUT_PATH + os.environ["NEUROCONV_DATA_PATH"] = str(DATA_PATH) + os.environ["NEUROCONV_OUTPUT_PATH"] = str(OUTPUT_PATH) deploy_process( command=( From 843049d2fd952a7ec2c51547a6ba99716b2ffde6 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:17:45 -0400 Subject: [PATCH 096/119] printout docker results; remove one test case --- ...ocker_yaml_conversion_specification_cli.py | 73 +++---------------- 1 file changed, 9 insertions(+), 64 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index d9d85fd58..3f2e6d2bc 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -27,14 +27,16 @@ def test_run_conversion_from_yaml_cli(self): path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "conversion_specifications" yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" - deploy_process( + output = deploy_process( command=( "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag} " f"neuroconv /neuroconv/tests/test_on_data/conversion_specifications/{yaml_file_path} " f"--data-folder-path /neuroconv/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" - ) + ), + catch_output=True, ) + print(output) nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " @@ -68,65 +70,6 @@ def test_run_conversion_from_yaml_cli(self): assert nwbfile.subject.subject_id == "Subject Name" assert "spike_times" in nwbfile.units - def test_run_conversion_from_yaml_default_nwbfile_name(self): - self.test_folder = self.test_folder / "test_organize" - self.test_folder.mkdir(exist_ok=True) - - path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "conversion_specifications" - yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification_missing_nwbfile_names.yml" - - deploy_process( - command=( - "docker run -t " - f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag} " - f"neuroconv {yaml_file_path} " - f"--data-folder-path {DATA_PATH} --output-folder-path {self.test_folder} --overwrite" - ) - ) - - nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201009T211909.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Subject navigating a Y-shaped maze." - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") - assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAP" in nwbfile.acquisition - - nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Subject navigating a Y-shaped maze." - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00") - assert nwbfile.subject.subject_id == "Mouse 1" - assert "ElectricalSeriesAP" in nwbfile.acquisition - - nwbfile_path = self.test_folder / "example_defined_name.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Subject navigating a Y-shaped maze." - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") - assert nwbfile.subject.subject_id == "MyMouse002" - - nwbfile_path = self.test_folder / "sub-Subject-Name_ses-20201011T211909.nwb" - assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - assert nwbfile.session_description == "Auto-generated by neuroconv" - assert nwbfile.lab == "My Lab" - assert nwbfile.institution == "My Institution" - assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") - assert nwbfile.subject.subject_id == "Subject Name" - assert "spike_times" in nwbfile.units - def test_run_conversion_from_yaml_variable(self): path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "conversion_specifications" yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" @@ -137,14 +80,16 @@ def test_run_conversion_from_yaml_variable(self): yaml_string = "".join(yaml_lines) os.environ["NEUROCONV_YAML"] = yaml_string os.environ["NEUROCONV_DATA_PATH"] = str(DATA_PATH) - os.environ["NEUROCONV_OUTPUT_PATH"] = str(OUTPUT_PATH) + os.environ["NEUROCONV_OUTPUT_PATH"] = str(self.test_folder) - deploy_process( + output = deploy_process( command=( "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" - ) + ), + catch_output=True, ) + print(output) nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " From f50ffab3cd71f1e28a55a57cffa55cb48fc31d47 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:19:14 -0400 Subject: [PATCH 097/119] correct GHCR base --- dockerfiles/neuroconv_latest_yaml_variable | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_latest_yaml_variable b/dockerfiles/neuroconv_latest_yaml_variable index dffad95b4..9487480e6 100644 --- a/dockerfiles/neuroconv_latest_yaml_variable +++ b/dockerfiles/neuroconv_latest_yaml_variable @@ -1,3 +1,3 @@ -FROM https://github.com/catalystneuro/neuroconv:latest +FROM ghcr.io/catalystneuro/neuroconv:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv CMD echo "$NEUROCONV_YAML" > run.yml && neuroconv run.yml --data-folder-path $NEUROCONV_DATA_PATH --output-folder-path $NEUROCONV_OUTPUT_PATH From bce202d6a5481a111b3eb644d16d42451bc65554 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:19:37 -0400 Subject: [PATCH 098/119] force rebuild --- .../workflows/build_and_upload_docker_image_yaml_variable.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml index 768dd0c30..daaf485f4 100644 --- a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml +++ b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml @@ -5,6 +5,7 @@ on: workflows: [build_and_upload_docker_image_latest_release] types: [completed] workflow_dispatch: + push: # TODO: remove when once main concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} From 8172be53aac4a794c96466e23e4cc2c8ce671b29 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:21:47 -0400 Subject: [PATCH 099/119] restore login --- .../build_and_upload_docker_image_yaml_variable.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml index daaf485f4..47e4c6879 100644 --- a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml +++ b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml @@ -22,6 +22,12 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ secrets.DOCKER_UPLOADER_USERNAME }} + password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} - name: Build and push YAML variable image based on latest uses: docker/build-push-action@v5 with: From e4666d04d67baa2464d0c9f3c5c9dfa642be6e6f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:53:51 -0400 Subject: [PATCH 100/119] debugging --- tests/docker_yaml_conversion_specification_cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index 3f2e6d2bc..b5348bf70 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -31,7 +31,7 @@ def test_run_conversion_from_yaml_cli(self): command=( "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag} " - f"neuroconv /neuroconv/tests/test_on_data/conversion_specifications/{yaml_file_path} " + f"neuroconv {yaml_file_path} " f"--data-folder-path /neuroconv/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ), catch_output=True, @@ -86,6 +86,9 @@ def test_run_conversion_from_yaml_variable(self): command=( "docker run -t " f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" + "-e NEUROCONV_YAML=$NEUROCONV_YAML" + "-e NEUROCONV_DATA_PATH=$NEUROCONV_DATA_PATH" + "-e NEUROCONV_OUTPUT_PATH=$NEUROCONV_OUTPUT_PATH" ), catch_output=True, ) From 61ccba51bd4cb15bc064a62dd258ee5b80149e6a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 13:05:11 -0400 Subject: [PATCH 101/119] debugging --- tests/docker_yaml_conversion_specification_cli.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index b5348bf70..bd2739b66 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -30,7 +30,8 @@ def test_run_conversion_from_yaml_cli(self): output = deploy_process( command=( "docker run -t " - f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:{self.tag} " + f"--volume {self.source_volume}:{self.source_volume} " + "ghcr.io/catalystneuro/neuroconv:{self.tag} " f"neuroconv {yaml_file_path} " f"--data-folder-path /neuroconv/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ), @@ -85,10 +86,11 @@ def test_run_conversion_from_yaml_variable(self): output = deploy_process( command=( "docker run -t " - f"--volume {self.source_volume}:/neuroconv ghcr.io/catalystneuro/neuroconv:yaml_variable" - "-e NEUROCONV_YAML=$NEUROCONV_YAML" - "-e NEUROCONV_DATA_PATH=$NEUROCONV_DATA_PATH" - "-e NEUROCONV_OUTPUT_PATH=$NEUROCONV_OUTPUT_PATH" + f"--volume {self.source_volume}:/neuroconv " + "--env NEUROCONV_YAML=$NEUROCONV_YAML " + "--env NEUROCONV_DATA_PATH=$NEUROCONV_DATA_PATH " + "--env NEUROCONV_OUTPUT_PATH=$NEUROCONV_OUTPUT_PATH " + "ghcr.io/catalystneuro/neuroconv:yaml_variable" ), catch_output=True, ) From a7a91a137634a869c9080b3fd8145fdeb1395f08 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 13:23:34 -0400 Subject: [PATCH 102/119] debugging --- tests/docker_yaml_conversion_specification_cli.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index bd2739b66..03cad8838 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -31,9 +31,10 @@ def test_run_conversion_from_yaml_cli(self): command=( "docker run -t " f"--volume {self.source_volume}:{self.source_volume} " + f"--volume {self.test_folder}:{self.test_folder} " "ghcr.io/catalystneuro/neuroconv:{self.tag} " f"neuroconv {yaml_file_path} " - f"--data-folder-path /neuroconv/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" + f"--data-folder-path {self.source_volume}/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ), catch_output=True, ) @@ -80,16 +81,17 @@ def test_run_conversion_from_yaml_variable(self): yaml_string = "".join(yaml_lines) os.environ["NEUROCONV_YAML"] = yaml_string - os.environ["NEUROCONV_DATA_PATH"] = str(DATA_PATH) + os.environ["NEUROCONV_DATA_PATH"] = self.source_volume + str(DATA_PATH) os.environ["NEUROCONV_OUTPUT_PATH"] = str(self.test_folder) output = deploy_process( command=( "docker run -t " - f"--volume {self.source_volume}:/neuroconv " - "--env NEUROCONV_YAML=$NEUROCONV_YAML " - "--env NEUROCONV_DATA_PATH=$NEUROCONV_DATA_PATH " - "--env NEUROCONV_OUTPUT_PATH=$NEUROCONV_OUTPUT_PATH " + f"--volume {self.source_volume}:{self.source_volume} " + f"--volume {self.test_folder}:{self.test_folder} " + "-e NEUROCONV_YAML=$NEUROCONV_YAML " + "-e NEUROCONV_DATA_PATH=$NEUROCONV_DATA_PATH " + "-e NEUROCONV_OUTPUT_PATH=$NEUROCONV_OUTPUT_PATH " "ghcr.io/catalystneuro/neuroconv:yaml_variable" ), catch_output=True, From 36d52f460ab29b3ef8dcef75f243d22b6b99b840 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:31:31 -0400 Subject: [PATCH 103/119] debugging --- tests/docker_yaml_conversion_specification_cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index 03cad8838..ca3b67db3 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -29,11 +29,11 @@ def test_run_conversion_from_yaml_cli(self): output = deploy_process( command=( - "docker run -t " + "docker run " f"--volume {self.source_volume}:{self.source_volume} " f"--volume {self.test_folder}:{self.test_folder} " "ghcr.io/catalystneuro/neuroconv:{self.tag} " - f"neuroconv {yaml_file_path} " + f"-t neuroconv {yaml_file_path} " f"--data-folder-path {self.source_volume}/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ), catch_output=True, @@ -89,9 +89,9 @@ def test_run_conversion_from_yaml_variable(self): "docker run -t " f"--volume {self.source_volume}:{self.source_volume} " f"--volume {self.test_folder}:{self.test_folder} " - "-e NEUROCONV_YAML=$NEUROCONV_YAML " - "-e NEUROCONV_DATA_PATH=$NEUROCONV_DATA_PATH " - "-e NEUROCONV_OUTPUT_PATH=$NEUROCONV_OUTPUT_PATH " + '-e NEUROCONV_YAML="$NEUROCONV_YAML" ' + '-e NEUROCONV_DATA_PATH="${NEUROCONV_DATA_PATH" ' + '-e NEUROCONV_OUTPUT_PATH="$NEUROCONV_OUTPUT_PATH" ' "ghcr.io/catalystneuro/neuroconv:yaml_variable" ), catch_output=True, From 008d44828bc1b48a8ff64da2957d822561fe31ce Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:41:19 -0400 Subject: [PATCH 104/119] debugging --- tests/docker_yaml_conversion_specification_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index ca3b67db3..12e6e4d18 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -90,7 +90,7 @@ def test_run_conversion_from_yaml_variable(self): f"--volume {self.source_volume}:{self.source_volume} " f"--volume {self.test_folder}:{self.test_folder} " '-e NEUROCONV_YAML="$NEUROCONV_YAML" ' - '-e NEUROCONV_DATA_PATH="${NEUROCONV_DATA_PATH" ' + '-e NEUROCONV_DATA_PATH="$NEUROCONV_DATA_PATH" ' '-e NEUROCONV_OUTPUT_PATH="$NEUROCONV_OUTPUT_PATH" ' "ghcr.io/catalystneuro/neuroconv:yaml_variable" ), From 45aafeed73a0ebc7746c0deec69eac16b9258169 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:42:36 -0400 Subject: [PATCH 105/119] try to pull images first --- .github/workflows/docker_testing.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/docker_testing.yml b/.github/workflows/docker_testing.yml index 075d1b81f..54f960fc5 100644 --- a/.github/workflows/docker_testing.yml +++ b/.github/workflows/docker_testing.yml @@ -77,5 +77,10 @@ jobs: name: Download behavior dataset from S3 run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/behavior_testing_data ./behavior_testing_data + - name: Pull docker image + run: | + docker pull ghcr.io/catalystneuro/neuroconv:latest + docker pull ghcr.io/catalystneuro/neuroconv:yaml_variable + - name: Run docker tests run: pytest tests/docker_yaml_conversion_specification_cli.py -vv -rsx From 0aa5d28bd69a1324158246c9d560ed1054c12002 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:42:57 -0400 Subject: [PATCH 106/119] already was apparently --- .github/workflows/docker_testing.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/docker_testing.yml b/.github/workflows/docker_testing.yml index 54f960fc5..7ab461c57 100644 --- a/.github/workflows/docker_testing.yml +++ b/.github/workflows/docker_testing.yml @@ -30,9 +30,6 @@ jobs: pip install pytest pip install . - - name: Pull docker image - run: docker pull ghcr.io/catalystneuro/neuroconv:latest - - name: Get ephy_testing_data current head hash id: ephys run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" From 38206fc4ea8da4ed830ad5b0bc49b52841af7175 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:49:12 -0400 Subject: [PATCH 107/119] add missing quotes --- dockerfiles/neuroconv_latest_yaml_variable | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_latest_yaml_variable b/dockerfiles/neuroconv_latest_yaml_variable index 9487480e6..09434373c 100644 --- a/dockerfiles/neuroconv_latest_yaml_variable +++ b/dockerfiles/neuroconv_latest_yaml_variable @@ -1,3 +1,3 @@ FROM ghcr.io/catalystneuro/neuroconv:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -CMD echo "$NEUROCONV_YAML" > run.yml && neuroconv run.yml --data-folder-path $NEUROCONV_DATA_PATH --output-folder-path $NEUROCONV_OUTPUT_PATH +CMD echo "$NEUROCONV_YAML" > run.yml && python -m neuroconv run.yml --data-folder-path "$NEUROCONV_DATA_PATH" --output-folder-path "$NEUROCONV_OUTPUT_PATH" From abc7a24717313cad32d7f74ead2e2ed9e0ff31f1 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:51:11 -0400 Subject: [PATCH 108/119] add missing f-string --- tests/docker_yaml_conversion_specification_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index 12e6e4d18..312b9ee70 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -29,11 +29,11 @@ def test_run_conversion_from_yaml_cli(self): output = deploy_process( command=( - "docker run " + "docker run -t " f"--volume {self.source_volume}:{self.source_volume} " f"--volume {self.test_folder}:{self.test_folder} " - "ghcr.io/catalystneuro/neuroconv:{self.tag} " - f"-t neuroconv {yaml_file_path} " + f"ghcr.io/catalystneuro/neuroconv:{self.tag} " + f"neuroconv {yaml_file_path} " f"--data-folder-path {self.source_volume}/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" ), catch_output=True, From 9ce262a73e59f18942de2e0a487c9b77c7328295 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:57:45 -0400 Subject: [PATCH 109/119] add description --- dockerfiles/neuroconv_dev_dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfiles/neuroconv_dev_dockerfile b/dockerfiles/neuroconv_dev_dockerfile index 646ad2a78..0de15ac74 100644 --- a/dockerfiles/neuroconv_dev_dockerfile +++ b/dockerfiles/neuroconv_dev_dockerfile @@ -1,5 +1,6 @@ FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv +LABEL org.opencontainers.image.description="A docker image for the most recent daily build of the main branch." ADD ./ neuroconv RUN cd neuroconv && pip install .[full] CMD ["python -m"] From a6ab69730409cf48fbca48d327c03cc95bb94e09 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:58:15 -0400 Subject: [PATCH 110/119] add description --- dockerfiles/neuroconv_latest_release_dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfiles/neuroconv_latest_release_dockerfile b/dockerfiles/neuroconv_latest_release_dockerfile index 7a534b3ff..918b64453 100644 --- a/dockerfiles/neuroconv_latest_release_dockerfile +++ b/dockerfiles/neuroconv_latest_release_dockerfile @@ -1,5 +1,6 @@ FROM python:3.11.7-slim LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv +LABEL org.opencontainers.image.description="A docker image for the most recent official release of the NeuroConv package." RUN apt update && apt install musl-dev python3-dev -y RUN pip install neuroconv[full] CMD ["python -m"] From 29cbcc52b119f3a072a35d283c1ef122f0ee895c Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:59:26 -0400 Subject: [PATCH 111/119] add description --- dockerfiles/neuroconv_latest_yaml_variable | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_latest_yaml_variable b/dockerfiles/neuroconv_latest_yaml_variable index 09434373c..ea411ee44 100644 --- a/dockerfiles/neuroconv_latest_yaml_variable +++ b/dockerfiles/neuroconv_latest_yaml_variable @@ -1,3 +1,4 @@ FROM ghcr.io/catalystneuro/neuroconv:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv -CMD echo "$NEUROCONV_YAML" > run.yml && python -m neuroconv run.yml --data-folder-path "$NEUROCONV_DATA_PATH" --output-folder-path "$NEUROCONV_OUTPUT_PATH" +LABEL org.opencontainers.image.description="A docker image for the most recent official release of the NeuroConv package. Modified to take in environment variables for the YAML conversion specification and other command line arguments." +CMD echo "$NEUROCONV_YAML" > run.yml && python -m neuroconv run.yml --data-folder-path "$NEUROCONV_DATA_PATH" --output-folder-path "$NEUROCONV_OUTPUT_PATH" --overwrite From 87704f3b39a9542834593035f1fb22afcaaeb90f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 22 Mar 2024 15:08:37 -0400 Subject: [PATCH 112/119] try this --- tests/docker_yaml_conversion_specification_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index 312b9ee70..dd5da8d17 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -21,7 +21,7 @@ class TestLatestDockerYAMLConversionSpecification(TestCase): test_folder = OUTPUT_PATH tag = os.getenv("NEUROCONV_DOCKER_TESTS_TAG", "latest") - source_volume = os.getenv("NEUROCONV_DOCKER_TESTS_SOURCE_VOLUME", "/home/runner/work/neuroconv") + source_volume = os.getenv("NEUROCONV_DOCKER_TESTS_SOURCE_VOLUME", "/home/runner/work/neuroconv/neuroconv") def test_run_conversion_from_yaml_cli(self): path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "conversion_specifications" From 36ce2c063712dad03b6fb3ba461a2f38c012cbdf Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 22 Mar 2024 15:34:13 -0400 Subject: [PATCH 113/119] remove pushes; add date tag to dev --- .github/workflows/build_and_upload_docker_image_dev.yml | 7 ++++++- .../build_and_upload_docker_image_latest_release.yml | 1 - .../build_and_upload_docker_image_yaml_variable.yml | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index 531a236cd..487a44f08 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -27,10 +27,15 @@ jobs: registry: ghcr.io username: ${{ secrets.DOCKER_UPLOADER_USERNAME }} password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} + - name: Get current date + id: date + run: | + date_tag="$(date +'%Y-%m-%d')" + echo "date_tag=$date_tag" >> $GITHUB_OUTPUT - name: Build and push uses: docker/build-push-action@v5 with: push: true # Push is a shorthand for --output=type=registry - tags: ghcr.io/catalystneuro/neuroconv:dev + tags: ghcr.io/catalystneuro/neuroconv:dev,ghcr.io/catalystneuro/neuroconv:${{ steps.date.outputs.date_tag }} context: . file: dockerfiles/neuroconv_dev_dockerfile diff --git a/.github/workflows/build_and_upload_docker_image_latest_release.yml b/.github/workflows/build_and_upload_docker_image_latest_release.yml index 47d138bb0..79e788ea3 100644 --- a/.github/workflows/build_and_upload_docker_image_latest_release.yml +++ b/.github/workflows/build_and_upload_docker_image_latest_release.yml @@ -5,7 +5,6 @@ on: workflows: [auto-publish] types: [completed] workflow_dispatch: - push: # remove after merge; can't trigger manually until on main concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml index 47e4c6879..92c37576d 100644 --- a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml +++ b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml @@ -5,7 +5,6 @@ on: workflows: [build_and_upload_docker_image_latest_release] types: [completed] workflow_dispatch: - push: # TODO: remove when once main concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} From 0e246b8a1ecb1b04dbb5670dfe27cd398a568225 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 22 Mar 2024 15:35:07 -0400 Subject: [PATCH 114/119] increase dev to weekly --- .github/workflows/build_and_upload_docker_image_dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index 487a44f08..bc057ac84 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -2,7 +2,7 @@ name: Build and Upload Docker Image of Current Dev Branch to GHCR on: schedule: - - cron: "0 16 * * *" # Daily at noon EST + - cron: "0 16 * * 1" # Weekly at noon EST on Monday workflow_dispatch: push: # remove after merge; can't trigger manually until on main From 8f5bdbcd3669103ba9fdd8591133c7cd3d1e08a3 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 22 Mar 2024 15:40:16 -0400 Subject: [PATCH 115/119] one last rebuild --- .../workflows/build_and_upload_docker_image_latest_release.yml | 1 + .../workflows/build_and_upload_docker_image_yaml_variable.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build_and_upload_docker_image_latest_release.yml b/.github/workflows/build_and_upload_docker_image_latest_release.yml index 79e788ea3..47d138bb0 100644 --- a/.github/workflows/build_and_upload_docker_image_latest_release.yml +++ b/.github/workflows/build_and_upload_docker_image_latest_release.yml @@ -5,6 +5,7 @@ on: workflows: [auto-publish] types: [completed] workflow_dispatch: + push: # remove after merge; can't trigger manually until on main concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml index 92c37576d..adca9206d 100644 --- a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml +++ b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml @@ -5,6 +5,7 @@ on: workflows: [build_and_upload_docker_image_latest_release] types: [completed] workflow_dispatch: + #push: # remove after merge; can't trigger manually until on main concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} From 1669604d724e30914fb23f4d45198e47be6dddb9 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 22 Mar 2024 15:46:56 -0400 Subject: [PATCH 116/119] try without provenance --- .github/workflows/build_and_upload_docker_image_dev.yml | 1 - .../workflows/build_and_upload_docker_image_latest_release.yml | 1 - .../workflows/build_and_upload_docker_image_yaml_variable.yml | 3 ++- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index bc057ac84..3bc0c083c 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -4,7 +4,6 @@ on: schedule: - cron: "0 16 * * 1" # Weekly at noon EST on Monday workflow_dispatch: - push: # remove after merge; can't trigger manually until on main concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/build_and_upload_docker_image_latest_release.yml b/.github/workflows/build_and_upload_docker_image_latest_release.yml index 47d138bb0..79e788ea3 100644 --- a/.github/workflows/build_and_upload_docker_image_latest_release.yml +++ b/.github/workflows/build_and_upload_docker_image_latest_release.yml @@ -5,7 +5,6 @@ on: workflows: [auto-publish] types: [completed] workflow_dispatch: - push: # remove after merge; can't trigger manually until on main concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml index adca9206d..507145bb1 100644 --- a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml +++ b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml @@ -5,7 +5,7 @@ on: workflows: [build_and_upload_docker_image_latest_release] types: [completed] workflow_dispatch: - #push: # remove after merge; can't trigger manually until on main + push: concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} @@ -35,3 +35,4 @@ jobs: tags: ghcr.io/catalystneuro/neuroconv:yaml_variable context: . file: dockerfiles/neuroconv_latest_yaml_variable + provenance: false From d7858bd1a104a2d733cda9674cc4a2fcb4d078e2 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 22 Mar 2024 15:47:20 -0400 Subject: [PATCH 117/119] try without provenance --- .github/workflows/build_and_upload_docker_image_dev.yml | 2 ++ .../workflows/build_and_upload_docker_image_latest_release.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index 3bc0c083c..4bae5f96e 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -4,6 +4,7 @@ on: schedule: - cron: "0 16 * * 1" # Weekly at noon EST on Monday workflow_dispatch: + push: concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} @@ -38,3 +39,4 @@ jobs: tags: ghcr.io/catalystneuro/neuroconv:dev,ghcr.io/catalystneuro/neuroconv:${{ steps.date.outputs.date_tag }} context: . file: dockerfiles/neuroconv_dev_dockerfile + provenance: false diff --git a/.github/workflows/build_and_upload_docker_image_latest_release.yml b/.github/workflows/build_and_upload_docker_image_latest_release.yml index 79e788ea3..00028b5ba 100644 --- a/.github/workflows/build_and_upload_docker_image_latest_release.yml +++ b/.github/workflows/build_and_upload_docker_image_latest_release.yml @@ -5,6 +5,7 @@ on: workflows: [auto-publish] types: [completed] workflow_dispatch: + push: concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} @@ -43,3 +44,4 @@ jobs: tags: ghcr.io/catalystneuro/neuroconv:latest,ghcr.io/catalystneuro/neuroconv:${{ steps.parsed_version.outputs.version_tag }} context: . file: dockerfiles/neuroconv_latest_release_dockerfile + provenance: false From d4ddb6759c28ffff5f889fc4a413392310533ec3 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 22 Mar 2024 15:52:01 -0400 Subject: [PATCH 118/119] disable all push --- .github/workflows/build_and_upload_docker_image_dev.yml | 1 - .../workflows/build_and_upload_docker_image_latest_release.yml | 1 - .../workflows/build_and_upload_docker_image_yaml_variable.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index 4bae5f96e..b2353bfc2 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -4,7 +4,6 @@ on: schedule: - cron: "0 16 * * 1" # Weekly at noon EST on Monday workflow_dispatch: - push: concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/build_and_upload_docker_image_latest_release.yml b/.github/workflows/build_and_upload_docker_image_latest_release.yml index 00028b5ba..423686ec9 100644 --- a/.github/workflows/build_and_upload_docker_image_latest_release.yml +++ b/.github/workflows/build_and_upload_docker_image_latest_release.yml @@ -5,7 +5,6 @@ on: workflows: [auto-publish] types: [completed] workflow_dispatch: - push: concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml index 507145bb1..7ff2dc63c 100644 --- a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml +++ b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml @@ -5,7 +5,6 @@ on: workflows: [build_and_upload_docker_image_latest_release] types: [completed] workflow_dispatch: - push: concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} From 87e53735fd6f097f8a9d97122b7726c0d31d2364 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 22 Mar 2024 15:52:26 -0400 Subject: [PATCH 119/119] disable all push --- .github/workflows/docker_testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker_testing.yml b/.github/workflows/docker_testing.yml index 7ab461c57..6916e0e4e 100644 --- a/.github/workflows/docker_testing.yml +++ b/.github/workflows/docker_testing.yml @@ -3,7 +3,6 @@ on: schedule: - cron: "0 16 * * *" # Daily at noon EST workflow_dispatch: - push: # remove once on main jobs: run: