action.yml

name: Sync and cache test resources specified by the project config
author: Data Intuitive
description: |
  Sync and cache test resources. This action will sync test resources from
  different sources (e.g. S3) and cache them for later use. The cache key is
  based on the hash of the test resources. For this action to work, the Viash
  project config should contain a list of test resources to sync.

  Supported storage types:

    - `s3`: Syncs resources from an S3 bucket.
    - Create a GitHub issue if you need support for other storage types.

  Example:

  ```yaml
  info:
    test_resources:
      - type: s3
        path: s3://my-bucket/my-folder
        dest: my-folder
  ```
inputs:
  project_directory:
    required: false
    description: Path to the project directory. This is the directory where the project config `_viash.yaml` is located.
  cache_key_prefix:
    required: false
    description: A prefix for the cache hash key. Prefix is also used for restoring stale cache if no cache hit occurred for key.
    default: cachekey__
outputs:
  cache_key:
    description: Caching key to use to restore the cache. If no test resources are detected, this will be an empty string.
    value: ${{ steps.cache_key.outputs.cache_key }}
  dest_paths:
    description: Paths to the synced resources. If no test resources are detected, this will be an empty string.
    value: ${{ steps.test_resources.outputs.dest_paths }}
runs:
  using: 'composite'
  steps:
    - name: Detect test resources
      id: test_resources
      shell: bash
      working-directory: ${{ inputs.project_directory }}
      run: |
        resources_detected=$(yq e '.info | has("test_resources")' "_viash.yaml")
        if [ "$resources_detected" == "false" ]; then
          echo "No test resources detected."
          echo "dest_paths=" >> $GITHUB_OUTPUT
          exit 0
        fi

        # reformat the test resources into a pseudo-json that can be read line-by-line by bash
        test_resources=$(
          yq e \
            '.info.test_resources[] | "{type: " + (.type // "s3") + ", path: " + .path + ", dest: " + .dest + "}"' \
            "_viash.yaml"
        )

        # fetch the test resource destination paths
        # todo: convert to absolute paths
        dest_paths=$(
          yq e '.info.test_resources[] | .dest' "_viash.yaml" | sed "s#.*#$(pwd)/&#"
        )

        # output multiline string
        cat >> $GITHUB_OUTPUT <<HERE
        test_resources<<EOF
        $test_resources
        EOF
        dest_paths<<EOF
        $dest_paths
        EOF
        HERE

    # create cache_key key
    # this is the combined hash of all of the files across the different test resource sources
    - name: Create hash key
      shell: bash
      id: cache_key
      working-directory: ${{ inputs.project_directory }}
      run: |
        if [ -z "${{ steps.test_resources.outputs.test_resources }}" ]; then
          echo "cache_key=" >> $GITHUB_OUTPUT
          exit 0
        fi

        function s3_ls() {
          local s3_path="$1"
          AWS_EC2_METADATA_DISABLED=true \
            aws s3 ls \
            "$s3_path" \
            --recursive \
            --no-sign-request
        }

        ls_content=$(
          echo "${{ steps.test_resources.outputs.test_resources }}" | \
            while read -r line; do
              type=$(echo "$line" | yq e '.type')
              path=$(echo "$line" | yq e '.path')
              dest=$(echo "$line" | yq e '.dest')

              echo "# type: $type, path: $path, dest: $dest"
              echo ""

              if [ "$type" == "s3" ]; then
                s3_ls "$path"
              fi

              echo ""
            done
        )
        
        hash=$(echo "${ls_content}" | md5sum | awk '{ print $1 }')

        echo "cache_key=${{ inputs.cache_key_prefix }}$hash" >> $GITHUB_OUTPUT
    
    - name: Print resources
      shell: bash
      if: ${{ steps.cache_key.outputs.cache_key != '' }}
      working-directory: ${{ inputs.project_directory }}
      run: |
        echo "### Cache key: ${{ steps.cache_key.outputs.cache_key }}"
        echo

        echo "### Contents of 'test_resources':"
        echo
        echo "${{ steps.test_resources.outputs.test_resources }}"
        echo

        echo "### Contents of 'dest_paths':"
        echo
        echo "${{ steps.test_resources.outputs.dest_paths }}"
        echo
    
    # initialize cache
    - name: Cache resources
      uses: actions/cache@v4
      if: ${{ steps.cache_key.outputs.cache_key != '' }}
      with:
        path: ${{ steps.test_resources.outputs.dest_paths }}
        key: ${{ steps.cache_key.outputs.cache_key }}
        restore-keys: ${{ inputs.cache_key_prefix }}

    # sync if need be
    - name: Sync resources
      shell: bash
      if: ${{ steps.cache_key.outputs.cache_key != '' }}
      working-directory: ${{ inputs.project_directory }}
      run: |
        function sync_s3() {
          local s3_path="$1"
          local dest_path="$2"
          AWS_EC2_METADATA_DISABLED=true \
            aws s3 sync \
            "$s3_path" \
            "$dest_path" \
            --no-sign-request
        }

        echo "${{ steps.test_resources.outputs.test_resources }}" | \
          while read -r line; do
            type=$(echo "$line" | yq e '.type')
            path=$(echo "$line" | yq e '.path')
            dest=$(echo "$line" | yq e '.dest')

            echo "Syncing '$path' to '$dest'..."

            if [ "$type" == "s3" ]; then
              sync_s3 "$path" "$dest"
            fi
          done
        
    - name: List resources
      shell: bash
      if: ${{ steps.cache_key.outputs.cache_key != '' }}
      working-directory: ${{ inputs.project_directory }}
      run: |
        echo "${{ steps.test_resources.outputs.test_resources }}" | \
          while read -r line; do
            type=$(echo "$line" | yq e '.type')
            path=$(echo "$line" | yq e '.path')
            dest=$(echo "$line" | yq e '.dest')

            echo "===================================="
            echo "type: $type"
            echo "path: $path"
            echo "dest: $dest"
            tree $dest -L 3
            echo ""
            echo "===================================="
          done