diff --git a/.github/workflows/pull-build-image.yaml b/.github/workflows/pull-build-image.yaml index 29082563..89b24cbd 100644 --- a/.github/workflows/pull-build-image.yaml +++ b/.github/workflows/pull-build-image.yaml @@ -15,6 +15,7 @@ on: - "scripts/**" - "tests/**" - "**/*.md" + - "doc_indexer/**" - CODEOWNERS - LICENSE diff --git a/.github/workflows/pull-evaluation-tests.yaml b/.github/workflows/pull-evaluation-tests.yaml index 6f818f59..7cdde316 100644 --- a/.github/workflows/pull-evaluation-tests.yaml +++ b/.github/workflows/pull-evaluation-tests.yaml @@ -23,6 +23,7 @@ on: - "scripts/**" - "tests/**" - "**/*.md" + - "doc_indexer/**" - CODEOWNERS - LICENSE diff --git a/.github/workflows/pull-integration-test.yaml b/.github/workflows/pull-integration-test.yaml index d0e2a93b..34dc3740 100644 --- a/.github/workflows/pull-integration-test.yaml +++ b/.github/workflows/pull-integration-test.yaml @@ -15,6 +15,7 @@ on: - ".reuse/**" - "config/**" - "data/**" + - "doc_indexer/**" - "scripts/**" jobs: diff --git a/.github/workflows/pull-lint-doc-indexer.yaml b/.github/workflows/pull-lint-doc-indexer.yaml new file mode 100644 index 00000000..c78036c0 --- /dev/null +++ b/.github/workflows/pull-lint-doc-indexer.yaml @@ -0,0 +1,46 @@ +name: pull-lint-doc-indexer + +on: + pull_request: + types: [ opened, synchronize, reopened, ready_for_review ] + branches: + - "main" + - "release-**" + paths: + - "doc_indexer/**" + paths-ignore: + - "doc_indexer/**.md" + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Extract Python version + id: python-version + working-directory: ./doc_indexer + run: | + ../scripts/shell/extract-python-version.sh + echo "Got PYTHON_VERSION=${PYTHON_VERSION}" + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install dependencies + working-directory: ./doc_indexer + run: poetry install --with test + + - name: Run lint + working-directory: ./doc_indexer + run: | + ls + poetry run poe codecheck diff --git a/.github/workflows/pull-tests-doc-indexer.yaml b/.github/workflows/pull-tests-doc-indexer.yaml new file mode 100644 index 00000000..7eab144e --- /dev/null +++ b/.github/workflows/pull-tests-doc-indexer.yaml @@ -0,0 +1,115 @@ +name: pull-tests-doc-indexer + +on: + pull_request_target: + types: [ opened, synchronize, reopened, ready_for_review ] + branches: + - "main" + - "release-**" + paths: + - "doc_indexer/**" + paths-ignore: + - "doc_indexer/**.md" + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + + - name: Extract Python version + id: python-version + working-directory: ./doc_indexer + run: | + ../scripts/shell/extract-python-version.sh + echo "Got PYTHON_VERSION=${PYTHON_VERSION}" + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install dependencies + working-directory: ./doc_indexer + run: poetry install --with test + + - name: Prepare test configurations + id: int-tests-configs + env: + AICORE_SECRET: ${{ secrets.AICORE_SECRET }} + run: | + echo "${AICORE_SECRET}" | base64 -d > aicore-secret.yaml + echo "AICORE_CLIENT_ID=$(yq eval '.stringData.clientid' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "AICORE_CLIENT_SECRET=$(yq eval '.stringData.clientsecret' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "AICORE_AUTH_URL=$(yq eval '.stringData.auth_url' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "AICORE_BASE_URL=$(yq eval '.stringData.base_url' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "AICORE_RESOURCE_GROUP=$(yq eval '.stringData.resource_group' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "EMBEDDING_MODEL_DEPLOYMENT_ID=$(yq eval '.stringData.embedding_model_deployment_id' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "EMBEDDING_MODEL_NAME=$(yq eval '.stringData.embedding_model_name' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "DATABASE_URL=$(yq eval '.stringData.hana_db_url' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "DATABASE_PORT=$(yq eval '.stringData.hana_db_port' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "DATABASE_USER=$(yq eval '.stringData.hana_db_user' aicore-secret.yaml)" >> $GITHUB_OUTPUT + echo "DATABASE_PASSWORD=$(yq eval '.stringData.hana_db_password' aicore-secret.yaml)" >> $GITHUB_OUTPUT + # cleanup + rm -f aicore-secret.yaml + + - name: Run unit tests + working-directory: ./doc_indexer + env: + LOG_LEVEL: "DEBUG" + # AI Core configurations. + AICORE_CLIENT_ID: ${{ steps.int-tests-configs.outputs.AICORE_CLIENT_ID }} + AICORE_CLIENT_SECRET: ${{ steps.int-tests-configs.outputs.AICORE_CLIENT_SECRET }} + AICORE_AUTH_URL: ${{ steps.int-tests-configs.outputs.AICORE_AUTH_URL }} + AICORE_BASE_URL: ${{ steps.int-tests-configs.outputs.AICORE_BASE_URL }} + AICORE_RESOURCE_GROUP: ${{ steps.int-tests-configs.outputs.AICORE_RESOURCE_GROUP }} + # EMBEDDING_MODEL. + EMBEDDING_MODEL_DEPLOYMENT_ID: ${{ steps.int-tests-configs.outputs.EMBEDDING_MODEL_DEPLOYMENT_ID }} + EMBEDDING_MODEL_NAME: ${{ steps.int-tests-configs.outputs.EMBEDDING_MODEL_NAME }} + # Database configurations. + DATABASE_URL: ${{ steps.int-tests-configs.outputs.DATABASE_URL }} + DATABASE_PORT: ${{ steps.int-tests-configs.outputs.DATABASE_PORT }} + DATABASE_USER: ${{ steps.int-tests-configs.outputs.DATABASE_USER }} + DATABASE_PASSWORD: ${{ steps.int-tests-configs.outputs.DATABASE_PASSWORD }} + # Kyma docs path. + DOCS_SOURCES_FILE_PATH: "./docs_sources.json" + DOCS_PATH: : "/test-data" + DOCS_TABLE_NAME: "kc_pr_${{ github.event.pull_request.number }}" + run: | + ls + poetry run poe test-unit + + - name: Run integration tests + working-directory: ./doc_indexer + env: + LOG_LEVEL: "DEBUG" + # AI Core configurations. + AICORE_CLIENT_ID: ${{ steps.int-tests-configs.outputs.AICORE_CLIENT_ID }} + AICORE_CLIENT_SECRET: ${{ steps.int-tests-configs.outputs.AICORE_CLIENT_SECRET }} + AICORE_AUTH_URL: ${{ steps.int-tests-configs.outputs.AICORE_AUTH_URL }} + AICORE_BASE_URL: ${{ steps.int-tests-configs.outputs.AICORE_BASE_URL }} + AICORE_RESOURCE_GROUP: ${{ steps.int-tests-configs.outputs.AICORE_RESOURCE_GROUP }} + # EMBEDDING_MODEL. + EMBEDDING_MODEL_DEPLOYMENT_ID: ${{ steps.int-tests-configs.outputs.EMBEDDING_MODEL_DEPLOYMENT_ID }} + EMBEDDING_MODEL_NAME: ${{ steps.int-tests-configs.outputs.EMBEDDING_MODEL_NAME }} + # Database configurations. + DATABASE_URL: ${{ steps.int-tests-configs.outputs.DATABASE_URL }} + DATABASE_PORT: ${{ steps.int-tests-configs.outputs.DATABASE_PORT }} + DATABASE_USER: ${{ steps.int-tests-configs.outputs.DATABASE_USER }} + DATABASE_PASSWORD: ${{ steps.int-tests-configs.outputs.DATABASE_PASSWORD }} + # Kyma docs path. + DOCS_SOURCES_FILE_PATH: "./docs_sources.json" + DOCS_PATH: "/test-data" + DOCS_TABLE_NAME: "kc_pr_${{ github.event.pull_request.number }}" + run: | + ls + poetry run poe test-integration diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index a96bb9f3..1f508d4f 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -8,6 +8,7 @@ on: paths-ignore: - "docs/**" - "**.md" + - "doc_indexer/**" - CODEOWNERS - LICENSE diff --git a/doc_indexer/tests/unit/fixtures/empty_dir/.gitkeep b/doc_indexer/tests/unit/fixtures/empty_dir/.gitkeep new file mode 100644 index 00000000..e69de29b