Skip to content

Commit

Permalink
refactor: extract current github jobs for reuse
Browse files Browse the repository at this point in the history
  • Loading branch information
NiklasKoehneckeAA committed Apr 9, 2024
1 parent 103f164 commit 492497a
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 197 deletions.
202 changes: 5 additions & 197 deletions .github/workflows/github-actions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,206 +9,14 @@ on:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
doctest:
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: |
poetry config installer.max-workers 10
poetry install --no-interaction
- name: install Sphinx
run: sudo apt-get update -y && sudo apt-get install python3-sphinx

- name: run doctest
env:
AA_TOKEN: ${{ secrets.AA_TOKEN }}
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
CLIENT_URL: "https://api.aleph-alpha.com"
run: ./scripts/doctest.sh

uses: ./.github/workflows/python/doctest.yml
lint:
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: |
poetry config installer.max-workers 10
poetry install --no-interaction
- name: set PY for pre-commit
run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV

- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}

- name: Run linters
run: |
./scripts/lint.sh
uses: ./.github/workflows/python/lint.yml
test:
runs-on: ubuntu-latest

services:
argilla-elastic-search:
image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3
env:
ES_JAVA_OPTS: "-Xms512m -Xmx512m"
discovery.type: "single-node"
xpack.security.enabled: "false"
argilla:
image: argilla/argilla-server
ports:
- "6900:6900"
env:
ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200"
open-telemetry-trace-service:
env:
COLLECTOR_OTLP_ENABLED: "true"
ports:
- "4317:4317"
- "4318:4318"
- "16686:16686"
image: jaegertracing/all-in-one:1.35

steps:
- name: Checkout repository
uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: |
poetry config installer.max-workers 10
poetry install --no-interaction
- name: Run pytest
env:
AA_TOKEN: ${{ secrets.AA_TOKEN }}
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
ARGILLA_API_URL: "http://localhost:6900/"
ARGILLA_API_KEY: "argilla.apikey"
CLIENT_URL: "https://api.aleph-alpha.com"
run: |
./scripts/test.sh
uses: ./.github/workflows/python/test.yml
run-notebooks:
runs-on: ubuntu-latest

services:
argilla-elastic-search:
image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3
env:
ES_JAVA_OPTS: "-Xms512m -Xmx512m"
discovery.type: "single-node"
xpack.security.enabled: "false"
argilla:
image: argilla/argilla-server
ports:
- "6900:6900"
env:
ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200"

steps:
- name: Checkout repository
uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: |
poetry config installer.max-workers 10
poetry install --no-interaction
uses: ./.github/workflows/python/run_notebooks.yml

- name: Configure Poetry for notebooks and run
env:
AA_TOKEN: ${{ secrets.AA_TOKEN }}
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
ARGILLA_API_URL: "http://localhost:6900/"
ARGILLA_API_KEY: "argilla.apikey"
CLIENT_URL: "https://api.aleph-alpha.com"
run: |
./scripts/notebook_runner.sh

42 changes: 42 additions & 0 deletions .github/workflows/python/doctest.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: "Doctest Runner"
description: "Runs doctests on python artifacts."
doctest:
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: |
poetry config installer.max-workers 10
poetry install --no-interaction

- name: install Sphinx
run: sudo apt-get update -y && sudo apt-get install python3-sphinx

- name: run doctest
env:
AA_TOKEN: ${{ secrets.AA_TOKEN }}
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
CLIENT_URL: "https://api.aleph-alpha.com"
run: ./scripts/doctest.sh
42 changes: 42 additions & 0 deletions .github/workflows/python/lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
lint:
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: |
poetry config installer.max-workers 10
poetry install --no-interaction

- name: set PY for pre-commit
run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV

- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}

- name: Run linters
run: |
./scripts/lint.sh
55 changes: 55 additions & 0 deletions .github/workflows/python/run_notebooks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
run-notebooks:
uses: ./.github/workflows/python/run_notebooks.yml
runs-on: ubuntu-latest

services:
argilla-elastic-search:
image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3
env:
ES_JAVA_OPTS: "-Xms512m -Xmx512m"
discovery.type: "single-node"
xpack.security.enabled: "false"
argilla:
image: argilla/argilla-server
ports:
- "6900:6900"
env:
ARGILLA_ELASTICSEARCH: "http://argilla-elastic-search:9200"

steps:
- name: Checkout repository
uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: |
poetry config installer.max-workers 10
poetry install --no-interaction

- name: Configure Poetry for notebooks and run
env:
AA_TOKEN: ${{ secrets.AA_TOKEN }}
HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
ARGILLA_API_URL: "http://localhost:6900/"
ARGILLA_API_KEY: "argilla.apikey"
CLIENT_URL: "https://api.aleph-alpha.com"
run: |
./scripts/notebook_runner.sh
Loading

0 comments on commit 492497a

Please sign in to comment.