CI #62
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # .github/workflows/ci.yaml | |
| name: CI | |
| on: | |
| push: | |
| pull_request: | |
| schedule: | |
| - cron: '0 4 * * 6' # weekly Saturday 04:00 UTC | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| # ------------------------------------------------------------- # | |
| # 1A) Quick quality check — every push/PR (Python 3.12) # | |
| # ------------------------------------------------------------- # | |
| jobs: | |
| quick-quality: | |
| name: quality-3.12 | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'schedule' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Cache pip wheels | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: >- | |
| pip-${{ runner.os }}-3.12-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: pip-${{ runner.os }}-3.12- | |
| - name: Install CI dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install \ | |
| "dvc[s3]" \ | |
| pre-commit \ | |
| ruff \ | |
| black \ | |
| isort \ | |
| mypy \ | |
| pytest \ | |
| pytest-cov \ | |
| coverage[toml] | |
| - name: Run pre‑commit hooks | |
| run: pre-commit run --all-files --color never | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::513065063624:role/GithubOIDCRole | |
| aws-region: eu-central-1 | |
| - name: Pull data / params | |
| run: dvc pull --verbose --jobs 4 | |
| - name: Sync MLflow runs to S3 | |
| if: always() | |
| run: aws s3 sync mlruns s3://ghcicd/mlruns --exact-timestamps | |
| # --- Tests + coverage (optional) ------------------------ | |
| # - name: Pytest + coverage | |
| # run: pytest -q --cov=src --cov-report=xml | |
| # - uses: codecov/codecov-action@v4 | |
| # with: | |
| # files: coverage.xml | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: mlruns-3.12 | |
| path: mlruns | |
| retention-days: 7 | |
| # ------------------------------------------------------------- # | |
| # 1B) Full quality matrix — weekly cron only (Py 3.10 & 3.11) # | |
| # ------------------------------------------------------------- # | |
| full-quality: | |
| name: quality-${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python-version: ['3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip wheels | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: >- | |
| pip-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: pip-${{ runner.os }}-${{ matrix.python-version }}- | |
| - name: Install CI dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install \ | |
| "dvc[s3]" \ | |
| pre-commit \ | |
| ruff \ | |
| black \ | |
| isort \ | |
| mypy \ | |
| pytest \ | |
| pytest-cov \ | |
| coverage[toml] | |
| - name: Run pre‑commit hooks | |
| run: pre-commit run --all-files --color never | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::513065063624:role/GithubOIDCRole | |
| aws-region: eu-central-1 | |
| - name: Pull data / params | |
| run: dvc pull --verbose --jobs 4 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: mlruns-${{ matrix.python-version }} | |
| path: mlruns | |
| retention-days: 7 | |
| # ------------------------------------------------------------- # | |
| # 2) Full pipeline on GPU — cron, manual, or [gpu] commit # | |
| # ------------------------------------------------------------- # | |
| gpu-pipeline: | |
| needs: [quick-quality, full-quality] | |
| runs-on: ubuntu-22.04-gpu | |
| continue-on-error: true | |
| timeout-minutes: 5 | |
| if: | | |
| github.event_name == 'schedule' || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'push' && contains(github.event.head_commit.message, '[gpu]')) | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install runtime deps | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install "dvc[s3]" | |
| - uses: iterative/setup-dvc@v1 | |
| - name: Assume AWS role via OIDC | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::513065063624:role/GithubOIDCRole | |
| aws-region: eu-central-1 | |
| - name: Reproduce DVC pipeline | |
| run: dvc repro --pull -P | |
| - name: Push artefacts to S3 | |
| run: | | |
| dvc push --verbose --jobs 4 | |
| aws s3 sync mlruns s3://ghcicd/mlruns --delete | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: outputs | |
| path: | | |
| data/** | |
| models/** | |
| logs/** |