Merge pull request #4 from shua-ie/feat/quality-assessment-gpu #12
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | |
| on: | |
| push: | |
| branches: [ main, develop ] | |
| pull_request: | |
| branches: [ main ] | |
| jobs: | |
| test: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| python-version: ["3.11"] | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Run linting | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| black --check src/ tests/ | |
| isort --check-only src/ tests/ | |
| ruff check src/ tests/ | |
| - name: Type check | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| mypy --strict src/quarrycore/ | |
| - name: Run tests with coverage | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| python -m pytest tests/ \ | |
| --cov=src/quarrycore \ | |
| --cov-report=xml \ | |
| --cov-report=term-missing \ | |
| --cov-report=html \ | |
| --timeout=60 \ | |
| -v \ | |
| -k "not slow" \ | |
| --ignore=tests/e2e/ | |
| - name: Check pipeline.py coverage | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| coverage xml -i | |
| coverage report --include="src/quarrycore/pipeline.py" --fail-under=90 | |
| - name: Check dead_letter.py coverage | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| coverage report --include="src/quarrycore/recovery/dead_letter.py" --fail-under=90 | |
| - name: Check critical file coverage | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| coverage report --include="src/quarrycore/pipeline.py" --fail-under=90 | |
| coverage report --include="src/quarrycore/recovery/dead_letter.py" --fail-under=90 | |
| - name: Enforce coverage requirements for critical files | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| # Check coverage for pipeline.py (must be ≥90%) | |
| coverage report --include="src/quarrycore/pipeline.py" --fail-under=90 || { | |
| echo "❌ Pipeline coverage below 90% requirement" | |
| exit 1 | |
| } | |
| # Check coverage for dead_letter.py (must be ≥90%) | |
| coverage report --include="src/quarrycore/recovery/dead_letter.py" --fail-under=90 || { | |
| echo "❌ Dead letter queue coverage below 90% requirement" | |
| exit 1 | |
| } | |
| echo "✅ Coverage requirements met for critical pipeline components" | |
| - name: Upload coverage to Codecov | |
| if: ${{ !env.ACT }} | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: ./coverage.xml | |
| flags: unittests | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Coverage upload skipped (local simulation) | |
| if: ${{ env.ACT }} | |
| run: echo "⏭️ Codecov upload skipped in local simulation" | |
| - name: Security scan | |
| run: | | |
| pip install bandit | |
| bandit -r src/ -ll || true | |
| - name: Run benchmark | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| python benchmarks/profile_1k_urls.py | |
| - name: Check benchmark retention | |
| env: | |
| QUARRY_TEST_MODE: "1" | |
| run: | | |
| python benchmarks/compare.py 95 | |
| build: | |
| needs: test | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Cache pip packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Build package | |
| run: | | |
| python -m pip install --upgrade pip build | |
| python -m build | |
| - name: Upload artifacts | |
| if: ${{ !env.ACT }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist-${{ github.run_id }} | |
| path: dist/ | |
| compression-level: 6 | |
| retention-days: 30 | |
| - name: Artifacts upload skipped (local simulation) | |
| if: ${{ env.ACT }} | |
| run: echo "⏭️ Artifact upload skipped in local simulation" |