Skip to content
Python SDK Tests

Python SDK Tests #65

Sign in to view logs

Python SDK Tests

Python SDK Tests #65

Workflow file for this run

.github/workflows/test.yml at 1343a98
name: Python SDK Tests
on:
push:
branches: [main, develop]
paths:
- 'src/**'
- 'tests/**'
- 'pyproject.toml'
- '.github/workflows/test.yml'
pull_request:
branches: [main, develop]
paths:
- 'src/**'
- 'tests/**'
- 'pyproject.toml'
- '.github/workflows/test.yml'
workflow_dispatch: # Allow manual trigger
schedule:
# Weekly mutation testing — Mondays at 03:00 UTC. The mutmut job is
# the only consumer of this trigger; other jobs ignore it via the
# `if: github.event_name == ...` guards on each job.
- cron: '0 3 * * 1'
env:
PYTHON_VERSION: '3.11'
COVERAGE_THRESHOLD: 85
jobs:
# =============================================================================
# Fast Unit Tests (< 5 minutes)
# =============================================================================
test-fast:
name: Unit Tests
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
matrix:
python-version: ['3.9', '3.10', '3.11', '3.12']
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Run unit tests
run: |
# integration_sepolia is also marker-gated in pyproject.toml,
# but ignore the dir explicitly so the runner doesn't even
# collect them (saves seconds + clearer "X deselected" output).
pytest tests/ \
--ignore=tests/integration/ \
--ignore=tests/integration_sepolia/ \
--ignore=tests/benchmarks/ \
-v \
--tb=short \
-x # Stop on first failure
- name: Upload test results
uses: actions/upload-artifact@v4
if: failure()
with:
name: test-results-${{ matrix.python-version }}
path: |
pytest.log
.pytest_cache/
# =============================================================================
# Coverage Report
# =============================================================================
test-coverage:
name: Coverage Report
runs-on: ubuntu-latest
timeout-minutes: 15
needs: test-fast
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
pip install pytest-cov
- name: Run tests with coverage
run: |
pytest tests/ \
--ignore=tests/integration/ \
--ignore=tests/benchmarks/ \
--cov=src/agirails \
--cov-report=xml \
--cov-report=html \
--cov-report=term-missing \
--cov-fail-under=${{ env.COVERAGE_THRESHOLD }}
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
flags: unittests
name: python-sdk-coverage
fail_ci_if_error: false
- name: Upload coverage HTML report
uses: actions/upload-artifact@v4
with:
name: coverage-report
path: htmlcov/
# =============================================================================
# Security Tests
# =============================================================================
test-security:
name: Security Tests
runs-on: ubuntu-latest
timeout-minutes: 10
needs: test-fast
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
pip install bandit pip-audit safety
- name: Run security tests
run: |
pytest tests/test_security/ -v --tb=short
- name: Run bandit security linter
run: |
bandit -r src/agirails -ll -ii --format json -o bandit-report.json || true
bandit -r src/agirails -ll -ii
- name: Run pip-audit for dependency vulnerabilities
run: |
pip-audit --format json --output pip-audit-report.json || true
pip-audit
- name: Upload security reports
uses: actions/upload-artifact@v4
if: always()
with:
name: security-reports
path: |
bandit-report.json
pip-audit-report.json
# =============================================================================
# Property-Based Tests (Hypothesis)
# =============================================================================
test-properties:
name: Property-Based Tests
runs-on: ubuntu-latest
timeout-minutes: 15
needs: test-fast
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Run property-based tests
run: |
pytest tests/test_properties/ \
-v \
--tb=short \
--hypothesis-show-statistics \
--hypothesis-seed=0 # Reproducible
# =============================================================================
# Parity Tests (Python <-> TypeScript SDK)
# =============================================================================
test-parity:
name: SDK Parity Tests
runs-on: ubuntu-latest
timeout-minutes: 10
needs: test-fast
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Run parity tests (Python-only fixtures)
run: |
pytest tests/test_parity.py -v --tb=short
- name: Run cross-SDK parity tests (TS-signed → Python verify)
# tests/test_cross_sdk/ uses pre-generated JSON fixtures in
# tests/fixtures/cross_sdk/ that are produced by the TypeScript
# SDK's CounterOfferBuilder/CounterAcceptBuilder. Fixtures are
# committed to the repo; no Node setup needed at CI time.
# To regenerate (after any wire-format change), run locally:
# NODE_PATH=../sdk-js/node_modules node scripts/generate_parity_vectors.js
run: |
pytest tests/test_cross_sdk/ -v --tb=short
- name: Run golden hash snapshots
# Pinned bytes32 outputs for canonical-JSON + builder.compute_hash.
# If anyone silently changes serialization, this fails before
# integrators see broken transactions.
run: |
pytest tests/test_golden/ -v --tb=short
# =============================================================================
# Integration Tests (Requires Anvil - Push only)
# =============================================================================
test-integration:
name: Integration Tests
runs-on: ubuntu-latest
timeout-minutes: 20
if: github.event_name == 'push'
needs: [test-fast, test-security]
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install Foundry (Anvil)
uses: foundry-rs/foundry-toolchain@v1
with:
version: nightly
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Start Anvil in background
run: |
anvil --fork-url ${{ secrets.BASE_SEPOLIA_RPC_URL || 'https://sepolia.base.org' }} &
sleep 5 # Wait for Anvil to start
- name: Run integration tests
env:
ANVIL_RPC_URL: http://localhost:8545
run: |
pytest tests/integration/ -v --tb=short || echo "Integration tests completed (some may be skipped)"
- name: Stop Anvil
if: always()
run: pkill anvil || true
# =============================================================================
# Benchmarks (Main only)
# =============================================================================
test-benchmarks:
name: Performance Benchmarks
runs-on: ubuntu-latest
timeout-minutes: 15
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
needs: test-fast
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
pip install pytest-benchmark
- name: Run benchmarks
run: |
pytest tests/benchmarks/ \
-v \
--benchmark-only \
--benchmark-json=benchmark-results.json \
--benchmark-min-rounds=10 \
--benchmark-warmup=on
- name: Upload benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark-results
path: benchmark-results.json
- name: Store benchmark result
uses: benchmark-action/github-action-benchmark@v1
if: github.event_name == 'push'
with:
tool: 'pytest'
output-file-path: benchmark-results.json
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
alert-threshold: '150%'
comment-on-alert: true
fail-on-alert: false
# =============================================================================
# Linting & Type Checking
# =============================================================================
lint:
name: Lint & Type Check
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install ruff mypy
pip install -e ".[dev]"
- name: Run Ruff linter
run: |
ruff check src/agirails --output-format github
- name: Run Ruff formatter check
run: |
ruff format --check src/agirails
- name: Run mypy type checker
run: |
mypy src/agirails --ignore-missing-imports --no-error-summary || true
# =============================================================================
# Summary Job
# =============================================================================
test-summary:
name: Test Summary
runs-on: ubuntu-latest
needs: [test-fast, test-coverage, test-security, test-properties, test-parity, lint]
if: always()
steps:
- name: Check test results
run: |
echo "## Test Results Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY
echo "| Unit Tests | ${{ needs.test-fast.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Coverage | ${{ needs.test-coverage.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Security | ${{ needs.test-security.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Properties | ${{ needs.test-properties.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Parity | ${{ needs.test-parity.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Lint | ${{ needs.lint.result }} |" >> $GITHUB_STEP_SUMMARY
- name: Fail if any required job failed
if: |
needs.test-fast.result == 'failure' ||
needs.test-coverage.result == 'failure' ||
needs.test-security.result == 'failure' ||
needs.lint.result == 'failure'
run: exit 1
# =============================================================================
# Live Base sepolia — manual gate only.
#
# Runs the gated `integration_sepolia` test suite against the real
# Base sepolia kernel (0x9d25…0021b). Costs 1-5 sepolia txes per
# full run, signed by the wallet whose keystore + password are
# supplied as repo secrets.
#
# Required secrets:
# ACTP_KEY_PASSWORD — keystore decryption password
# ACTP_KEYSTORE_BASE64 — base64-encoded keystore JSON (committed
# keystore-file path doesn't exist in CI)
# =============================================================================
live-sepolia:
name: Live Base sepolia integration
runs-on: ubuntu-latest
timeout-minutes: 20
if: github.event_name == 'workflow_dispatch'
needs: test-fast
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Materialize keystore from secret
env:
ACTP_KEYSTORE_BASE64: ${{ secrets.ACTP_KEYSTORE_BASE64 }}
run: |
if [ -z "$ACTP_KEYSTORE_BASE64" ]; then
echo "::error::ACTP_KEYSTORE_BASE64 secret not configured; skipping live tests"
exit 78 # neutral skip
fi
mkdir -p ~/.actp/mainnet-deployer
echo "$ACTP_KEYSTORE_BASE64" | base64 -d > ~/.actp/mainnet-deployer/deployer
chmod 600 ~/.actp/mainnet-deployer/deployer
- name: Run live sepolia integration
env:
ACTP_KEY_PASSWORD: ${{ secrets.ACTP_KEY_PASSWORD }}
run: |
pytest tests/integration_sepolia/ \
-m integration_sepolia \
-v --tb=short
# =============================================================================
# Mutation testing — weekly schedule + manual.
#
# mutmut introduces small code mutations (negate condition, swap
# operator, etc.) and re-runs the test suite. Each "survived"
# mutation means our tests didn't catch a behaviour change — points
# at gaps in test coverage that line coverage alone can't see.
# Configured in pyproject.toml [tool.mutmut].
# =============================================================================
mutation-test:
name: Mutation testing
runs-on: ubuntu-latest
timeout-minutes: 360 # 6h cap
if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev,mutation]"
- name: Run mutmut
run: |
mutmut run --paths-to-mutate src/agirails/utils/security.py,src/agirails/runtime/mock_runtime.py,src/agirails/protocol/proofs.py || true
mutmut results > mutmut_results.txt
mutmut junitxml > mutmut_results.xml || true
echo "## Mutation Results" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
tail -40 mutmut_results.txt >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
- name: Upload mutation report
uses: actions/upload-artifact@v4
if: always()
with:
name: mutmut-results
path: |
mutmut_results.txt
mutmut_results.xml