diff --git a/.github/workflows/ci-health-v3-master.yml b/.github/workflows/ci-health-v3-master.yml new file mode 100644 index 0000000000..0d053f2beb --- /dev/null +++ b/.github/workflows/ci-health-v3-master.yml @@ -0,0 +1,145 @@ +name: CI Health V3 Master +on: + schedule: + # Once a day at 05:00 UTC (9pm PDT / 10pm PST). + - cron: "0 5 * * *" + workflow_dispatch: + +permissions: + id-token: write # This is required for requesting the JWT + +jobs: + unit-test-v3: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops] + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + role-duration-seconds: 10800 + - name: Run Unit Tests V3 for ${{ matrix.submodule }} + uses: aws-actions/aws-codebuild-run-build@v1 + with: + project-name: sagemaker-python-sdk-ci-health-unit-test-v3 + source-version-override: refs/heads/master + env-vars-for-codebuild: | + SUBMODULE + env: + SUBMODULE: ${{ matrix.submodule }} + canaries-v3-master: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops] + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + role-duration-seconds: 10800 + - name: Run Canaries V3 master for ${{ matrix.submodule }} + uses: aws-actions/aws-codebuild-run-build@v1 + with: + project-name: sagemaker-python-sdk-ci-health-canaries-v3-master + source-version-override: refs/heads/master + env-vars-for-codebuild: | + SUBMODULE + env: + SUBMODULE: ${{ matrix.submodule }} + # GPU integ tests against the master source. Two region jobs trigger the same + # CodeBuild project name (deployed in both regions); only the region/creds and + # the region-override buildspec differ. Their results are AND-ed with + # import-model in report-result into a single per-run pass/fail metric. + gpu-integ-tests-master: + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + role-duration-seconds: 10800 + - name: Run GPU Integ Tests (master) + uses: aws-actions/aws-codebuild-run-build@v1 + with: + project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-master + source-version: refs/heads/master + gpu-integ-tests-master-us-east-1: + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials (us-east-1) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_US_EAST_1_ARN }} + aws-region: us-east-1 + role-duration-seconds: 10800 + - name: Run GPU Integ Tests (master, us-east-1) + uses: aws-actions/aws-codebuild-run-build@v1 + with: + project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-master + source-version: refs/heads/master + # Bedrock model-import integ tests. Run serially (concurrency 1) in their own + # CodeBuild project because the "Concurrent model import jobs" Bedrock quota is + # fixed at 1 and not raisable; running them in parallel (as PR checks did) + # makes them collide and flake. us-west-2 only (no us_east_1-marked tests). + # Folded into the master run-level pass/fail metric alongside the GPU jobs. + import-model-integ-tests: + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + role-duration-seconds: 10800 + - name: Run Bedrock Model-Import Integ Tests + uses: aws-actions/aws-codebuild-run-build@v1 + with: + project-name: sagemaker-python-sdk-ci-health-import-model-integ-tests + source-version: refs/heads/master + # Run-level result: the master GPU run is successful only if the us-west-2, + # us-east-1, and import-model jobs all succeeded. Emits GpuIntegMasterRunFailure + # = 1 (failed) / 0 (succeeded) to CloudWatch in us-west-2. The CDK alarm + # (GpuIntegMasterRunAlarm) sums this over a UTC day; it is folded into + # CIHealthCompositeAlarm-V3, which owns ticketing. Only scheduled runs emit the + # metric (manual workflow_dispatch runs are excluded). + report-result: + needs: [gpu-integ-tests-master, gpu-integ-tests-master-us-east-1, import-model-integ-tests] + if: always() + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }} + aws-region: us-west-2 + - name: Emit run-level pass/fail metric + run: | + # Manual (workflow_dispatch) runs must not contribute to the daily + # GpuIntegMasterRunFailure count that drives GpuIntegMasterRunAlarm; + # only scheduled runs count. + if [ "${{ github.event_name }}" != "schedule" ]; then + echo "Not a scheduled run (${{ github.event_name }}); skipping metric emission." + exit 0 + fi + if [ "${{ needs.gpu-integ-tests-master.result }}" == "success" ] && \ + [ "${{ needs.gpu-integ-tests-master-us-east-1.result }}" == "success" ] && \ + [ "${{ needs.import-model-integ-tests.result }}" == "success" ]; then + value=0 + echo "All master GPU/import jobs succeeded; emitting GpuIntegMasterRunFailure=0" + else + value=1 + echo "At least one master GPU/import job did not succeed; emitting GpuIntegMasterRunFailure=1" + fi + aws cloudwatch put-metric-data \ + --namespace GpuIntegRunMetrics \ + --metric-name GpuIntegMasterRunFailure \ + --value "$value" \ + --unit Count diff --git a/.github/workflows/ci-health-v3-release.yml b/.github/workflows/ci-health-v3-release.yml new file mode 100644 index 0000000000..d0b0f89475 --- /dev/null +++ b/.github/workflows/ci-health-v3-release.yml @@ -0,0 +1,103 @@ +name: CI Health V3 Release +on: + schedule: + # Once a day at 05:00 UTC (9pm PDT / 10pm PST). + - cron: "0 5 * * *" + workflow_dispatch: + +permissions: + id-token: write # This is required for requesting the JWT + +jobs: + canaries-v3-release: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops] + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + role-duration-seconds: 10800 + - name: Run Canaries V3 release for ${{ matrix.submodule }} + uses: aws-actions/aws-codebuild-run-build@v1 + with: + project-name: sagemaker-python-sdk-ci-health-canaries-v3-release + source-version-override: refs/heads/master + env-vars-for-codebuild: | + SUBMODULE + env: + SUBMODULE: ${{ matrix.submodule }} + # GPU integ tests against the published PyPI release. Two region jobs trigger + # the same CodeBuild project name (deployed in both regions); only the + # region/creds and the region-override buildspec differ. Their results are + # AND-ed in report-result into a single per-run pass/fail metric. + gpu-integ-tests-release: + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + role-duration-seconds: 10800 + - name: Run GPU Integ Tests (release) + uses: aws-actions/aws-codebuild-run-build@v1 + with: + project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-release + source-version: refs/heads/master + gpu-integ-tests-release-us-east-1: + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials (us-east-1) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_US_EAST_1_ARN }} + aws-region: us-east-1 + role-duration-seconds: 10800 + - name: Run GPU Integ Tests (release, us-east-1) + uses: aws-actions/aws-codebuild-run-build@v1 + with: + project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-release + source-version: refs/heads/master + # Run-level result: the release GPU run is successful only if both region jobs + # succeeded. Emits GpuIntegReleaseRunFailure = 1 (failed) / 0 (succeeded) to + # CloudWatch in us-west-2. The CDK alarm (GpuIntegReleaseRunAlarm) sums this + # over a UTC day; it is folded into CIHealthCompositeAlarm-V3, which owns + # ticketing. Only scheduled runs emit the metric (manual workflow_dispatch runs + # are excluded). + report-result: + needs: [gpu-integ-tests-release, gpu-integ-tests-release-us-east-1] + if: always() + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }} + aws-region: us-west-2 + - name: Emit run-level pass/fail metric + run: | + # Manual (workflow_dispatch) runs must not contribute to the daily + # GpuIntegReleaseRunFailure count that drives GpuIntegReleaseRunAlarm; + # only scheduled runs count. + if [ "${{ github.event_name }}" != "schedule" ]; then + echo "Not a scheduled run (${{ github.event_name }}); skipping metric emission." + exit 0 + fi + if [ "${{ needs.gpu-integ-tests-release.result }}" == "success" ] && \ + [ "${{ needs.gpu-integ-tests-release-us-east-1.result }}" == "success" ]; then + value=0 + echo "All release GPU jobs succeeded; emitting GpuIntegReleaseRunFailure=0" + else + value=1 + echo "At least one release GPU job did not succeed; emitting GpuIntegReleaseRunFailure=1" + fi + aws cloudwatch put-metric-data \ + --namespace GpuIntegRunMetrics \ + --metric-name GpuIntegReleaseRunFailure \ + --value "$value" \ + --unit Count diff --git a/.github/workflows/ci-health-v3.yml b/.github/workflows/ci-health-v3.yml deleted file mode 100644 index 8c687b5d24..0000000000 --- a/.github/workflows/ci-health-v3.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: CI Health V3 -on: - schedule: - # Once a day at 05:00 UTC (9pm PDT / 10pm PST). - - cron: "0 5 * * *" - workflow_dispatch: - -permissions: - id-token: write # This is required for requesting the JWT - -jobs: - unit-test-v3: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops] - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} - aws-region: us-west-2 - role-duration-seconds: 10800 - - name: Run Unit Tests V3 for ${{ matrix.submodule }} - uses: aws-actions/aws-codebuild-run-build@v1 - with: - project-name: sagemaker-python-sdk-ci-health-unit-test-v3 - source-version-override: refs/heads/master - env-vars-for-codebuild: | - SUBMODULE - env: - SUBMODULE: ${{ matrix.submodule }} - canaries-v3-master: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops] - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} - aws-region: us-west-2 - role-duration-seconds: 10800 - - name: Run Canaries V3 master for ${{ matrix.submodule }} - uses: aws-actions/aws-codebuild-run-build@v1 - with: - project-name: sagemaker-python-sdk-ci-health-canaries-v3-master - source-version-override: refs/heads/master - env-vars-for-codebuild: | - SUBMODULE - env: - SUBMODULE: ${{ matrix.submodule }} - canaries-v3-release: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops] - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} - aws-region: us-west-2 - role-duration-seconds: 10800 - - name: Run Canaries V3 release for ${{ matrix.submodule }} - uses: aws-actions/aws-codebuild-run-build@v1 - with: - project-name: sagemaker-python-sdk-ci-health-canaries-v3-release - source-version-override: refs/heads/master - env-vars-for-codebuild: | - SUBMODULE - env: - SUBMODULE: ${{ matrix.submodule }} diff --git a/.github/workflows/gpu-integ-tests.yml b/.github/workflows/gpu-integ-tests.yml deleted file mode 100644 index 11aba2673e..0000000000 --- a/.github/workflows/gpu-integ-tests.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: GPU Integ Tests -on: - schedule: - # US Pacific (PST, UTC-8): 0:00 AM / 3:00 AM / 6:00 AM -> 08/11/14 UTC. - # All three fire within the same UTC day so the run-level CloudWatch metric - # (GpuIntegRunFailure) aggregates correctly per day. - - cron: "0 8 * * *" - - cron: "0 11 * * *" - - cron: "0 14 * * *" - workflow_dispatch: - -permissions: - id-token: write # This is required for requesting the JWT - actions: read # required for the gate job to query prior runs of this workflow - -jobs: - # Gate: if an earlier scheduled run already succeeded today, skip the rest of - # today's scheduled runs. Manual (workflow_dispatch) runs always proceed. - check-prior-success: - runs-on: ubuntu-latest - outputs: - already_succeeded: ${{ steps.check.outputs.already_succeeded }} - steps: - - name: Check for a successful scheduled run earlier today - id: check - env: - GH_TOKEN: ${{ github.token }} - run: | - if [ "${{ github.event_name }}" != "schedule" ]; then - echo "Not a scheduled run; proceeding." - echo "already_succeeded=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - today=$(date -u +%Y-%m-%d) - count=$(gh api -X GET \ - "/repos/${{ github.repository }}/actions/workflows/gpu-integ-tests.yml/runs" \ - -f event=schedule \ - -f status=success \ - -f "created=>=${today}T00:00:00Z" \ - --jq '.workflow_runs | length') - echo "Successful scheduled runs today: $count" - if [ "$count" -gt 0 ]; then - echo "already_succeeded=true" >> "$GITHUB_OUTPUT" - else - echo "already_succeeded=false" >> "$GITHUB_OUTPUT" - fi - - gpu-integ-tests: - needs: check-prior-success - if: needs.check-prior-success.outputs.already_succeeded != 'true' - runs-on: ubuntu-latest - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} - aws-region: us-west-2 - role-duration-seconds: 10800 - - name: Run GPU Integ Tests - uses: aws-actions/aws-codebuild-run-build@v1 - with: - project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests - source-version: refs/heads/master - - gpu-integ-tests-us-east-1: - needs: check-prior-success - if: needs.check-prior-success.outputs.already_succeeded != 'true' - runs-on: ubuntu-latest - steps: - - name: Configure AWS Credentials (us-east-1) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_US_EAST_1_ARN }} - aws-region: us-east-1 - role-duration-seconds: 10800 - - name: Run GPU Integ Tests (us-east-1) - uses: aws-actions/aws-codebuild-run-build@v1 - with: - project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests - source-version: refs/heads/master - - # Bedrock model-import integ tests. Run serially (concurrency 1) in their own - # CodeBuild project because the "Concurrent model import jobs" Bedrock quota is - # fixed at 1 and not raisable; running them in parallel (as PR checks did) - # makes them collide and flake. us-west-2 only (no us_east_1-marked tests). - # Folded into the same run-level pass/fail metric as the GPU jobs below, so it - # shares the GpuIntegRunAlarm rather than getting a separate alarm. - import-model-integ-tests: - needs: check-prior-success - if: needs.check-prior-success.outputs.already_succeeded != 'true' - runs-on: ubuntu-latest - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} - aws-region: us-west-2 - role-duration-seconds: 10800 - - name: Run Bedrock Model-Import Integ Tests - uses: aws-actions/aws-codebuild-run-build@v1 - with: - project-name: sagemaker-python-sdk-ci-health-import-model-integ-tests - source-version: refs/heads/master - - # Run-level result: a run is successful only if BOTH region jobs succeeded. - # Emits GpuIntegRunFailure = 1 (failed) / 0 (succeeded) to CloudWatch in - # us-west-2. The CDK alarm (GpuIntegRunAlarm) sums this over a UTC day and - # cuts a daytime sev2 when all of the day's runs failed. Skipped when the gate - # short-circuited today's run (an earlier run already succeeded). - report-result: - needs: [check-prior-success, gpu-integ-tests, gpu-integ-tests-us-east-1, import-model-integ-tests] - # Only emit the daily alarm metric for scheduled runs that actually executed - # the test jobs: - # - check-prior-success.result == 'success': if the gate job itself failed, - # the test jobs are skipped; without this guard always() would still run - # report-result and read those skips as a (false) failure -> emit 1. - # - already_succeeded != 'true': an earlier run today already passed, so the - # gate short-circuited this run; nothing to report. - if: always() && needs.check-prior-success.result == 'success' && needs.check-prior-success.outputs.already_succeeded != 'true' - runs-on: ubuntu-latest - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }} - aws-region: us-west-2 - - name: Emit run-level pass/fail metric - run: | - # Manual (workflow_dispatch) runs must not contribute to the daily - # GpuIntegRunFailure count that drives GpuIntegRunAlarm; only scheduled - # runs count toward the "all of today's scheduled runs failed" alarm. - if [ "${{ github.event_name }}" != "schedule" ]; then - echo "Not a scheduled run (${{ github.event_name }}); skipping metric emission." - exit 0 - fi - if [ "${{ needs.gpu-integ-tests.result }}" == "success" ] && \ - [ "${{ needs.gpu-integ-tests-us-east-1.result }}" == "success" ] && \ - [ "${{ needs.import-model-integ-tests.result }}" == "success" ]; then - value=0 - echo "All region/import jobs succeeded; emitting GpuIntegRunFailure=0" - else - value=1 - echo "At least one region/import job did not succeed; emitting GpuIntegRunFailure=1" - fi - aws cloudwatch put-metric-data \ - --namespace GpuIntegRunMetrics \ - --metric-name GpuIntegRunFailure \ - --value "$value" \ - --unit Count