Skip to content

Commit 371e30e

Browse files
lucasjia-awslucasjia-aws
committed
ci: Split CI Health V3 into master and release workflows
Replace ci-health-v3.yml and the standalone gpu-integ-tests.yml with two workflows, and move the GPU integ tests into the V3 suite. - ci-health-v3-master.yml: unit-test-v3, canaries-v3-master, gpu-integ-tests-master (us-west-2 + us-east-1), and import-model-integ-tests. A report-result job ANDs the GPU + import-model jobs into a single per-run GpuIntegMasterRunFailure metric emitted to us-west-2. - ci-health-v3-release.yml: canaries-v3-release, gpu-integ-tests-release (us-west-2 + us-east-1), with report-result ANDing the two region jobs into GpuIntegReleaseRunFailure. Both run once a day (no gate / skip-on-success). Each region job pair triggers the same CodeBuild project name (region differs only by credentials and the region-override buildspec), and the AND-ed run-level metric preserves us-east-1 GPU coverage that a us-west-2-only per-project alarm cannot see. Only scheduled runs emit the metrics; manual dispatch runs are excluded.
1 parent b8b4518 commit 371e30e

4 files changed

Lines changed: 248 additions & 226 deletions

File tree

‎.github/workflows/ci-health-v3-master.yml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
name: CI Health V3 Master
2+
on:
3+
schedule:
4+
# Once a day at 05:00 UTC (9pm PDT / 10pm PST).
5+
- cron: "0 5 * * *"
6+
workflow_dispatch:
7+
8+
permissions:
9+
id-token: write # This is required for requesting the JWT
10+
11+
jobs:
12+
unit-test-v3:
13+
runs-on: ubuntu-latest
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops]
18+
steps:
19+
- name: Configure AWS Credentials
20+
uses: aws-actions/configure-aws-credentials@v4
21+
with:
22+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
23+
aws-region: us-west-2
24+
role-duration-seconds: 10800
25+
- name: Run Unit Tests V3 for ${{ matrix.submodule }}
26+
uses: aws-actions/aws-codebuild-run-build@v1
27+
with:
28+
project-name: sagemaker-python-sdk-ci-health-unit-test-v3
29+
source-version-override: refs/heads/master
30+
env-vars-for-codebuild: |
31+
SUBMODULE
32+
env:
33+
SUBMODULE: ${{ matrix.submodule }}
34+
canaries-v3-master:
35+
runs-on: ubuntu-latest
36+
strategy:
37+
fail-fast: false
38+
matrix:
39+
submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops]
40+
steps:
41+
- name: Configure AWS Credentials
42+
uses: aws-actions/configure-aws-credentials@v4
43+
with:
44+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
45+
aws-region: us-west-2
46+
role-duration-seconds: 10800
47+
- name: Run Canaries V3 master for ${{ matrix.submodule }}
48+
uses: aws-actions/aws-codebuild-run-build@v1
49+
with:
50+
project-name: sagemaker-python-sdk-ci-health-canaries-v3-master
51+
source-version-override: refs/heads/master
52+
env-vars-for-codebuild: |
53+
SUBMODULE
54+
env:
55+
SUBMODULE: ${{ matrix.submodule }}
56+
# GPU integ tests against the master source. Two region jobs trigger the same
57+
# CodeBuild project name (deployed in both regions); only the region/creds and
58+
# the region-override buildspec differ. Their results are AND-ed with
59+
# import-model in report-result into a single per-run pass/fail metric.
60+
gpu-integ-tests-master:
61+
runs-on: ubuntu-latest
62+
steps:
63+
- name: Configure AWS Credentials
64+
uses: aws-actions/configure-aws-credentials@v4
65+
with:
66+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
67+
aws-region: us-west-2
68+
role-duration-seconds: 10800
69+
- name: Run GPU Integ Tests (master)
70+
uses: aws-actions/aws-codebuild-run-build@v1
71+
with:
72+
project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-master
73+
source-version: refs/heads/master
74+
gpu-integ-tests-master-us-east-1:
75+
runs-on: ubuntu-latest
76+
steps:
77+
- name: Configure AWS Credentials (us-east-1)
78+
uses: aws-actions/configure-aws-credentials@v4
79+
with:
80+
role-to-assume: ${{ secrets.CI_AWS_ROLE_US_EAST_1_ARN }}
81+
aws-region: us-east-1
82+
role-duration-seconds: 10800
83+
- name: Run GPU Integ Tests (master, us-east-1)
84+
uses: aws-actions/aws-codebuild-run-build@v1
85+
with:
86+
project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-master
87+
source-version: refs/heads/master
88+
# Bedrock model-import integ tests. Run serially (concurrency 1) in their own
89+
# CodeBuild project because the "Concurrent model import jobs" Bedrock quota is
90+
# fixed at 1 and not raisable; running them in parallel (as PR checks did)
91+
# makes them collide and flake. us-west-2 only (no us_east_1-marked tests).
92+
# Folded into the master run-level pass/fail metric alongside the GPU jobs.
93+
import-model-integ-tests:
94+
runs-on: ubuntu-latest
95+
steps:
96+
- name: Configure AWS Credentials
97+
uses: aws-actions/configure-aws-credentials@v4
98+
with:
99+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
100+
aws-region: us-west-2
101+
role-duration-seconds: 10800
102+
- name: Run Bedrock Model-Import Integ Tests
103+
uses: aws-actions/aws-codebuild-run-build@v1
104+
with:
105+
project-name: sagemaker-python-sdk-ci-health-import-model-integ-tests
106+
source-version: refs/heads/master
107+
# Run-level result: the master GPU run is successful only if the us-west-2,
108+
# us-east-1, and import-model jobs all succeeded. Emits GpuIntegMasterRunFailure
109+
# = 1 (failed) / 0 (succeeded) to CloudWatch in us-west-2. The CDK alarm
110+
# (GpuIntegMasterRunAlarm) sums this over a UTC day; it is folded into
111+
# CIHealthCompositeAlarm-V3, which owns ticketing. Only scheduled runs emit the
112+
# metric (manual workflow_dispatch runs are excluded).
113+
report-result:
114+
needs: [gpu-integ-tests-master, gpu-integ-tests-master-us-east-1, import-model-integ-tests]
115+
if: always()
116+
runs-on: ubuntu-latest
117+
steps:
118+
- name: Configure AWS Credentials
119+
uses: aws-actions/configure-aws-credentials@v4
120+
with:
121+
role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }}
122+
aws-region: us-west-2
123+
- name: Emit run-level pass/fail metric
124+
run: |
125+
# Manual (workflow_dispatch) runs must not contribute to the daily
126+
# GpuIntegMasterRunFailure count that drives GpuIntegMasterRunAlarm;
127+
# only scheduled runs count.
128+
if [ "${{ github.event_name }}" != "schedule" ]; then
129+
echo "Not a scheduled run (${{ github.event_name }}); skipping metric emission."
130+
exit 0
131+
fi
132+
if [ "${{ needs.gpu-integ-tests-master.result }}" == "success" ] && \
133+
[ "${{ needs.gpu-integ-tests-master-us-east-1.result }}" == "success" ] && \
134+
[ "${{ needs.import-model-integ-tests.result }}" == "success" ]; then
135+
value=0
136+
echo "All master GPU/import jobs succeeded; emitting GpuIntegMasterRunFailure=0"
137+
else
138+
value=1
139+
echo "At least one master GPU/import job did not succeed; emitting GpuIntegMasterRunFailure=1"
140+
fi
141+
aws cloudwatch put-metric-data \
142+
--namespace GpuIntegRunMetrics \
143+
--metric-name GpuIntegMasterRunFailure \
144+
--value "$value" \
145+
--unit Count

‎.github/workflows/ci-health-v3-release.yml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: CI Health V3 Release
2+
on:
3+
schedule:
4+
# Once a day at 05:00 UTC (9pm PDT / 10pm PST).
5+
- cron: "0 5 * * *"
6+
workflow_dispatch:
7+
8+
permissions:
9+
id-token: write # This is required for requesting the JWT
10+
11+
jobs:
12+
canaries-v3-release:
13+
runs-on: ubuntu-latest
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops]
18+
steps:
19+
- name: Configure AWS Credentials
20+
uses: aws-actions/configure-aws-credentials@v4
21+
with:
22+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
23+
aws-region: us-west-2
24+
role-duration-seconds: 10800
25+
- name: Run Canaries V3 release for ${{ matrix.submodule }}
26+
uses: aws-actions/aws-codebuild-run-build@v1
27+
with:
28+
project-name: sagemaker-python-sdk-ci-health-canaries-v3-release
29+
source-version-override: refs/heads/master
30+
env-vars-for-codebuild: |
31+
SUBMODULE
32+
env:
33+
SUBMODULE: ${{ matrix.submodule }}
34+
# GPU integ tests against the published PyPI release. Two region jobs trigger
35+
# the same CodeBuild project name (deployed in both regions); only the
36+
# region/creds and the region-override buildspec differ. Their results are
37+
# AND-ed in report-result into a single per-run pass/fail metric.
38+
gpu-integ-tests-release:
39+
runs-on: ubuntu-latest
40+
steps:
41+
- name: Configure AWS Credentials
42+
uses: aws-actions/configure-aws-credentials@v4
43+
with:
44+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
45+
aws-region: us-west-2
46+
role-duration-seconds: 10800
47+
- name: Run GPU Integ Tests (release)
48+
uses: aws-actions/aws-codebuild-run-build@v1
49+
with:
50+
project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-release
51+
source-version: refs/heads/master
52+
gpu-integ-tests-release-us-east-1:
53+
runs-on: ubuntu-latest
54+
steps:
55+
- name: Configure AWS Credentials (us-east-1)
56+
uses: aws-actions/configure-aws-credentials@v4
57+
with:
58+
role-to-assume: ${{ secrets.CI_AWS_ROLE_US_EAST_1_ARN }}
59+
aws-region: us-east-1
60+
role-duration-seconds: 10800
61+
- name: Run GPU Integ Tests (release, us-east-1)
62+
uses: aws-actions/aws-codebuild-run-build@v1
63+
with:
64+
project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests-release
65+
source-version: refs/heads/master
66+
# Run-level result: the release GPU run is successful only if both region jobs
67+
# succeeded. Emits GpuIntegReleaseRunFailure = 1 (failed) / 0 (succeeded) to
68+
# CloudWatch in us-west-2. The CDK alarm (GpuIntegReleaseRunAlarm) sums this
69+
# over a UTC day; it is folded into CIHealthCompositeAlarm-V3, which owns
70+
# ticketing. Only scheduled runs emit the metric (manual workflow_dispatch runs
71+
# are excluded).
72+
report-result:
73+
needs: [gpu-integ-tests-release, gpu-integ-tests-release-us-east-1]
74+
if: always()
75+
runs-on: ubuntu-latest
76+
steps:
77+
- name: Configure AWS Credentials
78+
uses: aws-actions/configure-aws-credentials@v4
79+
with:
80+
role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }}
81+
aws-region: us-west-2
82+
- name: Emit run-level pass/fail metric
83+
run: |
84+
# Manual (workflow_dispatch) runs must not contribute to the daily
85+
# GpuIntegReleaseRunFailure count that drives GpuIntegReleaseRunAlarm;
86+
# only scheduled runs count.
87+
if [ "${{ github.event_name }}" != "schedule" ]; then
88+
echo "Not a scheduled run (${{ github.event_name }}); skipping metric emission."
89+
exit 0
90+
fi
91+
if [ "${{ needs.gpu-integ-tests-release.result }}" == "success" ] && \
92+
[ "${{ needs.gpu-integ-tests-release-us-east-1.result }}" == "success" ]; then
93+
value=0
94+
echo "All release GPU jobs succeeded; emitting GpuIntegReleaseRunFailure=0"
95+
else
96+
value=1
97+
echo "At least one release GPU job did not succeed; emitting GpuIntegReleaseRunFailure=1"
98+
fi
99+
aws cloudwatch put-metric-data \
100+
--namespace GpuIntegRunMetrics \
101+
--metric-name GpuIntegReleaseRunFailure \
102+
--value "$value" \
103+
--unit Count

‎.github/workflows/ci-health-v3.yml‎

Lines changed: 0 additions & 77 deletions
This file was deleted.

0 commit comments

Comments
 (0)