Skip to content

Commit 155e484

Browse files
abnobdossabnobdoss
committed
ci: share sccache setup across workflows
1 parent c5a5b5c commit 155e484

8 files changed

Lines changed: 199 additions & 186 deletions

File tree

‎.github/actions/save-sccache/action.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Save sccache
19+
description: Show sccache stats and save the cache on main branch runs after setup-sccache.
20+
21+
inputs:
22+
key-prefix:
23+
description: Cache key prefix without the run id or Windows compiler suffix.
24+
required: true
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- name: Show sccache stats
30+
if: always()
31+
shell: bash
32+
run: sccache --show-stats
33+
34+
- name: Save sccache cache
35+
if: ${{ github.ref == 'refs/heads/main' }}
36+
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
37+
with:
38+
path: ${{ github.workspace }}/.sccache
39+
key: ${{ inputs.key-prefix }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }}

‎.github/actions/sccache/resolve-msvc-key-suffix.ps1‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Resolve the MSVC toolset version for use as an sccache cache key suffix.
19+
# GitHub runner images roll out new cl.exe builds roughly weekly; keying the
20+
# cache on the toolset version avoids silent misses after each bump.
21+
22+
$PSNativeCommandUseErrorActionPreference = $false
23+
24+
function Resolve-Suffix {
25+
# Locate vswhere, which ships with every VS installation.
26+
$vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe'
27+
if (-not (Test-Path $vswhere)) { return $null }
28+
29+
# Ask vswhere for the newest installation that includes the C++ toolset.
30+
$installationPath = (& $vswhere -latest -products * `
31+
-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
32+
-property installationPath | Select-Object -First 1)
33+
if (-not $installationPath) { return $null }
34+
35+
# Read the default toolset version (e.g. "14.44.35217").
36+
$versionFile = Join-Path $installationPath.Trim() 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt'
37+
if (-not (Test-Path $versionFile)) { return $null }
38+
39+
$version = (Get-Content -Path $versionFile -Raw).Trim()
40+
if ($version) { return "-$version" }
41+
return $null
42+
}
43+
44+
$suffix = Resolve-Suffix
45+
if (-not $suffix) {
46+
# Degrade gracefully: a shared bucket is better than a failed build.
47+
$suffix = '-unknown'
48+
Write-Host "::warning::could not resolve MSVC toolset version for sccache key, using '$suffix'"
49+
}
50+
51+
# Export to both step outputs (for setup-sccache's restore key) and env (for
52+
# save-sccache's save key later in the same job).
53+
Add-Content -Path $env:GITHUB_OUTPUT -Value "suffix=$suffix"
54+
Add-Content -Path $env:GITHUB_ENV -Value "SCCACHE_KEY_SUFFIX=$suffix"
55+
Write-Host "Resolved SCCACHE_KEY_SUFFIX=$suffix"

‎.github/actions/setup-sccache/action.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Set up sccache
19+
description: Configure sccache, restore its cache, and start the sccache action.
20+
21+
inputs:
22+
key-prefix:
23+
description: Cache key prefix without the run id or Windows compiler suffix.
24+
required: true
25+
cache-size:
26+
description: Maximum sccache cache size.
27+
required: false
28+
default: 2G
29+
30+
runs:
31+
using: composite
32+
steps:
33+
- name: Configure sccache environment
34+
shell: bash
35+
run: |
36+
echo "SCCACHE_DIR=${{ github.workspace }}/.sccache" >> "$GITHUB_ENV"
37+
echo "SCCACHE_CACHE_SIZE=${{ inputs.cache-size }}" >> "$GITHUB_ENV"
38+
echo "SCCACHE_KEY_SUFFIX=" >> "$GITHUB_ENV"
39+
40+
# Windows caches include the MSVC toolset version because GitHub runner image
41+
# rollouts can change the compiler while keeping the same image label.
42+
- name: Resolve MSVC toolset version for sccache key
43+
if: ${{ runner.os == 'Windows' }}
44+
id: msvc-key
45+
shell: pwsh
46+
run: '& "${{ github.action_path }}/../sccache/resolve-msvc-key-suffix.ps1"'
47+
48+
- name: Restore sccache cache
49+
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
50+
with:
51+
path: ${{ github.workspace }}/.sccache
52+
key: ${{ inputs.key-prefix }}${{ steps.msvc-key.outputs.suffix }}-${{ github.run_id }}
53+
restore-keys: |
54+
${{ inputs.key-prefix }}${{ steps.msvc-key.outputs.suffix }}-
55+
56+
- name: Setup sccache
57+
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10

‎.github/workflows/aws_test.yml‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,6 @@ jobs:
7373
AWS_DEFAULT_REGION: us-east-1
7474
AWS_ENDPOINT_URL: http://127.0.0.1:9000
7575
AWS_EC2_METADATA_DISABLED: "TRUE"
76-
SCCACHE_DIR: ${{ github.workspace }}/.sccache
77-
SCCACHE_CACHE_SIZE: "2G"
7876
steps:
7977
- name: Checkout iceberg-cpp
8078
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -114,26 +112,16 @@ jobs:
114112
if: ${{ matrix.s3 == 'ON' }}
115113
shell: bash
116114
run: bash ci/scripts/start_minio.sh
117-
- name: Restore sccache cache
118-
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
115+
- name: Set up sccache
116+
uses: ./.github/actions/setup-sccache
119117
with:
120-
path: ${{ github.workspace }}/.sccache
121-
key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }}
122-
restore-keys: |
123-
sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-
124-
- name: Setup sccache
125-
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
118+
key-prefix: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}
126119
- name: Build and test Iceberg
127120
shell: bash
128121
env:
129122
CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
130123
run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }}
131-
- name: Show sccache stats
132-
shell: bash
133-
run: sccache --show-stats
134-
- name: Save sccache cache
135-
if: github.ref == 'refs/heads/main'
136-
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
124+
- name: Save sccache
125+
uses: ./.github/actions/save-sccache
137126
with:
138-
path: ${{ github.workspace }}/.sccache
139-
key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }}
127+
key-prefix: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}

‎.github/workflows/cpp-linter.yml‎

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,6 @@ jobs:
4545
permissions:
4646
contents: read
4747
pull-requests: write
48-
env:
49-
SCCACHE_DIR: ${{ github.workspace }}/.sccache
50-
SCCACHE_CACHE_SIZE: "2G"
5148
steps:
5249
- name: Checkout iceberg-cpp
5350
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -58,15 +55,10 @@ jobs:
5855
run: |
5956
sudo apt-get update
6057
sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev
61-
- name: Restore sccache cache
62-
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
58+
- name: Set up sccache
59+
uses: ./.github/actions/setup-sccache
6360
with:
64-
path: ${{ github.workspace }}/.sccache
65-
key: sccache-cpp-linter-ubuntu-${{ github.run_id }}
66-
restore-keys: |
67-
sccache-cpp-linter-ubuntu-
68-
- name: Setup sccache
69-
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
61+
key-prefix: sccache-cpp-linter-ubuntu
7062
- name: Run build
7163
env:
7264
CC: gcc-14
@@ -82,15 +74,10 @@ jobs:
8274
-DICEBERG_SQL_POSTGRESQL=ON \
8375
-DICEBERG_SQL_MYSQL=ON
8476
cmake --build .
85-
- name: Show sccache stats
86-
shell: bash
87-
run: sccache --show-stats
88-
- name: Save sccache cache
89-
if: github.ref == 'refs/heads/main'
90-
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
77+
- name: Save sccache
78+
uses: ./.github/actions/save-sccache
9179
with:
92-
path: ${{ github.workspace }}/.sccache
93-
key: sccache-cpp-linter-ubuntu-${{ github.run_id }}
80+
key-prefix: sccache-cpp-linter-ubuntu
9481
- uses: cpp-linter/cpp-linter-action@0f6d1b8d7e38b584cbee606eb23d850c217d54f8 # v2.15.1
9582
id: linter
9683
if: github.event_name == 'pull_request'

‎.github/workflows/sanitizer_test.yml‎

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,6 @@ jobs:
3838
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
3939
name: "ASAN and UBSAN Tests"
4040
runs-on: ubuntu-26.04
41-
env:
42-
SCCACHE_DIR: ${{ github.workspace }}/.sccache
43-
SCCACHE_CACHE_SIZE: "2G"
4441
steps:
4542
- name: Checkout iceberg-cpp
4643
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -49,15 +46,10 @@ jobs:
4946
- name: Install dependencies
5047
shell: bash
5148
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
52-
- name: Restore sccache cache
53-
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
49+
- name: Set up sccache
50+
uses: ./.github/actions/setup-sccache
5451
with:
55-
path: ${{ github.workspace }}/.sccache
56-
key: sccache-sanitizer-ubuntu-${{ github.run_id }}
57-
restore-keys: |
58-
sccache-sanitizer-ubuntu-
59-
- name: Setup sccache
60-
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
52+
key-prefix: sccache-sanitizer-ubuntu
6153
- name: Configure and Build with ASAN & UBSAN
6254
env:
6355
CC: gcc-14
@@ -67,15 +59,10 @@ jobs:
6759
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \
6860
-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
6961
cmake --build . --verbose
70-
- name: Show sccache stats
71-
shell: bash
72-
run: sccache --show-stats
73-
- name: Save sccache cache
74-
if: github.ref == 'refs/heads/main'
75-
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
62+
- name: Save sccache
63+
uses: ./.github/actions/save-sccache
7664
with:
77-
path: ${{ github.workspace }}/.sccache
78-
key: sccache-sanitizer-ubuntu-${{ github.run_id }}
65+
key-prefix: sccache-sanitizer-ubuntu
7966
- name: Run Tests
8067
working-directory: build
8168
env:

‎.github/workflows/sql_catalog_test.yml‎

Lines changed: 6 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,6 @@ jobs:
6060
runs-on: windows-2025
6161
cmake_build_type: Debug
6262
cmake_extra_args: -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake
63-
env:
64-
SCCACHE_DIR: ${{ github.workspace }}/.sccache
65-
SCCACHE_CACHE_SIZE: "2G"
6663
steps:
6764
- name: Checkout iceberg-cpp
6865
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -73,26 +70,6 @@ jobs:
7370
uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
7471
with:
7572
arch: x64
76-
# GitHub patches the Windows image roughly weekly via gradual rollouts, so
77-
# back-to-back runs can hit different cl.exe builds. sccache keys on the compiler,
78-
# so a shared key would miss after each bump; the version keeps caches separate.
79-
# Non-Windows legs get an empty suffix, leaving their keys unchanged.
80-
- name: Resolve MSVC version for sccache key
81-
if: ${{ startsWith(matrix.runs-on, 'windows') }}
82-
shell: pwsh
83-
run: |
84-
$PSNativeCommandUseErrorActionPreference = $false
85-
$banner = (cl.exe 2>&1 | Out-String)
86-
if ($banner -match 'Version ([\d.]+)') {
87-
$suffix = "-$($Matches[1])"
88-
} else {
89-
# The key is only a cache hint, so degrade to a shared bucket rather than
90-
# fail the build; the warning flags that the parse needs fixing.
91-
$suffix = '-unknown'
92-
Write-Host "::warning::could not parse cl.exe version for sccache key, using '$suffix'; banner was: $banner"
93-
}
94-
Add-Content -Path $env:GITHUB_ENV -Value "SCCACHE_KEY_SUFFIX=$suffix"
95-
Write-Host "SCCACHE_KEY_SUFFIX=$suffix"
9673
- name: Install dependencies on Ubuntu
9774
if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
9875
shell: bash
@@ -114,15 +91,10 @@ jobs:
11491
shell: pwsh
11592
run: |
11693
vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows
117-
- name: Restore sccache cache
118-
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
94+
- name: Set up sccache
95+
uses: ./.github/actions/setup-sccache
11996
with:
120-
path: ${{ github.workspace }}/.sccache
121-
key: sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }}
122-
restore-keys: |
123-
sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}-
124-
- name: Setup sccache
125-
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
97+
key-prefix: sccache-sqlcatalog-${{ matrix.runs-on }}
12698
- name: Configure Iceberg
12799
shell: bash
128100
run: |
@@ -140,15 +112,10 @@ jobs:
140112
- name: Build SQL catalog tests
141113
shell: bash
142114
run: cmake --build build --target sql_catalog_test
143-
- name: Show sccache stats
144-
shell: bash
145-
run: sccache --show-stats
146-
- name: Save sccache cache
147-
if: github.ref == 'refs/heads/main'
148-
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
115+
- name: Save sccache
116+
uses: ./.github/actions/save-sccache
149117
with:
150-
path: ${{ github.workspace }}/.sccache
151-
key: sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }}
118+
key-prefix: sccache-sqlcatalog-${{ matrix.runs-on }}
152119
- name: Run SQL catalog tests
153120
shell: bash
154121
run: ctest --test-dir build -R '^sql_catalog_test$' --output-on-failure -C ${{ matrix.cmake_build_type }}

0 commit comments

Comments
 (0)