Skip to content

Commit 2095a6a

Browse files
justsmthjustsmth
committed
Add CI benchmarks workflow
1 parent 616e676 commit 2095a6a

File tree

2 files changed

+269
-0
lines changed

2 files changed

+269
-0
lines changed

‎.github/workflows/benchmarks.yml‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
name: Benchmarks
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
push:
7+
branches: [main]
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
env:
15+
RUST_BACKTRACE: 1
16+
17+
permissions:
18+
contents: read
19+
pull-requests: write
20+
21+
jobs:
22+
benchmark:
23+
name: Benchmark (${{ matrix.target }})
24+
runs-on: ${{ matrix.os }}
25+
strategy:
26+
fail-fast: false
27+
matrix:
28+
include:
29+
- os: ubuntu-22.04
30+
target: x86_64-unknown-linux-gnu
31+
- os: ubuntu-22.04-arm
32+
target: aarch64-unknown-linux-gnu
33+
steps:
34+
- name: Checkout repository
35+
uses: actions/checkout@v4
36+
with:
37+
submodules: recursive
38+
fetch-depth: 0
39+
40+
- name: Install Rust toolchain
41+
uses: dtolnay/rust-toolchain@stable
42+
with:
43+
targets: ${{ matrix.target }}
44+
45+
- name: Install valgrind
46+
run: |
47+
sudo apt-get update
48+
sudo apt-get install -y valgrind
49+
ls -la /usr/include/valgrind/
50+
51+
- name: Cache cargo registry
52+
uses: actions/cache@v4
53+
with:
54+
path: |
55+
~/.cargo/registry
56+
~/.cargo/git
57+
target
58+
key: ${{ runner.os }}-${{ matrix.target }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}
59+
restore-keys: |
60+
${{ runner.os }}-${{ matrix.target }}-cargo-bench-
61+
62+
- name: Build benchmark tool (candidate)
63+
run: cargo build --release -p aws-lc-rs-bench
64+
65+
- name: Checkout baseline (main branch)
66+
if: github.event_name == 'pull_request'
67+
run: |
68+
git worktree add ../baseline origin/${{ github.base_ref }}
69+
cd ../baseline
70+
git submodule update --init --recursive
71+
72+
- name: Build benchmark tool (baseline)
73+
if: github.event_name == 'pull_request'
74+
run: |
75+
cd ../baseline
76+
cargo build --release -p aws-lc-rs-bench
77+
78+
- name: Run baseline benchmarks
79+
if: github.event_name == 'pull_request'
80+
run: |
81+
cd ../baseline
82+
./target/release/aws-lc-rs-bench run-all --output-dir baseline-results
83+
84+
- name: Run candidate benchmarks
85+
run: |
86+
./target/release/aws-lc-rs-bench run-all --output-dir candidate-results
87+
88+
- name: Compare results
89+
if: github.event_name == 'pull_request'
90+
id: compare
91+
run: |
92+
./target/release/aws-lc-rs-bench compare ../baseline/baseline-results candidate-results > benchmark-report-${{ matrix.target }}.md
93+
cat benchmark-report-${{ matrix.target }}.md >> $GITHUB_STEP_SUMMARY
94+
95+
- name: Upload benchmark results
96+
uses: actions/upload-artifact@v4
97+
with:
98+
name: benchmark-results-${{ matrix.target }}
99+
path: |
100+
candidate-results/
101+
benchmark-report-${{ matrix.target }}.md
102+
retention-days: 30
103+
104+
- name: Store main branch results
105+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
106+
uses: actions/upload-artifact@v4
107+
with:
108+
name: main-benchmark-results-${{ matrix.target }}
109+
path: candidate-results/
110+
retention-days: 90
111+
112+
benchmark-fips:
113+
name: Benchmark FIPS (${{ matrix.target }})
114+
runs-on: ${{ matrix.os }}
115+
strategy:
116+
fail-fast: false
117+
matrix:
118+
include:
119+
- os: ubuntu-22.04
120+
target: x86_64-unknown-linux-gnu
121+
- os: ubuntu-22.04-arm
122+
target: aarch64-unknown-linux-gnu
123+
steps:
124+
- name: Checkout repository
125+
uses: actions/checkout@v4
126+
with:
127+
submodules: recursive
128+
fetch-depth: 0
129+
130+
- name: Install Rust toolchain
131+
uses: dtolnay/rust-toolchain@stable
132+
with:
133+
targets: ${{ matrix.target }}
134+
135+
- name: Install valgrind
136+
run: |
137+
sudo apt-get update
138+
sudo apt-get install -y valgrind
139+
ls -la /usr/include/valgrind/
140+
141+
- name: Install Go (required for FIPS build)
142+
uses: actions/setup-go@v5
143+
with:
144+
go-version: '1.22'
145+
146+
- name: Cache cargo registry
147+
uses: actions/cache@v4
148+
with:
149+
path: |
150+
~/.cargo/registry
151+
~/.cargo/git
152+
target
153+
key: ${{ runner.os }}-${{ matrix.target }}-cargo-bench-fips-${{ hashFiles('**/Cargo.lock') }}
154+
restore-keys: |
155+
${{ runner.os }}-${{ matrix.target }}-cargo-bench-fips-
156+
157+
- name: Build benchmark tool with FIPS (candidate)
158+
run: cargo build --release -p aws-lc-rs-bench --features fips
159+
160+
- name: Checkout baseline (main branch)
161+
if: github.event_name == 'pull_request'
162+
run: |
163+
git worktree add ../baseline origin/${{ github.base_ref }}
164+
cd ../baseline
165+
git submodule update --init --recursive
166+
167+
- name: Build benchmark tool with FIPS (baseline)
168+
if: github.event_name == 'pull_request'
169+
run: |
170+
cd ../baseline
171+
cargo build --release -p aws-lc-rs-bench --features fips
172+
173+
- name: Run baseline FIPS benchmarks
174+
if: github.event_name == 'pull_request'
175+
run: |
176+
cd ../baseline
177+
./target/release/aws-lc-rs-bench run-all --output-dir baseline-fips-results
178+
179+
- name: Run candidate FIPS benchmarks
180+
run: |
181+
./target/release/aws-lc-rs-bench run-all --output-dir candidate-fips-results
182+
183+
- name: Compare FIPS results
184+
if: github.event_name == 'pull_request'
185+
id: compare
186+
run: |
187+
./target/release/aws-lc-rs-bench compare ../baseline/baseline-fips-results candidate-fips-results > benchmark-fips-report-${{ matrix.target }}.md
188+
cat benchmark-fips-report-${{ matrix.target }}.md >> $GITHUB_STEP_SUMMARY
189+
190+
- name: Upload FIPS benchmark results
191+
uses: actions/upload-artifact@v4
192+
with:
193+
name: benchmark-fips-results-${{ matrix.target }}
194+
path: |
195+
candidate-fips-results/
196+
benchmark-fips-report-${{ matrix.target }}.md
197+
retention-days: 30
198+
199+
- name: Store main branch FIPS results
200+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
201+
uses: actions/upload-artifact@v4
202+
with:
203+
name: main-benchmark-fips-results-${{ matrix.target }}
204+
path: candidate-fips-results/
205+
retention-days: 90
206+
207+
post-comment:
208+
name: Post PR Comment
209+
needs: [benchmark, benchmark-fips]
210+
if: github.event_name == 'pull_request'
211+
runs-on: ubuntu-latest
212+
steps:
213+
- name: Download all benchmark reports
214+
uses: actions/download-artifact@v4
215+
with:
216+
pattern: benchmark-*-results-*
217+
merge-multiple: true
218+
219+
- name: Combine reports
220+
run: |
221+
echo "<!-- aws-lc-rs-benchmark-results -->" > comment-body.md
222+
echo "" >> comment-body.md
223+
echo "# Benchmark Results" >> comment-body.md
224+
echo "" >> comment-body.md
225+
echo "## Standard Build" >> comment-body.md
226+
echo "" >> comment-body.md
227+
for f in benchmark-report-*.md; do
228+
if [ -f "$f" ]; then
229+
target=$(echo "$f" | sed 's/benchmark-report-\(.*\)\.md/\1/')
230+
echo "### Target: \`$target\`" >> comment-body.md
231+
echo "" >> comment-body.md
232+
cat "$f" >> comment-body.md
233+
echo "" >> comment-body.md
234+
fi
235+
done
236+
echo "## FIPS Build" >> comment-body.md
237+
echo "" >> comment-body.md
238+
for f in benchmark-fips-report-*.md; do
239+
if [ -f "$f" ]; then
240+
target=$(echo "$f" | sed 's/benchmark-fips-report-\(.*\)\.md/\1/')
241+
echo "### Target: \`$target\`" >> comment-body.md
242+
echo "" >> comment-body.md
243+
cat "$f" >> comment-body.md
244+
echo "" >> comment-body.md
245+
fi
246+
done
247+
echo "" >> comment-body.md
248+
echo "---" >> comment-body.md
249+
echo "> **Note**: Benchmark results are measured using CPU instruction counts via Valgrind's callgrind tool." >> comment-body.md
250+
echo "> Changes greater than 2% are considered significant." >> comment-body.md
251+
echo ">" >> comment-body.md
252+
echo "> ✅ = improvement, ⚠️ = regression" >> comment-body.md
253+
254+
- name: Find existing comment
255+
uses: peter-evans/find-comment@v3
256+
id: find-comment
257+
with:
258+
issue-number: ${{ github.event.pull_request.number }}
259+
comment-author: 'github-actions[bot]'
260+
body-includes: '<!-- aws-lc-rs-benchmark-results -->'
261+
262+
- name: Create or update comment
263+
uses: peter-evans/create-or-update-comment@v4
264+
with:
265+
comment-id: ${{ steps.find-comment.outputs.comment-id }}
266+
issue-number: ${{ github.event.pull_request.number }}
267+
body-path: comment-body.md
268+
edit-mode: replace

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
Cargo.lock
88
deps/aws-lc-sys/src/bindings.rs
99
/flamegraph.svg
10+
/results/
1011

1112
# These are backup files generated by rustfmt
1213
**/*.rs.bk

0 commit comments

Comments
 (0)