Skip to content

Commit 51a1a26

Browse files
justsmthjustsmth
committed
Add CI benchmarks workflow
1 parent c520a20 commit 51a1a26

File tree

4 files changed

+271
-4
lines changed

4 files changed

+271
-4
lines changed

‎.github/workflows/analysis.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -137,9 +137,9 @@ jobs:
137137
run: cargo install cargo-udeps
138138

139139
- name: Run cargo udeps
140-
# we only use openssl when the openssl-benchmarks feature is enabled.
141-
# openssl is a dev-dependency so it can't be optional.
142-
run: cargo udeps --workspace --all-targets --features openssl-benchmarks
140+
# ring and openssl are optional dependencies in aws-lc-rs-testing,
141+
# gated by ring-benchmarks and openssl-benchmarks features respectively.
142+
run: cargo udeps --workspace --all-targets --features ring-benchmarks,openssl-benchmarks
143143
env:
144144
RUSTC_WRAPPER: ""
145145

‎.github/workflows/benchmarks.yml‎

Lines changed: 266 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
name: Benchmarks
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
push:
7+
branches: [main]
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
env:
15+
RUST_BACKTRACE: 1
16+
17+
permissions:
18+
contents: read
19+
pull-requests: write
20+
21+
jobs:
22+
benchmark:
23+
name: Benchmark (${{ matrix.target }})
24+
runs-on: ${{ matrix.os }}
25+
strategy:
26+
fail-fast: false
27+
matrix:
28+
include:
29+
- os: ubuntu-22.04
30+
target: x86_64-unknown-linux-gnu
31+
- os: ubuntu-22.04-arm
32+
target: aarch64-unknown-linux-gnu
33+
steps:
34+
- name: Checkout repository
35+
uses: actions/checkout@v4
36+
with:
37+
submodules: recursive
38+
fetch-depth: 0
39+
40+
- name: Install Rust toolchain
41+
uses: dtolnay/rust-toolchain@stable
42+
with:
43+
targets: ${{ matrix.target }}
44+
45+
- name: Install valgrind
46+
run: |
47+
sudo apt-get update
48+
sudo apt-get install -y valgrind
49+
50+
- name: Cache cargo registry
51+
uses: actions/cache@v4
52+
with:
53+
path: |
54+
~/.cargo/registry
55+
~/.cargo/git
56+
target
57+
key: ${{ runner.os }}-${{ matrix.target }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}
58+
restore-keys: |
59+
${{ runner.os }}-${{ matrix.target }}-cargo-bench-
60+
61+
- name: Build benchmark tool (candidate)
62+
run: cargo build --release -p aws-lc-rs-bench
63+
64+
- name: Checkout baseline (main branch)
65+
if: github.event_name == 'pull_request'
66+
run: |
67+
git worktree add ../baseline origin/${{ github.base_ref }}
68+
cd ../baseline
69+
git submodule update --init --recursive
70+
71+
- name: Build benchmark tool (baseline)
72+
if: github.event_name == 'pull_request'
73+
run: |
74+
cd ../baseline
75+
cargo build --release -p aws-lc-rs-bench
76+
77+
- name: Run baseline benchmarks
78+
if: github.event_name == 'pull_request'
79+
run: |
80+
cd ../baseline
81+
./target/release/aws-lc-rs-bench run-all --output-dir baseline-results
82+
83+
- name: Run candidate benchmarks
84+
run: |
85+
./target/release/aws-lc-rs-bench run-all --output-dir candidate-results
86+
87+
- name: Compare results
88+
if: github.event_name == 'pull_request'
89+
id: compare
90+
run: |
91+
./target/release/aws-lc-rs-bench compare ../baseline/baseline-results candidate-results > benchmark-report-${{ matrix.target }}.md
92+
cat benchmark-report-${{ matrix.target }}.md >> $GITHUB_STEP_SUMMARY
93+
94+
- name: Upload benchmark results
95+
uses: actions/upload-artifact@v4
96+
with:
97+
name: benchmark-results-${{ matrix.target }}
98+
path: |
99+
candidate-results/
100+
benchmark-report-${{ matrix.target }}.md
101+
retention-days: 30
102+
103+
- name: Store main branch results
104+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
105+
uses: actions/upload-artifact@v4
106+
with:
107+
name: main-benchmark-results-${{ matrix.target }}
108+
path: candidate-results/
109+
retention-days: 90
110+
111+
benchmark-fips:
112+
name: Benchmark FIPS (${{ matrix.target }})
113+
runs-on: ${{ matrix.os }}
114+
strategy:
115+
fail-fast: false
116+
matrix:
117+
include:
118+
- os: ubuntu-22.04
119+
target: x86_64-unknown-linux-gnu
120+
- os: ubuntu-22.04-arm
121+
target: aarch64-unknown-linux-gnu
122+
steps:
123+
- name: Checkout repository
124+
uses: actions/checkout@v4
125+
with:
126+
submodules: recursive
127+
fetch-depth: 0
128+
129+
- name: Install Rust toolchain
130+
uses: dtolnay/rust-toolchain@stable
131+
with:
132+
targets: ${{ matrix.target }}
133+
134+
- name: Install valgrind
135+
run: |
136+
sudo apt-get update
137+
sudo apt-get install -y valgrind
138+
139+
- name: Install Go (required for FIPS build)
140+
uses: actions/setup-go@v5
141+
with:
142+
go-version: '1.22'
143+
144+
- name: Cache cargo registry
145+
uses: actions/cache@v4
146+
with:
147+
path: |
148+
~/.cargo/registry
149+
~/.cargo/git
150+
target
151+
key: ${{ runner.os }}-${{ matrix.target }}-cargo-bench-fips-${{ hashFiles('**/Cargo.lock') }}
152+
restore-keys: |
153+
${{ runner.os }}-${{ matrix.target }}-cargo-bench-fips-
154+
155+
- name: Build benchmark tool with FIPS (candidate)
156+
run: cargo build --release -p aws-lc-rs-bench --features fips
157+
158+
- name: Checkout baseline (main branch)
159+
if: github.event_name == 'pull_request'
160+
run: |
161+
git worktree add ../baseline origin/${{ github.base_ref }}
162+
cd ../baseline
163+
git submodule update --init --recursive
164+
165+
- name: Build benchmark tool with FIPS (baseline)
166+
if: github.event_name == 'pull_request'
167+
run: |
168+
cd ../baseline
169+
cargo build --release -p aws-lc-rs-bench --features fips
170+
171+
- name: Run baseline FIPS benchmarks
172+
if: github.event_name == 'pull_request'
173+
run: |
174+
cd ../baseline
175+
./target/release/aws-lc-rs-bench run-all --output-dir baseline-fips-results
176+
177+
- name: Run candidate FIPS benchmarks
178+
run: |
179+
./target/release/aws-lc-rs-bench run-all --output-dir candidate-fips-results
180+
181+
- name: Compare FIPS results
182+
if: github.event_name == 'pull_request'
183+
id: compare
184+
run: |
185+
./target/release/aws-lc-rs-bench compare ../baseline/baseline-fips-results candidate-fips-results > benchmark-fips-report-${{ matrix.target }}.md
186+
cat benchmark-fips-report-${{ matrix.target }}.md >> $GITHUB_STEP_SUMMARY
187+
188+
- name: Upload FIPS benchmark results
189+
uses: actions/upload-artifact@v4
190+
with:
191+
name: benchmark-fips-results-${{ matrix.target }}
192+
path: |
193+
candidate-fips-results/
194+
benchmark-fips-report-${{ matrix.target }}.md
195+
retention-days: 30
196+
197+
- name: Store main branch FIPS results
198+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
199+
uses: actions/upload-artifact@v4
200+
with:
201+
name: main-benchmark-fips-results-${{ matrix.target }}
202+
path: candidate-fips-results/
203+
retention-days: 90
204+
205+
post-comment:
206+
name: Post PR Comment
207+
needs: [benchmark, benchmark-fips]
208+
if: github.event_name == 'pull_request'
209+
runs-on: ubuntu-latest
210+
steps:
211+
- name: Download all benchmark reports
212+
uses: actions/download-artifact@v4
213+
with:
214+
pattern: benchmark-*-results-*
215+
merge-multiple: true
216+
217+
- name: Combine reports
218+
run: |
219+
echo "<!-- aws-lc-rs-benchmark-results -->" > comment-body.md
220+
echo "" >> comment-body.md
221+
echo "# Benchmark Results" >> comment-body.md
222+
echo "" >> comment-body.md
223+
echo "## Standard Build" >> comment-body.md
224+
echo "" >> comment-body.md
225+
for f in benchmark-report-*.md; do
226+
if [ -f "$f" ]; then
227+
target=$(echo "$f" | sed 's/benchmark-report-\(.*\)\.md/\1/')
228+
echo "### Target: \`$target\`" >> comment-body.md
229+
echo "" >> comment-body.md
230+
cat "$f" >> comment-body.md
231+
echo "" >> comment-body.md
232+
fi
233+
done
234+
echo "## FIPS Build" >> comment-body.md
235+
echo "" >> comment-body.md
236+
for f in benchmark-fips-report-*.md; do
237+
if [ -f "$f" ]; then
238+
target=$(echo "$f" | sed 's/benchmark-fips-report-\(.*\)\.md/\1/')
239+
echo "### Target: \`$target\`" >> comment-body.md
240+
echo "" >> comment-body.md
241+
cat "$f" >> comment-body.md
242+
echo "" >> comment-body.md
243+
fi
244+
done
245+
echo "" >> comment-body.md
246+
echo "---" >> comment-body.md
247+
echo "> **Note**: Benchmark results are measured using CPU instruction counts via Valgrind's callgrind tool." >> comment-body.md
248+
echo "> Changes greater than 2% are considered significant." >> comment-body.md
249+
echo ">" >> comment-body.md
250+
echo "> ✅ = improvement, ⚠️ = regression" >> comment-body.md
251+
252+
- name: Find existing comment
253+
uses: peter-evans/find-comment@v3
254+
id: find-comment
255+
with:
256+
issue-number: ${{ github.event.pull_request.number }}
257+
comment-author: 'github-actions[bot]'
258+
body-includes: '<!-- aws-lc-rs-benchmark-results -->'
259+
260+
- name: Create or update comment
261+
uses: peter-evans/create-or-update-comment@v4
262+
with:
263+
comment-id: ${{ steps.find-comment.outputs.comment-id }}
264+
issue-number: ${{ github.event.pull_request.number }}
265+
body-path: comment-body.md
266+
edit-mode: replace

‎.github/workflows/tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
run: cargo test ${{ matrix.args }}
5656
- name: Run extra tests
5757
working-directory: ./aws-lc-rs-testing
58-
run: cargo test --all-targets
58+
run: cargo test --all-targets --features ring-benchmarks,openssl-benchmarks
5959

6060
bindgen-test:
6161
if: github.repository_owner == 'aws'

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
Cargo.lock
88
deps/aws-lc-sys/src/bindings.rs
99
/flamegraph.svg
10+
/results/
1011

1112
# These are backup files generated by rustfmt
1213
**/*.rs.bk

0 commit comments

Comments
 (0)