Skip to content

Commit 5c4f4b8

Browse files
edewataclaude
edewata
and
claude
committed
Add test for server memory leaks
A script has been added to compare Java heap snapshots to find memory leaks in PKI server. A new test has been added to detect memory leaks after idle time and after load test. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 1de2daa commit 5c4f4b8

3 files changed

Lines changed: 411 additions & 0 deletions

File tree

‎.github/workflows/server-memory-leak-test.yml‎

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,291 @@
1+
name: Server memory leak test
2+
3+
on: workflow_call
4+
5+
env:
6+
DS_IMAGE: ${{ vars.DS_IMAGE || 'quay.io/389ds/dirsrv' }}
7+
8+
jobs:
9+
# docs/installation/ca/Installing_CA.md
10+
test:
11+
name: Test
12+
runs-on: ubuntu-latest
13+
env:
14+
SHARED: /tmp/workdir/pki
15+
steps:
16+
- name: Clone repository
17+
uses: actions/checkout@v7
18+
19+
- name: Retrieve PKI images
20+
uses: actions/cache@v5
21+
with:
22+
key: pki-images-${{ github.sha }}
23+
path: pki-images.tar
24+
25+
- name: Load PKI images
26+
run: docker load --input pki-images.tar
27+
28+
- name: Create network
29+
run: docker network create example
30+
31+
- name: Set up DS container
32+
run: |
33+
tests/bin/ds-create.sh \
34+
--image=${{ env.DS_IMAGE }} \
35+
--hostname=ds.example.com \
36+
--network=example \
37+
--network-alias=ds.example.com \
38+
--password=Secret.123 \
39+
ds
40+
41+
- name: Set up PKI container
42+
run: |
43+
tests/bin/runner-init.sh \
44+
--hostname=pki.example.com \
45+
--network=example \
46+
--network-alias=pki.example.com \
47+
pki
48+
49+
docker exec pki rpm -qa | grep openjdk
50+
docker exec pki dnf install -y java-25-openjdk-devel
51+
52+
- name: Create PKI server
53+
run: |
54+
docker exec pki pki-server create -v
55+
56+
- name: Create NSS database in PKI server
57+
run: |
58+
docker exec pki pki-server nss-create --no-password
59+
60+
- name: Create CA signing cert
61+
run: |
62+
# generate CA signing CSR
63+
docker exec pki pki \
64+
-d /var/lib/pki/pki-tomcat/alias \
65+
nss-cert-request \
66+
--key-type MLDSA \
67+
--key-strength 65 \
68+
--subject "CN=CA Signing Certificate" \
69+
--ext /usr/share/pki/server/certs/ca_signing.conf \
70+
--csr $SHARED/ca_signing.csr
71+
72+
# create CA signing cert
73+
docker exec pki pki \
74+
-d /var/lib/pki/pki-tomcat/alias \
75+
nss-cert-issue \
76+
--csr $SHARED/ca_signing.csr \
77+
--ext /usr/share/pki/server/certs/ca_signing.conf \
78+
--validity-length 1 \
79+
--validity-unit year \
80+
--cert $SHARED/ca_signing.crt
81+
82+
# import CA signing cert
83+
docker exec pki pki \
84+
-d /var/lib/pki/pki-tomcat/alias \
85+
nss-cert-import \
86+
--cert $SHARED/ca_signing.crt \
87+
--trust CT,C,C \
88+
ca_signing
89+
90+
- name: Create SSL server cert
91+
run: |
92+
# generate SSL server CSR
93+
docker exec pki pki \
94+
-d /var/lib/pki/pki-tomcat/alias \
95+
nss-cert-request \
96+
--key-type MLDSA \
97+
--key-strength 65 \
98+
--subject "CN=pki.example.com" \
99+
--ext /usr/share/pki/server/certs/sslserver.conf \
100+
--csr $SHARED/sslserver.csr
101+
102+
# issue SSL server cert that expires in 2 minutes
103+
docker exec pki pki \
104+
-d /var/lib/pki/pki-tomcat/alias \
105+
nss-cert-issue \
106+
--issuer ca_signing \
107+
--csr $SHARED/sslserver.csr \
108+
--ext /usr/share/pki/server/certs/sslserver.conf \
109+
--validity-length 2 \
110+
--validity-unit minute \
111+
--cert $SHARED/sslserver.crt
112+
113+
# import SSL server cert
114+
docker exec pki pki \
115+
-d /var/lib/pki/pki-tomcat/alias \
116+
nss-cert-import \
117+
--cert $SHARED/sslserver.crt \
118+
sslserver
119+
120+
- name: Create HTTPS connector with NSS database
121+
run: |
122+
docker exec pki pki-server jss-enable
123+
docker exec pki pki-server http-connector-add \
124+
--port 8443 \
125+
--scheme https \
126+
--secure true \
127+
--sslEnabled true \
128+
--sslProtocol SSL \
129+
--sslImpl org.dogtagpki.jss.tomcat.JSSImplementation \
130+
Secure
131+
docker exec pki pki-server http-connector-cert-add \
132+
--keyAlias sslserver \
133+
--keystoreType pkcs11 \
134+
--keystoreProvider Mozilla-JSS
135+
136+
- name: Deploy webapps
137+
run: |
138+
docker exec pki pki-server webapp-deploy \
139+
--descriptor /usr/share/pki/server/conf/Catalina/localhost/ROOT.xml \
140+
ROOT
141+
142+
docker exec pki pki-server webapp-deploy \
143+
--descriptor /usr/share/pki/server/conf/Catalina/localhost/pki.xml \
144+
pki
145+
146+
- name: Start PKI server
147+
run: |
148+
docker exec pki pki-server start
149+
150+
- name: Install CA
151+
run: |
152+
#docker exec pki pkispawn \
153+
# -f /usr/share/pki/server/examples/installation/ca.cfg \
154+
# -s CA \
155+
# -D pki_ds_url=ldap://ds.example.com:3389 \
156+
# --debug \
157+
# > >(tee stdout) 2> >(tee stderr >&2)
158+
159+
PID=$(docker exec pki ps -C java -o pid --no-headers | awk '{print $1;}')
160+
echo "PID=$PID"
161+
echo "$PID" > java.pid
162+
163+
docker exec pki pki-server cert-export \
164+
--cert-file ca_signing.crt \
165+
ca_signing
166+
167+
docker exec pki pki nss-cert-import \
168+
--cert $SHARED/ca_signing.crt \
169+
--trust CT,C,C \
170+
ca_signing
171+
172+
- name: Check initial heap
173+
run: |
174+
PID=$(cat java.pid)
175+
176+
# run garbage collector
177+
docker exec pki jcmd $PID GC.run
178+
sleep 5
179+
180+
# get object class names and counts
181+
docker exec pki jhsdb jmap --histo --pid $PID \
182+
| tail -n +9 \
183+
| grep -E 'org.dogtagpki|org.mozilla' \
184+
| awk '{print $4, $2}' \
185+
| sort \
186+
| tee heap1
187+
188+
- name: Idle for 1 minute
189+
run: |
190+
sleep 60
191+
192+
- name: Check heap after idle
193+
run: |
194+
PID=$(cat java.pid)
195+
196+
# run garbage collector
197+
docker exec pki jcmd $PID GC.run
198+
sleep 5
199+
200+
# get object class names and counts
201+
docker exec pki jhsdb jmap --histo --pid $PID \
202+
| tail -n +9 \
203+
| grep -E 'org.dogtagpki|org.mozilla' \
204+
| awk '{print $4, $2}' \
205+
| sort \
206+
| tee heap2
207+
208+
- name: Check memory leak after idle
209+
run: |
210+
tests/bin/check-memory-leak.py heap1 heap2
211+
212+
# if the object count increased it's likely that
213+
# the object was not released properly for garbage
214+
# collection which causes a memory leak
215+
216+
# TODO: test should fail if there's a memory leak
217+
218+
- name: Run load test
219+
run: |
220+
PID=$(cat java.pid)
221+
222+
# run pki info command 100 times
223+
for i in $(seq 1 100); do
224+
docker exec pki pki info
225+
done
226+
227+
- name: Check heap after load test
228+
run: |
229+
PID=$(cat java.pid)
230+
231+
# run garbage collector
232+
docker exec pki jcmd $PID GC.run
233+
sleep 5
234+
235+
# get object class names and counts
236+
docker exec pki jhsdb jmap --histo --pid $PID \
237+
| tail -n +9 \
238+
| grep -E 'org.dogtagpki|org.mozilla' \
239+
| awk '{print $4, $2}' \
240+
| sort \
241+
| tee heap3
242+
243+
- name: Check memory leak after load test
244+
run: |
245+
tests/bin/check-memory-leak.py heap2 heap3
246+
247+
# if the object count increased by 100 times (i.e. matching
248+
# the load test) it's likely that the object was not released
249+
# properly for garbage collection which causes a memory leak
250+
251+
# TODO: test should fail if there's a memory leak
252+
253+
- name: Remove CA
254+
run: |
255+
#docker exec pki pkidestroy \
256+
# -s CA \
257+
# --debug \
258+
# > >(tee stdout) 2> >(tee stderr >&2)
259+
260+
- name: Stop PKI server
261+
run: |
262+
docker exec pki pki-server stop --wait -v
263+
264+
- name: Remove PKI server
265+
run: |
266+
docker exec pki pki-server remove -v
267+
268+
- name: Check DS server systemd journal
269+
if: always()
270+
run: |
271+
docker exec ds journalctl -x --no-pager -u dirsrv@localhost.service
272+
273+
- name: Check DS container logs
274+
if: always()
275+
run: |
276+
docker logs ds
277+
278+
- name: Check PKI server systemd journal
279+
if: always()
280+
run: |
281+
docker exec pki journalctl -x --no-pager -u pki-tomcatd@pki-tomcat.service
282+
283+
- name: Check PKI server access log
284+
if: always()
285+
run: |
286+
docker exec pki find /var/log/pki/pki-tomcat -name "localhost_access_log.*" -exec cat {} \;
287+
288+
- name: Check CA debug log
289+
if: always()
290+
run: |
291+
docker exec pki find /var/lib/pki/pki-tomcat/logs/ca -name "debug.*" -exec cat {} \;

‎.github/workflows/server-tests.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ jobs:
5353
needs: build
5454
uses: ./.github/workflows/server-https-kryoptic-pqc-test.yml
5555

56+
server-memory-leak-test:
57+
name: Server memory leak
58+
needs: build
59+
uses: ./.github/workflows/server-memory-leak-test.yml
60+
5661
server-backup-test:
5762
name: Server backup
5863
needs: build

0 commit comments

Comments
 (0)