Skip to content

Commit c00d983

Browse files
edewataclaude
edewata
and
claude
committed
Add test for server memory leaks
A script has been added to compare Java heap snapshots to find memory leaks in PKI server. A new test has been added to detect memory leaks after idle time and after load test. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 1de2daa commit c00d983

3 files changed

Lines changed: 407 additions & 0 deletions

File tree

‎.github/workflows/server-memory-leak-test.yml‎

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
name: Server memory leak test
2+
3+
on: workflow_call
4+
5+
env:
6+
DS_IMAGE: ${{ vars.DS_IMAGE || 'quay.io/389ds/dirsrv' }}
7+
8+
jobs:
9+
# docs/installation/ca/Installing_CA.md
10+
test:
11+
name: Test
12+
runs-on: ubuntu-latest
13+
env:
14+
SHARED: /tmp/workdir/pki
15+
steps:
16+
- name: Clone repository
17+
uses: actions/checkout@v7
18+
19+
- name: Retrieve PKI images
20+
uses: actions/cache@v5
21+
with:
22+
key: pki-images-${{ github.sha }}
23+
path: pki-images.tar
24+
25+
- name: Load PKI images
26+
run: docker load --input pki-images.tar
27+
28+
- name: Create network
29+
run: docker network create example
30+
31+
- name: Set up DS container
32+
run: |
33+
tests/bin/ds-create.sh \
34+
--image=${{ env.DS_IMAGE }} \
35+
--hostname=ds.example.com \
36+
--network=example \
37+
--network-alias=ds.example.com \
38+
--password=Secret.123 \
39+
ds
40+
41+
- name: Set up PKI container
42+
run: |
43+
tests/bin/runner-init.sh \
44+
--hostname=pki.example.com \
45+
--network=example \
46+
--network-alias=pki.example.com \
47+
pki
48+
49+
docker exec pki rpm -qa | grep openjdk
50+
docker exec pki dnf install -y java-25-openjdk-devel
51+
52+
- name: Create PKI server
53+
run: |
54+
docker exec pki pki-server create -v
55+
56+
- name: Create NSS database in PKI server
57+
run: |
58+
docker exec pki pki-server nss-create --no-password
59+
60+
- name: Create CA signing cert
61+
run: |
62+
# generate CA signing CSR
63+
docker exec pki pki \
64+
-d /var/lib/pki/pki-tomcat/alias \
65+
nss-cert-request \
66+
--key-type MLDSA \
67+
--key-strength 65 \
68+
--subject "CN=CA Signing Certificate" \
69+
--ext /usr/share/pki/server/certs/ca_signing.conf \
70+
--csr $SHARED/ca_signing.csr
71+
72+
# create CA signing cert
73+
docker exec pki pki \
74+
-d /var/lib/pki/pki-tomcat/alias \
75+
nss-cert-issue \
76+
--csr $SHARED/ca_signing.csr \
77+
--ext /usr/share/pki/server/certs/ca_signing.conf \
78+
--cert $SHARED/ca_signing.crt
79+
80+
# import CA signing cert
81+
docker exec pki pki \
82+
-d /var/lib/pki/pki-tomcat/alias \
83+
nss-cert-import \
84+
--cert $SHARED/ca_signing.crt \
85+
--trust CT,C,C \
86+
ca_signing
87+
88+
- name: Create SSL server cert
89+
run: |
90+
# generate SSL server CSR
91+
docker exec pki pki \
92+
-d /var/lib/pki/pki-tomcat/alias \
93+
nss-cert-request \
94+
--key-type MLDSA \
95+
--key-strength 65 \
96+
--subject "CN=pki.example.com" \
97+
--ext /usr/share/pki/server/certs/sslserver.conf \
98+
--csr $SHARED/sslserver.csr
99+
100+
# issue SSL server cert
101+
docker exec pki pki \
102+
-d /var/lib/pki/pki-tomcat/alias \
103+
nss-cert-issue \
104+
--issuer ca_signing \
105+
--csr $SHARED/sslserver.csr \
106+
--ext /usr/share/pki/server/certs/sslserver.conf \
107+
--cert $SHARED/sslserver.crt
108+
109+
# import SSL server cert
110+
docker exec pki pki \
111+
-d /var/lib/pki/pki-tomcat/alias \
112+
nss-cert-import \
113+
--cert $SHARED/sslserver.crt \
114+
sslserver
115+
116+
- name: Create HTTPS connector with NSS database
117+
run: |
118+
docker exec pki pki-server jss-enable
119+
docker exec pki pki-server http-connector-add \
120+
--port 8443 \
121+
--scheme https \
122+
--secure true \
123+
--sslEnabled true \
124+
--sslProtocol SSL \
125+
--sslImpl org.dogtagpki.jss.tomcat.JSSImplementation \
126+
Secure
127+
docker exec pki pki-server http-connector-cert-add \
128+
--keyAlias sslserver \
129+
--keystoreType pkcs11 \
130+
--keystoreProvider Mozilla-JSS
131+
132+
- name: Deploy webapps
133+
run: |
134+
docker exec pki pki-server webapp-deploy \
135+
--descriptor /usr/share/pki/server/conf/Catalina/localhost/ROOT.xml \
136+
ROOT
137+
138+
docker exec pki pki-server webapp-deploy \
139+
--descriptor /usr/share/pki/server/conf/Catalina/localhost/pki.xml \
140+
pki
141+
142+
- name: Start PKI server
143+
run: |
144+
docker exec pki pki-server start
145+
146+
- name: Install CA
147+
run: |
148+
#docker exec pki pkispawn \
149+
# -f /usr/share/pki/server/examples/installation/ca.cfg \
150+
# -s CA \
151+
# -D pki_ds_url=ldap://ds.example.com:3389 \
152+
# --debug \
153+
# > >(tee stdout) 2> >(tee stderr >&2)
154+
155+
PID=$(docker exec pki ps -C java -o pid --no-headers | awk '{print $1;}')
156+
echo "PID=$PID"
157+
echo "$PID" > java.pid
158+
159+
#docker exec pki pki-server cert-export \
160+
# --cert-file ca_signing.crt \
161+
# ca_signing
162+
163+
docker exec pki pki nss-cert-import \
164+
--cert $SHARED/ca_signing.crt \
165+
--trust CT,C,C \
166+
ca_signing
167+
168+
- name: Check initial heap
169+
run: |
170+
PID=$(cat java.pid)
171+
172+
# run garbage collector
173+
docker exec pki jcmd $PID GC.run
174+
sleep 5
175+
176+
# get object class names and counts
177+
docker exec pki jhsdb jmap --histo --pid $PID \
178+
| tail -n +9 \
179+
| grep -E 'org.dogtagpki|org.mozilla' \
180+
| awk '{print $4, $2}' \
181+
| sort \
182+
| tee heap1
183+
184+
- name: Idle for 1 minute
185+
run: |
186+
sleep 60
187+
188+
- name: Check heap after idle
189+
run: |
190+
PID=$(cat java.pid)
191+
192+
# run garbage collector
193+
docker exec pki jcmd $PID GC.run
194+
sleep 5
195+
196+
# get object class names and counts
197+
docker exec pki jhsdb jmap --histo --pid $PID \
198+
| tail -n +9 \
199+
| grep -E 'org.dogtagpki|org.mozilla' \
200+
| awk '{print $4, $2}' \
201+
| sort \
202+
| tee heap2
203+
204+
- name: Check memory leak after idle
205+
run: |
206+
tests/bin/check-memory-leak.py heap1 heap2
207+
208+
# if the object count increased it's likely that
209+
# the object was not released properly for garbage
210+
# collection which causes a memory leak
211+
212+
# TODO: test should fail if there's a memory leak
213+
214+
- name: Run load test
215+
run: |
216+
PID=$(cat java.pid)
217+
218+
# run pki info command 50 times
219+
for i in $(seq 1 50); do
220+
docker exec pki pki info
221+
done
222+
223+
- name: Check heap after load test
224+
run: |
225+
PID=$(cat java.pid)
226+
227+
# run garbage collector
228+
docker exec pki jcmd $PID GC.run
229+
sleep 5
230+
231+
# get object class names and counts
232+
docker exec pki jhsdb jmap --histo --pid $PID \
233+
| tail -n +9 \
234+
| grep -E 'org.dogtagpki|org.mozilla' \
235+
| awk '{print $4, $2}' \
236+
| sort \
237+
| tee heap3
238+
239+
- name: Check memory leak after load test
240+
run: |
241+
tests/bin/check-memory-leak.py heap2 heap3
242+
243+
# if the object count increased by 50 times (i.e. matching
244+
# the load test) it's likely that the object was not released
245+
# properly for garbage collection which causes a memory leak
246+
247+
# TODO: test should fail if there's a memory leak
248+
249+
- name: Remove CA
250+
run: |
251+
#docker exec pki pkidestroy \
252+
# -s CA \
253+
# --debug \
254+
# > >(tee stdout) 2> >(tee stderr >&2)
255+
256+
- name: Stop PKI server
257+
run: |
258+
docker exec pki pki-server stop --wait -v
259+
260+
- name: Remove PKI server
261+
run: |
262+
docker exec pki pki-server remove -v
263+
264+
- name: Check DS server systemd journal
265+
if: always()
266+
run: |
267+
docker exec ds journalctl -x --no-pager -u dirsrv@localhost.service
268+
269+
- name: Check DS container logs
270+
if: always()
271+
run: |
272+
docker logs ds
273+
274+
- name: Check PKI server systemd journal
275+
if: always()
276+
run: |
277+
docker exec pki journalctl -x --no-pager -u pki-tomcatd@pki-tomcat.service
278+
279+
- name: Check PKI server access log
280+
if: always()
281+
run: |
282+
docker exec pki find /var/log/pki/pki-tomcat -name "localhost_access_log.*" -exec cat {} \;
283+
284+
- name: Check CA debug log
285+
if: always()
286+
run: |
287+
docker exec pki find /var/lib/pki/pki-tomcat/logs/ca -name "debug.*" -exec cat {} \;

‎.github/workflows/server-tests.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ jobs:
5353
needs: build
5454
uses: ./.github/workflows/server-https-kryoptic-pqc-test.yml
5555

56+
server-memory-leak-test:
57+
name: Server memory leak
58+
needs: build
59+
uses: ./.github/workflows/server-memory-leak-test.yml
60+
5661
server-backup-test:
5762
name: Server backup
5863
needs: build

0 commit comments

Comments
 (0)