Skip to content

Commit 6bf79ee

Browse files
edewataclaude
edewata
and
claude
committed
Add test for server memory leak
A script has been added to compare Java heap snapshots to find memory leak in PKI server. A new test has been added to detect memory leak after idle time and after load test. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 1de2daa commit 6bf79ee

3 files changed

Lines changed: 391 additions & 0 deletions

File tree

‎.github/workflows/server-memory-leak-test.yml‎

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
name: Server memory leak test
2+
3+
on: workflow_call
4+
5+
env:
6+
DS_IMAGE: ${{ vars.DS_IMAGE || 'quay.io/389ds/dirsrv' }}
7+
8+
jobs:
9+
# docs/installation/ca/Installing_CA.md
10+
test:
11+
name: Test
12+
runs-on: ubuntu-latest
13+
env:
14+
SHARED: /tmp/workdir/pki
15+
steps:
16+
- name: Clone repository
17+
uses: actions/checkout@v7
18+
19+
- name: Retrieve PKI images
20+
uses: actions/cache@v5
21+
with:
22+
key: pki-images-${{ github.sha }}
23+
path: pki-images.tar
24+
25+
- name: Load PKI images
26+
run: docker load --input pki-images.tar
27+
28+
- name: Create network
29+
run: docker network create example
30+
31+
- name: Set up PKI container
32+
run: |
33+
tests/bin/runner-init.sh \
34+
--hostname=pki.example.com \
35+
--network=example \
36+
--network-alias=pki.example.com \
37+
pki
38+
39+
docker exec pki rpm -qa | grep openjdk
40+
docker exec pki dnf install -y java-25-openjdk-devel
41+
42+
- name: Create PKI server
43+
run: |
44+
docker exec pki pki-server create -v
45+
docker exec pki pki-server nss-create --no-password
46+
47+
- name: Create CA signing cert
48+
run: |
49+
docker exec pki pki-server cert-request \
50+
--subject "CN=CA Signing Certificate" \
51+
--ext /usr/share/pki/server/certs/ca_signing.conf \
52+
ca_signing
53+
54+
docker exec pki pki-server cert-create \
55+
--ext /usr/share/pki/server/certs/ca_signing.conf \
56+
ca_signing
57+
58+
docker exec pki ls -l /var/lib/pki/pki-tomcat/conf/certs
59+
60+
docker exec pki pki-server cert-import ca_signing
61+
62+
# generate CA signing CSR
63+
#docker exec pki pki \
64+
# -d /var/lib/pki/pki-tomcat/alias \
65+
# nss-cert-request \
66+
# --key-type MLDSA \
67+
# --key-strength 65 \
68+
# --subject "CN=CA Signing Certificate" \
69+
# --ext /usr/share/pki/server/certs/ca_signing.conf \
70+
# --csr $SHARED/ca_signing.csr
71+
72+
# create CA signing cert
73+
#docker exec pki pki \
74+
# -d /var/lib/pki/pki-tomcat/alias \
75+
# nss-cert-issue \
76+
# --csr $SHARED/ca_signing.csr \
77+
# --ext /usr/share/pki/server/certs/ca_signing.conf \
78+
# --cert $SHARED/ca_signing.crt
79+
80+
# import CA signing cert
81+
#docker exec pki pki \
82+
# -d /var/lib/pki/pki-tomcat/alias \
83+
# nss-cert-import \
84+
# --cert $SHARED/ca_signing.crt \
85+
# --trust CT,C,C \
86+
# ca_signing
87+
88+
- name: Create SSL server cert
89+
run: |
90+
docker exec pki pki-server cert-request \
91+
--subject "CN=pki.example.com" \
92+
--ext /usr/share/pki/server/certs/sslserver.conf \
93+
sslserver
94+
95+
docker exec pki pki-server cert-create \
96+
--issuer ca_signing \
97+
--ext /usr/share/pki/server/certs/sslserver.conf \
98+
sslserver
99+
100+
docker exec pki ls -l /var/lib/pki/pki-tomcat/conf/certs
101+
102+
docker exec pki pki-server cert-import sslserver
103+
104+
# generate SSL server CSR
105+
#docker exec pki pki \
106+
# -d /var/lib/pki/pki-tomcat/alias \
107+
# nss-cert-request \
108+
# --key-type MLDSA \
109+
# --key-strength 65 \
110+
# --subject "CN=pki.example.com" \
111+
# --ext /usr/share/pki/server/certs/sslserver.conf \
112+
# --csr $SHARED/sslserver.csr
113+
114+
# issue SSL server cert
115+
#docker exec pki pki \
116+
# -d /var/lib/pki/pki-tomcat/alias \
117+
# nss-cert-issue \
118+
# --issuer ca_signing \
119+
# --csr $SHARED/sslserver.csr \
120+
# --ext /usr/share/pki/server/certs/sslserver.conf \
121+
# --cert $SHARED/sslserver.crt
122+
123+
# import SSL server cert
124+
#docker exec pki pki \
125+
# -d /var/lib/pki/pki-tomcat/alias \
126+
# nss-cert-import \
127+
# --cert $SHARED/sslserver.crt \
128+
# sslserver
129+
130+
- name: Create HTTPS connector with NSS database
131+
run: |
132+
docker exec pki pki-server jss-enable
133+
134+
docker exec pki pki-server http-connector-add \
135+
--port 8443 \
136+
--scheme https \
137+
--secure true \
138+
--sslEnabled true \
139+
--sslProtocol SSL \
140+
--sslImpl org.dogtagpki.jss.tomcat.JSSImplementation \
141+
Secure
142+
143+
docker exec pki pki-server http-connector-cert-add \
144+
--keyAlias sslserver \
145+
--keystoreType pkcs11 \
146+
--keystoreProvider Mozilla-JSS
147+
148+
- name: Deploy webapps
149+
run: |
150+
docker exec pki pki-server webapp-deploy \
151+
--descriptor /usr/share/pki/server/conf/Catalina/localhost/ROOT.xml \
152+
ROOT
153+
154+
docker exec pki pki-server webapp-deploy \
155+
--descriptor /usr/share/pki/server/conf/Catalina/localhost/pki.xml \
156+
pki
157+
158+
- name: Start PKI server
159+
run: |
160+
docker exec pki pki-server start
161+
162+
PID=$(docker exec pki ps -C java -o pid --no-headers | awk '{print $1;}')
163+
echo "PID=$PID"
164+
echo "$PID" > java.pid
165+
166+
- name: Initialize PKI client
167+
run: |
168+
docker exec pki pki nss-cert-import \
169+
--cert /var/lib/pki/pki-tomcat/conf/certs/ca_signing.crt \
170+
--trust CT,C,C \
171+
ca_signing
172+
173+
- name: Check initial heap
174+
run: |
175+
PID=$(cat java.pid)
176+
177+
# run garbage collector
178+
docker exec pki jcmd $PID GC.run
179+
sleep 5
180+
181+
# get object class names and counts
182+
docker exec pki jhsdb jmap --histo --pid $PID \
183+
| tail -n +9 \
184+
| grep -E 'org.dogtagpki|org.mozilla' \
185+
| awk '{print $4, $2}' \
186+
| sort \
187+
| tee heap1
188+
189+
- name: Idle for 1 minute
190+
run: |
191+
sleep 60
192+
193+
- name: Check heap after idle
194+
run: |
195+
PID=$(cat java.pid)
196+
197+
# run garbage collector
198+
docker exec pki jcmd $PID GC.run
199+
sleep 5
200+
201+
# get object class names and counts
202+
docker exec pki jhsdb jmap --histo --pid $PID \
203+
| tail -n +9 \
204+
| grep -E 'org.dogtagpki|org.mozilla' \
205+
| awk '{print $4, $2}' \
206+
| sort \
207+
| tee heap2
208+
209+
- name: Check memory leak after idle
210+
run: |
211+
tests/bin/check-memory-leak.py heap1 heap2
212+
213+
# if the object count increased it's likely that
214+
# the object was not released properly for garbage
215+
# collection which causes a memory leak
216+
217+
# TODO: test should fail if there's a memory leak
218+
219+
- name: Run load test
220+
run: |
221+
PID=$(cat java.pid)
222+
223+
# run pki info command 50 times
224+
for i in $(seq 1 50); do
225+
docker exec pki pki info
226+
done
227+
228+
- name: Check heap after load test
229+
run: |
230+
PID=$(cat java.pid)
231+
232+
# run garbage collector
233+
docker exec pki jcmd $PID GC.run
234+
sleep 5
235+
236+
# get object class names and counts
237+
docker exec pki jhsdb jmap --histo --pid $PID \
238+
| tail -n +9 \
239+
| grep -E 'org.dogtagpki|org.mozilla' \
240+
| awk '{print $4, $2}' \
241+
| sort \
242+
| tee heap3
243+
244+
- name: Check memory leak after load test
245+
run: |
246+
tests/bin/check-memory-leak.py heap2 heap3
247+
248+
# if the object count increased by 50 times (i.e. matching
249+
# the load test) it's likely that the object was not released
250+
# properly for garbage collection which causes a memory leak
251+
252+
# TODO: test should fail if there's a memory leak
253+
254+
- name: Stop PKI server
255+
run: |
256+
docker exec pki pki-server stop --wait -v
257+
258+
- name: Remove PKI server
259+
run: |
260+
docker exec pki pki-server remove -v
261+
262+
- name: Check PKI server systemd journal
263+
if: always()
264+
run: |
265+
docker exec pki journalctl -x --no-pager -u pki-tomcatd@pki-tomcat.service
266+
267+
- name: Check PKI server access log
268+
if: always()
269+
run: |
270+
docker exec pki find /var/log/pki/pki-tomcat -name "localhost_access_log.*" -exec cat {} \;

‎.github/workflows/server-tests.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ jobs:
5353
needs: build
5454
uses: ./.github/workflows/server-https-kryoptic-pqc-test.yml
5555

56+
server-memory-leak-test:
57+
name: Server memory leak
58+
needs: build
59+
uses: ./.github/workflows/server-memory-leak-test.yml
60+
5661
server-backup-test:
5762
name: Server backup
5863
needs: build

‎tests/bin/check-memory-leak.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#!/usr/bin/python3
2+
3+
import argparse
4+
import sys
5+
from typing import Dict, Tuple, List
6+
7+
8+
def parse_input(filename: str) -> Dict[str, int]:
9+
"""Parse input file and return a dictionary of object class name to count."""
10+
data = {}
11+
try:
12+
with open(filename, 'r') as f:
13+
for line in f:
14+
line = line.strip()
15+
if not line:
16+
continue
17+
parts = line.split()
18+
if len(parts) != 2:
19+
print(f"Error: File '{filename}' line {line_num} does not contain exactly 2 columns", file=sys.stderr)
20+
sys.exit(1)
21+
class_name, count = parts
22+
try:
23+
data[class_name] = int(count)
24+
except ValueError:
25+
print(f"Error: File '{filename}' line {line_num} has invalid count value: {count}", file=sys.stderr)
26+
sys.exit(1)
27+
except FileNotFoundError:
28+
print(f"Error: File '{filename}' not found", file=sys.stderr)
29+
sys.exit(1)
30+
except Exception as e:
31+
print(f"Error reading file '{filename}': {e}", file=sys.stderr)
32+
sys.exit(1)
33+
return data
34+
35+
36+
def calculate_changes(snapshot1: Dict[str, int], snapshot2: Dict[str, int]) -> List[Tuple[str, int, int, float]]:
37+
"""Calculate changes between two heap snapshots.
38+
39+
Returns a list of tuples: (class_name, before, after, change, percentage)
40+
"""
41+
results = []
42+
43+
# Find all objects present in either snapshots
44+
objects = set(snapshot1.keys()) | set(snapshot2.keys())
45+
46+
for class_name in objects:
47+
before = snapshot1.get(class_name, 0)
48+
after = snapshot2.get(class_name, 0)
49+
change = after - before
50+
51+
# Calculate percentage of change
52+
if before == 0:
53+
# New object appeared in second file
54+
if change > 0:
55+
percentage = float('inf')
56+
else:
57+
continue
58+
else:
59+
percentage = (change / before) * 100
60+
61+
# Only include classes that have changed
62+
if change > 0:
63+
results.append((class_name, before, after, change, percentage))
64+
65+
return results
66+
67+
68+
def main():
69+
parser = argparse.ArgumentParser(
70+
description='Compare two heap snapshots to detect memory leak'
71+
)
72+
parser.add_argument('snapshot1', help='First heap snapshot')
73+
parser.add_argument('snapshot2', help='Second heao snapshot')
74+
75+
args = parser.parse_args()
76+
77+
# Parse both params
78+
snapshot1 = parse_input(args.snapshot1)
79+
snapshot2 = parse_input(args.snapshot2)
80+
81+
if not snapshot1:
82+
print("Error: First heap snapshot is empty or invalid", file=sys.stderr)
83+
sys.exit(1)
84+
85+
if not snapshot2:
86+
print("Error: Second heap snapshot is empty or invalid", file=sys.stderr)
87+
sys.exit(1)
88+
89+
# Calculate changes
90+
results = calculate_changes(snapshot1, snapshot2)
91+
92+
if not results:
93+
print("No heap changes")
94+
return
95+
96+
# Sort by percentage (descending)
97+
results.sort(key=lambda x: (x[3] != float('inf'), x[3]), reverse=True)
98+
99+
# Display results
100+
print(f"{'Class Name':<60} {'Before':>10} {'After':>10} {'Change':>8} {'Percentage':>10}")
101+
print("=" * 100)
102+
103+
for i, (class_name, before, after, change, percentage) in enumerate(results):
104+
if percentage == float('inf'):
105+
percentage_str = "NEW"
106+
else:
107+
percentage_str = f"{percentage:+.0f}%"
108+
109+
# Truncate long class names
110+
display_name = class_name[:60] if len(class_name) <= 60 else class_name[:57] + "..."
111+
112+
print(f"{display_name:<60} {before:>10} {after:>10} {change:>+8} {percentage_str:>10}")
113+
114+
115+
if __name__ == '__main__':
116+
main()

0 commit comments

Comments
 (0)