Skip to content

Commit fd96f5e

Browse files
edewataclaude
edewata
and
claude
committed
Add test for server memory leak
A script has been added to compare Java heap snapshots to find memory leak in PKI server. A new test has been added to detect memory leak after idle time and after load test. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 1de2daa commit fd96f5e

3 files changed

Lines changed: 370 additions & 0 deletions

File tree

‎.github/workflows/server-memory-leak-test.yml‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
name: Server memory leak test
2+
3+
on: workflow_call
4+
5+
env:
6+
DS_IMAGE: ${{ vars.DS_IMAGE || 'quay.io/389ds/dirsrv' }}
7+
8+
jobs:
9+
# docs/installation/ca/Installing_CA.md
10+
test:
11+
name: Test
12+
runs-on: ubuntu-latest
13+
env:
14+
SHARED: /tmp/workdir/pki
15+
steps:
16+
- name: Clone repository
17+
uses: actions/checkout@v7
18+
19+
- name: Retrieve PKI images
20+
uses: actions/cache@v5
21+
with:
22+
key: pki-images-${{ github.sha }}
23+
path: pki-images.tar
24+
25+
- name: Load PKI images
26+
run: docker load --input pki-images.tar
27+
28+
- name: Create network
29+
run: docker network create example
30+
31+
- name: Set up PKI container
32+
run: |
33+
tests/bin/runner-init.sh \
34+
--hostname=pki.example.com \
35+
--network=example \
36+
--network-alias=pki.example.com \
37+
pki
38+
39+
docker exec pki rpm -qa | grep openjdk
40+
docker exec pki dnf install -y java-25-openjdk-devel
41+
42+
- name: Create PKI server
43+
run: |
44+
docker exec pki pki-server create -v
45+
docker exec pki pki-server nss-create --no-password
46+
47+
- name: Create CA signing cert
48+
run: |
49+
docker exec pki pki-server cert-request \
50+
--subject "CN=CA Signing Certificate" \
51+
--ext /usr/share/pki/server/certs/ca_signing.conf \
52+
ca_signing
53+
54+
docker exec pki pki-server cert-create \
55+
--ext /usr/share/pki/server/certs/ca_signing.conf \
56+
ca_signing
57+
58+
docker exec pki pki-server cert-import ca_signing
59+
60+
- name: Create SSL server cert
61+
run: |
62+
docker exec pki pki-server cert-request \
63+
--subject "CN=pki.example.com" \
64+
--ext /usr/share/pki/server/certs/sslserver.conf \
65+
sslserver
66+
67+
docker exec pki pki-server cert-create \
68+
--issuer ca_signing \
69+
--ext /usr/share/pki/server/certs/sslserver.conf \
70+
sslserver
71+
72+
docker exec pki pki-server cert-import sslserver
73+
74+
- name: Create HTTPS connector
75+
run: |
76+
docker exec pki pki-server jss-enable
77+
78+
docker exec pki pki-server http-connector-add \
79+
--port 8443 \
80+
--scheme https \
81+
--secure true \
82+
--sslEnabled true \
83+
--sslProtocol SSL \
84+
--sslImpl org.dogtagpki.jss.tomcat.JSSImplementation \
85+
Secure
86+
87+
docker exec pki pki-server http-connector-cert-add \
88+
--keyAlias sslserver \
89+
--keystoreType pkcs11 \
90+
--keystoreProvider Mozilla-JSS
91+
92+
- name: Deploy webapps
93+
run: |
94+
docker exec pki pki-server webapp-deploy \
95+
--descriptor /usr/share/pki/server/conf/Catalina/localhost/ROOT.xml \
96+
ROOT
97+
98+
docker exec pki pki-server webapp-deploy \
99+
--descriptor /usr/share/pki/server/conf/Catalina/localhost/pki.xml \
100+
pki
101+
102+
- name: Start PKI server
103+
run: |
104+
docker exec pki pki-server start
105+
106+
PID=$(docker exec pki ps -C java -o pid --no-headers | awk '{print $1;}')
107+
echo "PID=$PID"
108+
echo "$PID" > java.pid
109+
110+
- name: Initialize PKI client
111+
run: |
112+
docker exec pki pki nss-cert-import \
113+
--cert /var/lib/pki/pki-tomcat/conf/certs/ca_signing.crt \
114+
--trust CT,C,C \
115+
ca_signing
116+
117+
- name: Check initial heap
118+
run: |
119+
PID=$(cat java.pid)
120+
121+
# run garbage collector
122+
docker exec pki jcmd $PID GC.run
123+
sleep 5
124+
125+
# get object class names and counts
126+
docker exec pki jhsdb jmap --histo --pid $PID \
127+
| tail -n +9 \
128+
| grep -E 'org.dogtagpki|org.mozilla' \
129+
| awk '{print $4, $2}' \
130+
| sort \
131+
| tee heap1
132+
133+
- name: Idle for 1 minute
134+
run: |
135+
sleep 60
136+
137+
- name: Check heap after idle
138+
run: |
139+
PID=$(cat java.pid)
140+
141+
# run garbage collector
142+
docker exec pki jcmd $PID GC.run
143+
sleep 5
144+
145+
# get object class names and counts
146+
docker exec pki jhsdb jmap --histo --pid $PID \
147+
| tail -n +9 \
148+
| grep -E 'org.dogtagpki|org.mozilla' \
149+
| awk '{print $4, $2}' \
150+
| sort \
151+
| tee heap2
152+
153+
- name: Check memory leak after idle
154+
run: |
155+
tests/bin/check-memory-leak.py heap1 heap2
156+
157+
# if the object count increased it's likely that
158+
# the object was not released properly for garbage
159+
# collection which causes a memory leak
160+
161+
# TODO: test should fail if there's a memory leak
162+
163+
- name: Run load test with HTTP
164+
run: |
165+
PID=$(cat java.pid)
166+
167+
# run pki info command 50 times
168+
for i in $(seq 1 50); do
169+
docker exec pki pki -U http://pki.example.com:8080 info
170+
done
171+
172+
- name: Check heap after load test with HTTP
173+
run: |
174+
PID=$(cat java.pid)
175+
176+
# run garbage collector
177+
docker exec pki jcmd $PID GC.run
178+
sleep 5
179+
180+
# get object class names and counts
181+
docker exec pki jhsdb jmap --histo --pid $PID \
182+
| tail -n +9 \
183+
| grep -E 'org.dogtagpki|org.mozilla' \
184+
| awk '{print $4, $2}' \
185+
| sort \
186+
| tee heap3
187+
188+
- name: Check memory leak after load test with HTTP
189+
run: |
190+
tests/bin/check-memory-leak.py heap2 heap3
191+
192+
# if the object count increased by 50 times (i.e. matching
193+
# the load test) it's likely that the object was not released
194+
# properly for garbage collection which causes a memory leak
195+
196+
# TODO: test should fail if there's a memory leak
197+
198+
- name: Run load test with HTTPS
199+
run: |
200+
PID=$(cat java.pid)
201+
202+
# run pki info command 50 times
203+
for i in $(seq 1 50); do
204+
docker exec pki pki info
205+
done
206+
207+
- name: Check heap after load test with HTTPS
208+
run: |
209+
PID=$(cat java.pid)
210+
211+
# run garbage collector
212+
docker exec pki jcmd $PID GC.run
213+
sleep 5
214+
215+
# get object class names and counts
216+
docker exec pki jhsdb jmap --histo --pid $PID \
217+
| tail -n +9 \
218+
| grep -E 'org.dogtagpki|org.mozilla' \
219+
| awk '{print $4, $2}' \
220+
| sort \
221+
| tee heap4
222+
223+
- name: Check memory leak after load test with HTTPS
224+
run: |
225+
tests/bin/check-memory-leak.py heap3 heap4
226+
227+
# if the object count increased by 50 times (i.e. matching
228+
# the load test) it's likely that the object was not released
229+
# properly for garbage collection which causes a memory leak
230+
231+
# TODO: test should fail if there's a memory leak
232+
233+
- name: Stop PKI server
234+
run: |
235+
docker exec pki pki-server stop --wait -v
236+
237+
- name: Remove PKI server
238+
run: |
239+
docker exec pki pki-server remove -v
240+
241+
- name: Check PKI server systemd journal
242+
if: always()
243+
run: |
244+
docker exec pki journalctl -x --no-pager -u pki-tomcatd@pki-tomcat.service
245+
246+
- name: Check PKI server access log
247+
if: always()
248+
run: |
249+
docker exec pki find /var/log/pki/pki-tomcat -name "localhost_access_log.*" -exec cat {} \;

‎.github/workflows/server-tests.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ jobs:
5353
needs: build
5454
uses: ./.github/workflows/server-https-kryoptic-pqc-test.yml
5555

56+
server-memory-leak-test:
57+
name: Server memory leak
58+
needs: build
59+
uses: ./.github/workflows/server-memory-leak-test.yml
60+
5661
server-backup-test:
5762
name: Server backup
5863
needs: build

‎tests/bin/check-memory-leak.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#!/usr/bin/python3
2+
3+
import argparse
4+
import sys
5+
from typing import Dict, Tuple, List
6+
7+
8+
def parse_input(filename: str) -> Dict[str, int]:
9+
"""Parse input file and return a dictionary of object class name to count."""
10+
data = {}
11+
try:
12+
with open(filename, 'r') as f:
13+
for line in f:
14+
line = line.strip()
15+
if not line:
16+
continue
17+
parts = line.split()
18+
if len(parts) != 2:
19+
print(f"Error: File '{filename}' line {line_num} does not contain exactly 2 columns", file=sys.stderr)
20+
sys.exit(1)
21+
class_name, count = parts
22+
try:
23+
data[class_name] = int(count)
24+
except ValueError:
25+
print(f"Error: File '{filename}' line {line_num} has invalid count value: {count}", file=sys.stderr)
26+
sys.exit(1)
27+
except FileNotFoundError:
28+
print(f"Error: File '{filename}' not found", file=sys.stderr)
29+
sys.exit(1)
30+
except Exception as e:
31+
print(f"Error reading file '{filename}': {e}", file=sys.stderr)
32+
sys.exit(1)
33+
return data
34+
35+
36+
def calculate_changes(snapshot1: Dict[str, int], snapshot2: Dict[str, int]) -> List[Tuple[str, int, int, float]]:
37+
"""Calculate changes between two heap snapshots.
38+
39+
Returns a list of tuples: (class_name, before, after, change, percentage)
40+
"""
41+
results = []
42+
43+
# Find all objects present in either snapshots
44+
objects = set(snapshot1.keys()) | set(snapshot2.keys())
45+
46+
for class_name in objects:
47+
before = snapshot1.get(class_name, 0)
48+
after = snapshot2.get(class_name, 0)
49+
change = after - before
50+
51+
# Calculate percentage of change
52+
if before == 0:
53+
# New object appeared in second file
54+
if change > 0:
55+
percentage = float('inf')
56+
else:
57+
continue
58+
else:
59+
percentage = (change / before) * 100
60+
61+
# Only include classes that have changed
62+
if change > 0:
63+
results.append((class_name, before, after, change, percentage))
64+
65+
return results
66+
67+
68+
def main():
69+
parser = argparse.ArgumentParser(
70+
description='Compare two heap snapshots to detect memory leak'
71+
)
72+
parser.add_argument('snapshot1', help='First heap snapshot')
73+
parser.add_argument('snapshot2', help='Second heao snapshot')
74+
75+
args = parser.parse_args()
76+
77+
# Parse both params
78+
snapshot1 = parse_input(args.snapshot1)
79+
snapshot2 = parse_input(args.snapshot2)
80+
81+
if not snapshot1:
82+
print("Error: First heap snapshot is empty or invalid", file=sys.stderr)
83+
sys.exit(1)
84+
85+
if not snapshot2:
86+
print("Error: Second heap snapshot is empty or invalid", file=sys.stderr)
87+
sys.exit(1)
88+
89+
# Calculate changes
90+
results = calculate_changes(snapshot1, snapshot2)
91+
92+
if not results:
93+
print("No heap changes")
94+
return
95+
96+
# Sort by percentage (descending)
97+
results.sort(key=lambda x: (x[4] != float('inf'), x[4]), reverse=True)
98+
99+
# Display results
100+
print(f"{'Class Name':<60} {'Before':>8} {'After':>8} {'Change':>8} {'Percentage':>11}")
101+
print("=" * 100)
102+
103+
for i, (class_name, before, after, change, percentage) in enumerate(results):
104+
if percentage == float('inf'):
105+
percentage_str = "NEW"
106+
else:
107+
percentage_str = f"{percentage:+.0f}%"
108+
109+
# Truncate long class names
110+
display_name = class_name[:60] if len(class_name) <= 60 else class_name[:57] + "..."
111+
112+
print(f"{display_name:<60} {before:>8} {after:>8} {change:>+8} {percentage_str:>11}")
113+
114+
115+
if __name__ == '__main__':
116+
main()

0 commit comments

Comments
 (0)