Add test to detect memory leak

edewata · claude · edewata · commit c8cdf60c1139 · 2026-07-27T23:20:17.000-05:00
Add script to compare Java heap snapshots for memory leak analysis

Co-Authored-By: Claude Sonnet 4.5 &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/server-memory-leak-test.yml b/.github/workflows/server-memory-leak-test.yml
@@ -0,0 +1,159 @@
+name: Server memory leak test
+
+on: workflow_call
+
+env:
+  DS_IMAGE: ${{ vars.DS_IMAGE || 'quay.io/389ds/dirsrv' }}
+
+jobs:
+  # docs/installation/ca/Installing_CA.md
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    env:
+      SHARED: /tmp/workdir/pki
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v7
+
+      - name: Retrieve PKI images
+        uses: actions/cache@v5
+        with:
+          key: pki-images-${{ github.sha }}
+          path: pki-images.tar
+
+      - name: Load PKI images
+        run: docker load --input pki-images.tar
+
+      - name: Create network
+        run: docker network create example
+
+      - name: Set up DS container
+        run: |
+          tests/bin/ds-create.sh \
+              --image=${{ env.DS_IMAGE }} \
+              --hostname=ds.example.com \
+              --network=example \
+              --network-alias=ds.example.com \
+              --password=Secret.123 \
+              ds
+
+      - name: Set up PKI container
+        run: |
+          tests/bin/runner-init.sh \
+              --hostname=pki.example.com \
+              --network=example \
+              --network-alias=pki.example.com \
+              pki
+
+          docker exec pki rpm -qa | grep openjdk
+          docker exec pki dnf install -y java-25-openjdk-devel
+
+      - name: Install CA
+        run: |
+          docker exec pki pkispawn \
+              -f /usr/share/pki/server/examples/installation/ca.cfg \
+              -s CA \
+              -D pki_ds_url=ldap://ds.example.com:3389 \
+              --debug \
+              > >(tee stdout) 2> >(tee stderr >&2)
+
+          PID=$(docker exec pki ps -C java -o pid --no-headers | awk '{print $1;}')
+          echo "PID=$PID"
+          echo "$PID" > java.pid
+
+      - name: Check heap info
+        run: |
+          PID=$(cat java.pid)
+
+          docker exec pki jhsdb jmap --heap --pid $PID
+
+      - name: Check object counts before load test
+        run: |
+          PID=$(cat java.pid)
+
+          docker exec pki jhsdb jmap --histo --pid $PID \
+              | tail -n +9 \
+              | grep -E 'org.dogtagpki|org.mozilla' \
+              | awk '{print $4, $2}' \
+              | sort \
+              | tee objects.before
+
+      - name: Run load test
+        run: |
+          PID=$(cat java.pid)
+
+          docker exec pki pki-server cert-export \
+              --cert-file ca_signing.crt \
+              ca_signing
+
+          docker exec pki pki nss-cert-import \
+              --cert ca_signing.crt \
+              --trust CT,C,C \
+              ca_signing
+
+          # run garbage collection before test
+          docker exec pki jcmd $PID GC.run
+
+          # run pki info command 100 times
+          for i in $(seq 1 100); do
+              docker exec pki pki info
+          done
+
+          # run garbage collection after test
+          docker exec pki jcmd $PID GC.run
+
+      - name: Check object counts after load test
+        run: |
+          PID=$(cat java.pid)
+
+          # get object class names and number of instances
+          docker exec pki jhsdb jmap --histo --pid $PID \
+              | tail -n +9 \
+              | grep -E 'org.dogtagpki|org.mozilla' \
+              | awk '{print $4, $2}' \
+              | sort \
+              | tee objects.after
+
+      - name: Check object count growths
+        run: |
+          tests/bin/check-memory-leak.py objects.before objects.after
+
+          # if the object count increased by 100 times (i.e. matching
+          # the load test) it's likely that the object is not released
+          # properly for garbage collection and causing a memory leak
+
+          # TODO: fail the test if there are objects that increased by
+          # 100 times
+
+      - name: Remove CA
+        run: |
+          docker exec pki pkidestroy \
+              -s CA \
+              --debug \
+              > >(tee stdout) 2> >(tee stderr >&2)
+
+      - name: Check DS server systemd journal
+        if: always()
+        run: |
+          docker exec ds journalctl -x --no-pager -u dirsrv@localhost.service
+
+      - name: Check DS container logs
+        if: always()
+        run: |
+          docker logs ds
+
+      - name: Check PKI server systemd journal
+        if: always()
+        run: |
+          docker exec pki journalctl -x --no-pager -u pki-tomcatd@pki-tomcat.service
+
+      - name: Check PKI server access log
+        if: always()
+        run: |
+          docker exec pki find /var/log/pki/pki-tomcat -name "localhost_access_log.*" -exec cat {} \;
+
+      - name: Check CA debug log
+        if: always()
+        run: |
+          docker exec pki find /var/lib/pki/pki-tomcat/logs/ca -name "debug.*" -exec cat {} \;
diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml
@@ -53,6 +53,11 @@ jobs:
     needs: build
     uses: ./.github/workflows/server-https-kryoptic-pqc-test.yml
 
+  server-memory-leak-test:
+    name: Server memory leak
+    needs: build
+    uses: ./.github/workflows/server-memory-leak-test.yml
+
   server-backup-test:
     name: Server backup
     needs: build
diff --git a/tests/bin/check-memory-leak.py b/tests/bin/check-memory-leak.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python3
+
+import argparse
+import sys
+from typing import Dict, Tuple, List
+
+
+def parse_input(filename: str) -> Dict[str, int]:
+    """Parse input file and return a dictionary of class name to object count."""
+    data = {}
+    try:
+        with open(filename, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                parts = line.split()
+                if len(parts) != 2:
+                    print(f"Error: File '{filename}' line {line_num} does not contain exactly 2 columns", file=sys.stderr)
+                    sys.exit(1)
+                class_name, count = parts
+                try:
+                    data[class_name] = int(count)
+                except ValueError:
+                    print(f"Error: File '{filename}' line {line_num} has invalid count value: {count}", file=sys.stderr)
+                    sys.exit(1)
+    except FileNotFoundError:
+        print(f"Error: File '{filename}' not found", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading file '{filename}': {e}", file=sys.stderr)
+        sys.exit(1)
+    return data
+
+
+def calculate_growth(before: Dict[str, int], after: Dict[str, int]) -> List[Tuple[str, int, int, float]]:
+    """Calculate the growth between two snapshots.
+
+    Returns a list of tuples: (class_name, orig_count, growth, growth_pct)
+    """
+    growth_data = []
+
+    # Find all classes present in either files
+    all_classes = set(before.keys()) | set(after.keys())
+
+    for class_name in all_classes:
+        orig_count = before.get(class_name, 0)
+        growth = after.get(class_name, 0) - orig_count
+
+        # Calculate percentage growth
+        if orig_count == 0:
+            # New class appeared in second file
+            if growth > 0:
+                growth_pct = float('inf')
+            else:
+                continue
+        else:
+            growth_pct = (growth / orig_count) * 100
+
+        # Only include classes that have grown
+        if growth > 0:
+            growth_data.append((class_name, orig_count, growth, growth_pct))
+
+    return growth_data
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Compare object counts to identify potential memory leak'
+    )
+    parser.add_argument('before', help='First input file')
+    parser.add_argument('after', help='Second input file')
+
+    args = parser.parse_args()
+
+    # Parse both params
+    before = parse_input(args.before)
+    after = parse_input(args.after)
+
+    if not before:
+        print("Error: First input file is empty or invalid", file=sys.stderr)
+        sys.exit(1)
+
+    if not after:
+        print("Error: Second input file is empty or invalid", file=sys.stderr)
+        sys.exit(1)
+
+    # Calculate growth
+    growth_data = calculate_growth(before, after)
+
+    if not growth_data:
+        print("No objects with increased counts found")
+        return
+
+    # Sort by growth percentage (descending)
+    growth_data.sort(key=lambda x: (x[3] != float('inf'), x[3]), reverse=True)
+
+    # Display growth
+    print(f"{'Class Name':<60} {'Original':>12} {'Growth':>12} {'Percentage':>12}")
+    print("=" * 100)
+
+    for i, (class_name, orig_count, growth, growth_pct) in enumerate(growth_data):
+        if growth_pct == float('inf'):
+            growth_str = "NEW"
+        else:
+            growth_str = f"{growth_pct:+.1f}%"
+
+        # Truncate long class names
+        display_name = class_name[:60] if len(class_name) <= 60 else class_name[:57] + "..."
+
+        print(f"{display_name:<60} {orig_count:>12,} {growth:>12,} {growth_str:>12}")
+
+
+if __name__ == '__main__':
+    main()
