Skip to content

Commit cc5a0b8

Browse files
FrooodleFrooodle
authored
Cleanup work + stream endpoints to reduce memory usage (Stirling-Tools#6106)
1 parent 702f4e5 commit cc5a0b8

116 files changed

Lines changed: 3013 additions & 1520 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎app/common/src/main/java/stirling/software/common/service/FileStorage.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
package stirling.software.common.service;
22

33
import java.io.BufferedInputStream;
4+
import java.io.BufferedOutputStream;
45
import java.io.IOException;
56
import java.io.InputStream;
7+
import java.io.OutputStream;
68
import java.nio.file.Files;
79
import java.nio.file.Path;
810
import java.util.UUID;
911

1012
import org.springframework.beans.factory.annotation.Value;
1113
import org.springframework.stereotype.Service;
1214
import org.springframework.web.multipart.MultipartFile;
15+
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
1316

1417
import lombok.RequiredArgsConstructor;
1518
import lombok.extern.slf4j.Slf4j;
@@ -143,6 +146,24 @@ public StoredFile storeInputStream(InputStream inputStream, String originalName)
143146
return new StoredFile(fileId, size);
144147
}
145148

149+
public String storeFromStreamingBody(StreamingResponseBody body, String originalName)
150+
throws IOException {
151+
String fileId = generateFileId();
152+
Path filePath = getFilePath(fileId);
153+
Files.createDirectories(filePath.getParent());
154+
boolean success = false;
155+
try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(filePath))) {
156+
body.writeTo(os);
157+
success = true;
158+
} finally {
159+
if (!success) {
160+
Files.deleteIfExists(filePath);
161+
}
162+
}
163+
log.debug("Stored StreamingResponseBody with ID: {}", fileId);
164+
return fileId;
165+
}
166+
146167
/**
147168
* Delete a file by its ID
148169
*

‎app/common/src/main/java/stirling/software/common/service/JobExecutorService.java‎

Lines changed: 26 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.springframework.http.ResponseEntity;
1717
import org.springframework.stereotype.Service;
1818
import org.springframework.web.multipart.MultipartFile;
19+
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
1920

2021
import jakarta.servlet.http.HttpServletRequest;
2122

@@ -305,33 +306,21 @@ private void processJobResult(String jobId, Object result) {
305306
Object body = response.getBody();
306307

307308
if (body instanceof byte[]) {
308-
// Extract filename from content-disposition header if available
309-
String filename = "result.pdf";
310-
String contentType = MediaType.APPLICATION_PDF_VALUE;
311-
312-
if (response.getHeaders().getContentDisposition() != null) {
313-
String disposition =
314-
response.getHeaders().getContentDisposition().toString();
315-
if (disposition.contains("filename=")) {
316-
filename =
317-
disposition.substring(
318-
disposition.indexOf("filename=") + 9,
319-
disposition.lastIndexOf('"'));
320-
}
321-
}
322-
323-
MediaType mediaType = response.getHeaders().getContentType();
309+
String filename = extractResponseFilename(response);
310+
String contentType = extractResponseContentType(response);
324311

325-
if (mediaType != null) {
326-
contentType = mediaType.toString();
327-
}
328-
329-
// Store byte array directly to disk
330312
String fileId = fileStorage.storeBytes((byte[]) body, filename);
331313
taskManager.setFileResult(jobId, fileId, filename, contentType);
332314
log.debug("Stored ResponseEntity<byte[]> result with fileId: {}", fileId);
315+
} else if (body instanceof StreamingResponseBody streamingBody) {
316+
String filename = extractResponseFilename(response);
317+
String contentType = extractResponseContentType(response);
333318

334-
// Let the GC handle the memory naturally
319+
String fileId = fileStorage.storeFromStreamingBody(streamingBody, filename);
320+
taskManager.setFileResult(jobId, fileId, filename, contentType);
321+
log.debug(
322+
"Stored ResponseEntity<StreamingResponseBody> result with fileId: {}",
323+
fileId);
335324
} else {
336325
// Check if the response body contains a fileId
337326
if (body != null && body.toString().contains("fileId")) {
@@ -481,6 +470,21 @@ private ResponseEntity<?> handleResultForSyncJob(Object result) throws IOExcepti
481470
}
482471
}
483472

473+
private static String extractResponseFilename(ResponseEntity<?> response) {
474+
if (response.getHeaders().getContentDisposition() != null) {
475+
String filename = response.getHeaders().getContentDisposition().getFilename();
476+
if (filename != null && !filename.isEmpty()) {
477+
return filename;
478+
}
479+
}
480+
return "result.pdf";
481+
}
482+
483+
private static String extractResponseContentType(ResponseEntity<?> response) {
484+
MediaType mediaType = response.getHeaders().getContentType();
485+
return mediaType != null ? mediaType.toString() : MediaType.APPLICATION_PDF_VALUE;
486+
}
487+
484488
/**
485489
* Parse session timeout string (e.g., "30m", "1h") to milliseconds
486490
*

‎app/common/src/main/java/stirling/software/common/service/JobQueue.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ private void executeJob(QueuedJob job) {
401401
* @throws Exception If there is an execution error
402402
*/
403403
private <T> T executeWithTimeout(Supplier<T> supplier, long timeoutMs) throws Exception {
404-
CompletableFuture<T> future = CompletableFuture.supplyAsync(supplier);
404+
CompletableFuture<T> future = CompletableFuture.supplyAsync(supplier, jobExecutor);
405405

406406
try {
407407
if (timeoutMs <= 0) {

‎app/common/src/main/java/stirling/software/common/service/PostHogService.java‎

Lines changed: 0 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,8 @@
77
import java.lang.management.OperatingSystemMXBean;
88
import java.lang.management.RuntimeMXBean;
99
import java.lang.management.ThreadMXBean;
10-
import java.net.InetAddress;
11-
import java.net.NetworkInterface;
1210
import java.nio.file.Files;
1311
import java.nio.file.Paths;
14-
import java.util.Enumeration;
1512
import java.util.HashMap;
1613
import java.util.Locale;
1714
import java.util.Map;
@@ -94,21 +91,12 @@ public Map<String, Object> captureServerMetrics() {
9491
metrics.put("os_name", System.getProperty("os.name"));
9592
metrics.put("os_version", System.getProperty("os.version"));
9693
metrics.put("java_version", System.getProperty("java.version"));
97-
metrics.put("user_name", System.getProperty("user.name"));
98-
metrics.put("user_home", System.getProperty("user.home"));
99-
metrics.put("user_dir", System.getProperty("user.dir"));
10094

10195
// CPU and Memory
10296
metrics.put("cpu_cores", Runtime.getRuntime().availableProcessors());
10397
metrics.put("total_memory", Runtime.getRuntime().totalMemory());
10498
metrics.put("free_memory", Runtime.getRuntime().freeMemory());
10599

106-
// Network and Server Identity
107-
InetAddress localHost = InetAddress.getLocalHost();
108-
metrics.put("ip_address", localHost.getHostAddress());
109-
metrics.put("hostname", localHost.getHostName());
110-
metrics.put("mac_address", getMacAddress());
111-
112100
// JVM info
113101
metrics.put("jvm_vendor", System.getProperty("java.vendor"));
114102
metrics.put("jvm_version", System.getProperty("java.vm.version"));
@@ -153,9 +141,6 @@ public Map<String, Object> captureServerMetrics() {
153141
metrics.put("gc_" + gcBean.getName() + "_time", gcBean.getCollectionTime());
154142
}
155143

156-
// Network interfaces
157-
metrics.put("network_interfaces", getNetworkInterfacesInfo());
158-
159144
// Docker detection and stats
160145
boolean isDocker = isRunningInDocker();
161146
if (isDocker) {
@@ -353,30 +338,6 @@ public Map<String, Object> captureApplicationProperties() {
353338
.getProFeatures()
354339
.getCustomMetadata()
355340
.isAutoUpdateMetadata());
356-
addIfNotEmpty(
357-
properties,
358-
"enterpriseEdition_customMetadata_author",
359-
applicationProperties
360-
.getPremium()
361-
.getProFeatures()
362-
.getCustomMetadata()
363-
.getAuthor());
364-
addIfNotEmpty(
365-
properties,
366-
"enterpriseEdition_customMetadata_creator",
367-
applicationProperties
368-
.getPremium()
369-
.getProFeatures()
370-
.getCustomMetadata()
371-
.getCreator());
372-
addIfNotEmpty(
373-
properties,
374-
"enterpriseEdition_customMetadata_producer",
375-
applicationProperties
376-
.getPremium()
377-
.getProFeatures()
378-
.getCustomMetadata()
379-
.getProducer());
380341
}
381342
// Capture AutoPipeline properties
382343
addIfNotEmpty(
@@ -386,39 +347,4 @@ public Map<String, Object> captureApplicationProperties() {
386347

387348
return properties;
388349
}
389-
390-
private String getMacAddress() {
391-
try {
392-
Enumeration<NetworkInterface> networkInterfaces =
393-
NetworkInterface.getNetworkInterfaces();
394-
while (networkInterfaces.hasMoreElements()) {
395-
NetworkInterface ni = networkInterfaces.nextElement();
396-
byte[] hardwareAddress = ni.getHardwareAddress();
397-
if (hardwareAddress != null) {
398-
String[] hexadecimal = new String[hardwareAddress.length];
399-
for (int i = 0; i < hardwareAddress.length; i++) {
400-
hexadecimal[i] = String.format("%02X", hardwareAddress[i]);
401-
}
402-
return String.join("-", hexadecimal);
403-
}
404-
}
405-
} catch (Exception e) {
406-
// Handle exception
407-
}
408-
return "Unknown";
409-
}
410-
411-
private Map<String, String> getNetworkInterfacesInfo() {
412-
Map<String, String> interfacesInfo = new HashMap<>();
413-
try {
414-
Enumeration<NetworkInterface> nets = NetworkInterface.getNetworkInterfaces();
415-
while (nets.hasMoreElements()) {
416-
NetworkInterface netint = nets.nextElement();
417-
interfacesInfo.put(netint.getName(), netint.getDisplayName());
418-
}
419-
} catch (Exception e) {
420-
interfacesInfo.put("error", e.getMessage());
421-
}
422-
return interfacesInfo;
423-
}
424350
}

‎app/common/src/main/java/stirling/software/common/util/FileToPdf.java‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package stirling.software.common.util;
22

3-
import java.io.ByteArrayInputStream;
43
import java.io.FileOutputStream;
54
import java.io.IOException;
65
import java.io.UncheckedIOException;
@@ -66,16 +65,7 @@ public static byte[] convertHtmlToPdf(
6665
ProcessExecutor.getInstance(ProcessExecutor.Processes.WEASYPRINT)
6766
.runCommandWithOutputHandling(command);
6867

69-
byte[] pdfBytes = Files.readAllBytes(tempOutputFile.getPath());
70-
try {
71-
return pdfBytes;
72-
} catch (Exception e) {
73-
pdfBytes = Files.readAllBytes(tempOutputFile.getPath());
74-
if (pdfBytes.length < 1) {
75-
throw e;
76-
}
77-
return pdfBytes;
78-
}
68+
return Files.readAllBytes(tempOutputFile.getPath());
7969
} // tempInputFile auto-closed
8070
} // tempOutputFile auto-closed
8171
}
@@ -92,8 +82,7 @@ private static void sanitizeHtmlFilesInZip(
9282
throws IOException {
9383
try (TempDirectory tempUnzippedDir = new TempDirectory(tempFileManager)) {
9484
try (ZipInputStream zipIn =
95-
ZipSecurity.createHardenedInputStream(
96-
new ByteArrayInputStream(Files.readAllBytes(zipFilePath)))) {
85+
ZipSecurity.createHardenedInputStream(Files.newInputStream(zipFilePath))) {
9786
ZipEntry entry = zipIn.getNextEntry();
9887
while (entry != null) {
9988
Path filePath =

0 commit comments

Comments
 (0)