Merge branch '7.13' into gh-5546-proxy-s3-forwarding

Author: at055612

gradle/libs.versions.toml

@@ -4,9 +4,9 @@ eventLogging = "5.0-beta.31_schema-v4.0-beta.3"
 stroomStats = "1.0-alpha.6"
 
 # ------------3rd-party------------
-aws = "2.46.7"
+aws = "2.46.16"
 commons-io = "2.21.0"
-dropwizard = "5.0.1" # used to set the dropwizard-bom version, that controls lots of dependency versions
+dropwizard = "5.0.2" # used to set the dropwizard-bom version, that controls lots of dependency versions
 elasticsearch = "9.2.1"
 flyway = "12.0.0"
 guice = "7.0.0"
@@ -87,7 +87,7 @@ gwt-dev = { module = "org.gwtproject:gwt-dev", version.ref = "gwt" }
 gwt-user = { module = "org.gwtproject:gwt-user", version.ref = "gwt" }
 gwtp-mvp-client = { module = "com.gwtplatform:gwtp-mvp-client", version = "0.7" }
 #gson = { module = "com.google.code.gson:gson", version = "2.6.2" }
-hbase-shaded-netty = { module = "org.apache.hbase.thirdparty:hbase-shaded-netty", version = "4.1.12" } # Needed by our copied HBase classes in stroom.bytebuffer.hbase
+hbase-shaded-netty = { module = "org.apache.hbase.thirdparty:hbase-shaded-netty", version = "4.1.13" } # Needed by our copied HBase classes in stroom.bytebuffer.hbase
 hessian = { module = "com.caucho:hessian", version = "4.0.65" }
 hikari = { module = "com.zaxxer:HikariCP", version = "7.0.2" }
 http-client = { module = "org.apache.httpcomponents.client5:httpclient5" } # version controlled by dropwizard-dependencies

stroom-ai/stroom-ai-impl/src/main/java/stroom/ai/impl/ApacheHttpClient.java

@@ -3,6 +3,9 @@
 import stroom.util.http.HttpClientConfiguration;
 import stroom.util.jersey.HttpClientProvider;
 import stroom.util.jersey.HttpClientProviderCache;
+import stroom.util.logging.LambdaLogger;
+import stroom.util.logging.LambdaLoggerFactory;
+import stroom.util.logging.LogUtil;
 
 import dev.langchain4j.http.client.HttpClient;
 import dev.langchain4j.http.client.HttpRequest;
@@ -35,6 +38,8 @@
 
 public class ApacheHttpClient implements HttpClient {
 
+    private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(ApacheHttpClient.class);
+
     private final HttpClientProviderCache httpClientProviderCache;
     private final HttpClientConfiguration httpClientConfiguration;
 
@@ -63,6 +68,7 @@ public void execute(final HttpRequest request,
         final ClassicHttpRequest apacheRequest = createApacheRequest(request);
         try (final HttpClientProvider httpClientProvider = httpClientProviderCache.get(httpClientConfiguration)) {
             final org.apache.hc.client5.http.classic.HttpClient httpClient = httpClientProvider.get();
+
             if (httpClient instanceof final CloseableHttpClient closeableHttpClient) {
                 httpClient.execute(apacheRequest, response -> {
                     handleServerSentEvents(response, parser, listener);
@@ -126,6 +132,7 @@ private ClassicHttpRequest createApacheRequest(final HttpRequest request) {
                             apacheRequest.addHeader(key, value)));
         }
 
+        LOGGER.debug("createApacheRequest() - method: {}, url: {}, returning: {}", method, url, apacheRequest);
         return apacheRequest;
     }
 
@@ -135,19 +142,29 @@ private SuccessfulHttpResponse convertResponse(final ClassicHttpResponse respons
 
             final Map<String, List<String>> headers = new HashMap<>();
             for (final Header header : response.getHeaders()) {
-                headers.computeIfAbsent(header.getName(), k -> new ArrayList<>()).add(header.getValue());
+                headers.computeIfAbsent(header.getName(), ignored -> new ArrayList<>())
+                        .add(header.getValue());
             }
 
             String body = null;
             if (response.getEntity() != null) {
                 body = EntityUtils.toString(response.getEntity());
             }
-
-            return SuccessfulHttpResponse.builder()
-                    .statusCode(statusCode)
-                    .headers(headers)
-                    .body(body)
-                    .build();
+            LOGGER.debug("convertResponse() - statusCode: {}, headers: {}, body:\n{}",
+                    statusCode, headers, body);
+
+            try {
+                // This will throw if statusCode is not a 2xx
+                return SuccessfulHttpResponse.builder()
+                        .statusCode(statusCode)
+                        .headers(headers)
+                        .body(body)
+                        .build();
+            } catch (final RuntimeException e) {
+                throw new RuntimeException(LogUtil.message(
+                        "Unable to convert HTTP response body, statusCode: {}, headers: {}, body:\n{}",
+                        statusCode, headers, body), e);
+            }
         } catch (final IOException e) {
             throw new UncheckedIOException(e);
         } catch (final ParseException e) {

stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/SmtpConfig.java

@@ -31,11 +31,14 @@
 import jakarta.validation.constraints.NotNull;
 import org.simplejavamail.api.mailer.config.TransportStrategy;
 
+import java.util.Objects;
+
 @NotInjectableConfig
 @JsonPropertyOrder(alphabetic = true)
 public class SmtpConfig extends AbstractConfig implements IsStroomConfig {
 
     public static final String DEFAULT_TRANSPORT = "plain";
+    private static final int DEFAULT_PORT = 2525;
 
     @NotNull
     @JsonProperty("host")
@@ -64,7 +67,7 @@ public class SmtpConfig extends AbstractConfig implements IsStroomConfig {
 
     public SmtpConfig() {
         host = "localhost";
-        port = 2525;
+        port = DEFAULT_PORT;
         transport = DEFAULT_TRANSPORT;
         password = null;
         username = null;
@@ -73,12 +76,12 @@ public SmtpConfig() {
     @SuppressWarnings("unused")
     @JsonCreator
     public SmtpConfig(@JsonProperty("host") final String host,
-                      @JsonProperty("port") final int port,
+                      @JsonProperty("port") final Integer port,
                       @JsonProperty("transport") final String transport,
                       @JsonProperty("username") final String username,
                       @JsonProperty("password") final String password) {
         this.host = host;
-        this.port = port;
+        this.port = Objects.requireNonNullElse(port, DEFAULT_PORT);
         this.transport = transport;
         this.username = username;
         this.password = password;

stroom-annotation/stroom-annotation-impl-db/src/main/resources/stroom/annotation/impl/db/migration/V07_13_00_001__collation_fix.sql

@@ -0,0 +1,27 @@
+-- ------------------------------------------------------------------------
+-- Copyright 2016-2026 Crown Copyright
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+-- ------------------------------------------------------------------------
+
+-- Stop NOTE level warnings about objects (not)? existing
+SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0;
+
+ALTER TABLE annotation_link CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;
+ALTER TABLE annotation_subscription CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;
+ALTER TABLE annotation_tag CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;
+ALTER TABLE annotation_tag_link CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;
+
+SET SQL_NOTES=@OLD_SQL_NOTES;
+
+-- vim: set shiftwidth=4 tabstop=4 expandtab:

stroom-app/src/test/java/stroom/contentindex/TestLuceneContentIndex.java

@@ -16,20 +16,27 @@
 
 package stroom.contentindex;
 
+import stroom.cluster.api.ClusterNodeManager;
+import stroom.cluster.lock.api.ClusterLockService;
+import stroom.cluster.lock.mock.MockClusterLockService;
 import stroom.docref.DocRef;
 import stroom.explorer.api.ExplorerNodeService;
 import stroom.explorer.shared.DocContentHighlights;
 import stroom.explorer.shared.DocContentMatch;
 import stroom.explorer.shared.FetchHighlightsRequest;
 import stroom.explorer.shared.FindInContentRequest;
 import stroom.explorer.shared.StringMatch;
+import stroom.node.api.NodeInfo;
 import stroom.pipeline.shared.XsltDoc;
 import stroom.pipeline.xslt.XsltStore;
 import stroom.security.mock.MockSecurityContext;
 import stroom.task.api.ExecutorProvider;
 import stroom.task.api.SimpleTaskContextFactory;
 import stroom.task.shared.ThreadPool;
 import stroom.test.AbstractCoreIntegrationTest;
+import stroom.util.io.PathCreator;
+import stroom.util.io.SimplePathCreator;
+import stroom.util.io.TempDirProvider;
 import stroom.util.shared.PageRequest;
 import stroom.util.shared.ResultPage;
 
@@ -42,6 +49,7 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
+import java.nio.file.Path;
 import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
@@ -85,14 +93,22 @@ public class TestLuceneContentIndex extends AbstractCoreIntegrationTest {
     private static ExecutorService executorService;
     private static ExecutorProvider executorProvider;
 
+    private final ClusterLockService clusterLockService = new MockClusterLockService();
+
     @Inject
     private XsltStore xsltStore;
 
     private XsltDoc xsltDoc;
     private DocRef docRef;
+    private TempDirProvider tempDirProvider;
+    private PathCreator pathCreator;
 
     @Mock
     ExplorerNodeService explorerNodeService;
+    @Mock
+    private NodeInfo mockNodeInfo;
+    @Mock
+    private ClusterNodeManager mockClusterNodeManager;
 
     @BeforeAll
     static void beforeAll() {
@@ -121,6 +137,11 @@ void setup() {
         docRef = xsltStore.createDocument("Test");
         xsltDoc = xsltStore.readDocument(docRef).copy().data(TEXT).build();
         xsltStore.writeDocument(xsltDoc);
+        final Path testDir = getCurrentTestDir();
+        tempDirProvider = () -> testDir.resolve("temp");
+        pathCreator = new SimplePathCreator(
+                () -> testDir.resolve("home"),
+                tempDirProvider);
     }
 
     @Test
@@ -167,12 +188,17 @@ void test(final StringMatch stringMatch,
 
     private DocContentHighlights test(final StringMatch stringMatch) {
         final LuceneContentIndex contentIndex = new LuceneContentIndex(
-                this::getCurrentTestDir,
+                tempDirProvider,
+                pathCreator,
+                ContentIndexConfig::new,
                 Set.of(xsltStore),
                 new MockSecurityContext(),
                 new SimpleTaskContextFactory(),
                 explorerNodeService,
-                executorProvider);
+                executorProvider,
+                clusterLockService,
+                mockNodeInfo,
+                mockClusterNodeManager);
         contentIndex.reindex();
         contentIndex.flush();
 

stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3Manager.java

@@ -680,6 +680,32 @@ public static CIKey removeAwsPrefix(final CIKey key) {
                 });
     }
 
+//    private PutObjectRequest createPutObjectRequest(final String bucketName,
+//                                                    final String key,
+//                                                    final Meta meta,
+//                                                    final AttributeMap attributeMap,
+//                                                    final S3UploadProperties uploadProperties) {
+//        final Map<String, String> metadata = attributeMap
+//                .entrySet()
+//                .stream()
+//                .collect(Collectors.toMap(e -> createS3Name(e.getKey()), Entry::getValue));
+//
+//        final PutObjectRequest.Builder builder = PutObjectRequest.builder()
+//                .bucket(bucketName)
+//                .key(key)
+//                .tagging(createTags(meta))
+//                .metadata(metadata);
+//
+//        if (uploadProperties != null) {
+//            NullSafe.consumeNonBlankString(uploadProperties.cacheControl(), builder::cacheControl);
+//            NullSafe.consumeNonBlankString(uploadProperties.contentDisposition(), builder::contentDisposition);
+//            NullSafe.consumeNonBlankString(uploadProperties.contentEncoding(), builder::contentEncoding);
+//            NullSafe.consumeNonBlankString(uploadProperties.contentType(), builder::contentType);
+//                    }
+//                    return ciKey;
+//                });
+//    }
+
     /**
      * Remove a prefix if present.
      *

stroom-bytebuffer/src/main/java/stroom/bytebuffer/ByteBufferPoolConfig.java

@@ -28,17 +28,21 @@
 import jakarta.validation.constraints.Min;
 
 import java.util.Map;
+import java.util.Objects;
 import java.util.TreeMap;
 
 @JsonPropertyOrder(alphabetic = true)
 public class ByteBufferPoolConfig extends AbstractConfig implements IsStroomConfig {
 
+    private static final int DEFAULT_WARNING_THRESHOLD_PERCENTAGE = 90;
+    private static final boolean DEFAULT_BLOCK_ON_EXHAUSTED_POOL = false;
+
     private final int warningThresholdPercentage;
     private final Map<Integer, Integer> pooledByteBufferCounts;
     private final boolean blockOnExhaustedPool;
 
     public ByteBufferPoolConfig() {
-        warningThresholdPercentage = 90;
+        warningThresholdPercentage = DEFAULT_WARNING_THRESHOLD_PERCENTAGE;
         // Use a treemap so we get a consistent order in the yaml so TestYamlUtil doesn't fail
         pooledByteBufferCounts = new TreeMap<>(Map.of(
                 1, 50,
@@ -48,17 +52,19 @@ public ByteBufferPoolConfig() {
                 10_000, 50,
                 100_000, 10,
                 1_000_000, 3));
-        blockOnExhaustedPool = false;
+        blockOnExhaustedPool = DEFAULT_BLOCK_ON_EXHAUSTED_POOL;
     }
 
     @JsonCreator
     public ByteBufferPoolConfig(
-            @JsonProperty("warningThresholdPercentage") final int warningThresholdPercentage,
+            @JsonProperty("warningThresholdPercentage") final Integer warningThresholdPercentage,
             @JsonProperty("pooledByteBufferCounts") final Map<Integer, Integer> pooledByteBufferCounts,
-            @JsonProperty("blockOnExhaustedPool") final boolean blockOnExhaustedPool) {
-        this.warningThresholdPercentage = warningThresholdPercentage;
+            @JsonProperty("blockOnExhaustedPool") final Boolean blockOnExhaustedPool) {
+        this.warningThresholdPercentage = Objects.requireNonNullElse(warningThresholdPercentage,
+                DEFAULT_WARNING_THRESHOLD_PERCENTAGE);
         this.pooledByteBufferCounts = pooledByteBufferCounts;
-        this.blockOnExhaustedPool = blockOnExhaustedPool;
+        this.blockOnExhaustedPool = Objects.requireNonNullElse(blockOnExhaustedPool,
+                DEFAULT_BLOCK_ON_EXHAUSTED_POOL);
     }
 
     @Max(100)

stroom-cluster/stroom-cluster-api/src/main/java/stroom/cluster/api/ClusterConfig.java

@@ -26,32 +26,39 @@
 import com.fasterxml.jackson.annotation.JsonPropertyDescription;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 
+import java.util.Objects;
+
 
 @JsonPropertyOrder(alphabetic = true)
 public class ClusterConfig extends AbstractConfig implements IsStroomConfig {
 
+    private static final boolean DEFAULT_CLUSTER_CALL_USE_LOCAL = true;
+    private static final boolean DEFAULT_CLUSTER_CALL_IGNORE_SSL_HOSTNAME_VERIFIER = true;
+
     private final boolean clusterCallUseLocal;
     private final StroomDuration clusterCallReadTimeout;
     private final boolean clusterCallIgnoreSSLHostnameVerifier;
     private final StroomDuration clusterResponseTimeout;
 
     public ClusterConfig() {
-        clusterCallUseLocal = true;
+        clusterCallUseLocal = DEFAULT_CLUSTER_CALL_USE_LOCAL;
         clusterCallReadTimeout = StroomDuration.ofSeconds(30);
-        clusterCallIgnoreSSLHostnameVerifier = true;
+        clusterCallIgnoreSSLHostnameVerifier = DEFAULT_CLUSTER_CALL_IGNORE_SSL_HOSTNAME_VERIFIER;
         clusterResponseTimeout = StroomDuration.ofSeconds(30);
     }
 
     @SuppressWarnings("unused")
     @JsonCreator
     public ClusterConfig(
-            @JsonProperty("clusterCallUseLocal") final boolean clusterCallUseLocal,
+            @JsonProperty("clusterCallUseLocal") final Boolean clusterCallUseLocal,
             @JsonProperty("clusterCallReadTimeout") final StroomDuration clusterCallReadTimeout,
-            @JsonProperty("clusterCallIgnoreSSLHostnameVerifier") final boolean clusterCallIgnoreSSLHostnameVerifier,
+            @JsonProperty("clusterCallIgnoreSSLHostnameVerifier") final Boolean clusterCallIgnoreSSLHostnameVerifier,
             @JsonProperty("clusterResponseTimeout") final StroomDuration clusterResponseTimeout) {
-        this.clusterCallUseLocal = clusterCallUseLocal;
+        this.clusterCallUseLocal = Objects.requireNonNullElse(clusterCallUseLocal,
+                DEFAULT_CLUSTER_CALL_USE_LOCAL);
         this.clusterCallReadTimeout = clusterCallReadTimeout;
-        this.clusterCallIgnoreSSLHostnameVerifier = clusterCallIgnoreSSLHostnameVerifier;
+        this.clusterCallIgnoreSSLHostnameVerifier = Objects.requireNonNullElse(clusterCallIgnoreSSLHostnameVerifier,
+                DEFAULT_CLUSTER_CALL_IGNORE_SSL_HOSTNAME_VERIFIER);
         this.clusterResponseTimeout = clusterResponseTimeout;
     }
 

stroom-config/stroom-config-app/build.gradle

@@ -40,6 +40,7 @@ dependencies {
     implementation project(':stroom-importexport:stroom-importexport-impl')
     implementation project(':stroom-index:stroom-index-impl')
     implementation project(':stroom-index:stroom-index-impl-db')
+    implementation project(':stroom-index:stroom-index-lucene')
     implementation project(':stroom-job:stroom-job-impl')
     implementation project(':stroom-job:stroom-job-impl-db')
     implementation project(':stroom-kafka:stroom-kafka-impl')