MODEXPW-582 - TD: Replacing of custom AWS client by folio-s3-client library (#703)

MODEXPW-582 - TD: Replacing of custom AWS client by folio-s3-client library

Author: khandramai

pom.xml

@@ -51,7 +51,6 @@
         <folio-service-tools.version>4.1.1</folio-service-tools.version>
         <folio-spring-cql.version>9.0.0</folio-spring-cql.version>
         <folio-module-descriptor-validator.version>1.0.0</folio-module-descriptor-validator.version>
-        <minio.version>8.5.15</minio.version>
         <openapi-generator.version>6.2.1</openapi-generator.version>
         <commons-collections4.version>4.5.0</commons-collections4.version>
         <hypersistence-utils-hibernate-63.version>3.7.3</hypersistence-utils-hibernate-63.version>
@@ -62,12 +61,13 @@
 
         <!-- Test properties-->
         <junit-extensions.version>2.4.0</junit-extensions.version>
-        <embedded-minio.version>2.3.1</embedded-minio.version>
         <spring-cloud-starter-bootstrap.version>4.2.0</spring-cloud-starter-bootstrap.version>
         <wiremock.version>2.27.2</wiremock.version>
+        <localstack.version>1.20.5</localstack.version>
         <marc4j.version>2.9.1</marc4j.version>
         <testcontainers.version>1.17.6</testcontainers.version>
-        <aws.sdk.version>2.29.47</aws.sdk.version>
+        <aws.sdk.version>2.29.6</aws.sdk.version>
+        <folio-s3-client.version>3.0.0-SNAPSHOT</folio-s3-client.version>
         <hamcrest-all.version>1.3</hamcrest-all.version>
         <mockito-inline.version>5.2.0</mockito-inline.version>
 
@@ -155,11 +155,6 @@
         <artifactId>feign-jackson</artifactId>
         <version>${feign-jackson.version}</version>
       </dependency>
-      <dependency>
-          <groupId>io.minio</groupId>
-          <artifactId>minio</artifactId>
-          <version>${minio.version}</version>
-      </dependency>
 
       <dependency>
         <groupId>org.apache.sshd</groupId>
@@ -220,7 +215,7 @@
 
       <dependency>
         <groupId>software.amazon.awssdk</groupId>
-        <artifactId>s3</artifactId>
+        <artifactId>aws-sdk-java</artifactId>
         <version>${aws.sdk.version}</version>
       </dependency>
 
@@ -249,6 +244,12 @@
         <version>${streamex.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>org.folio</groupId>
+        <artifactId>folio-s3-client</artifactId>
+        <version>${folio-s3-client.version}</version>
+      </dependency>
+
         <!-- Test dependencies -->
 
       <dependency>
@@ -337,11 +338,11 @@
             <version>${junit-extensions.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>com.playtika.testcontainers</groupId>
-            <artifactId>embedded-minio</artifactId>
-            <version>${embedded-minio.version}</version>
-            <scope>test</scope>
+       <dependency>
+          <groupId>org.testcontainers</groupId>
+          <artifactId>localstack</artifactId>
+          <version>${localstack.version}</version>
+          <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>com.github.tomakehurst</groupId>
@@ -675,6 +676,8 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>${maven-surefire-plugin.version}</version>
                 <configuration>
+                  <forkCount>1</forkCount>
+                  <reuseForks>true</reuseForks>
                     <argLine>
                         @{argLine} -Xmx2G -Duser.language=en -Duser.region=US
                     </argLine>

src/main/java/org/folio/dew/batch/AbstractStorageStreamWriter.java

@@ -1,7 +1,9 @@
 package org.folio.dew.batch;
 
+import java.io.IOException;
+import lombok.Getter;
+import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.folio.dew.error.FileOperationException;
 import org.folio.dew.repository.LocalFilesStorage;
@@ -14,18 +16,15 @@
 import org.springframework.batch.item.file.transform.LineAggregator;
 import org.springframework.core.io.WritableResource;
 
-import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.folio.dew.utils.Constants.LINE_SEPARATOR;
-import static org.folio.dew.utils.Constants.LINE_SEPARATOR_REPLACEMENT;
 
 @Slf4j
 public class AbstractStorageStreamWriter<T, S extends S3CompatibleStorage> implements ItemWriter<T> {
 
+  @Getter
   private WritableResource resource;
   private S storage;
+  @Setter
   private LineAggregator<T> lineAggregator;
 
   public AbstractStorageStreamWriter(String tempOutputFilePath, LocalFilesStorage localFilesStorage) {
@@ -62,32 +61,23 @@ public AbstractStorageStreamWriter(String tempOutputFilePath, String columnHeade
     log.info("Creating file {}.", tempOutputFilePath);
   }
 
-  public WritableResource getResource() {
-    return resource;
-  }
-
   public void setResource(S3CompatibleResource<S> resource) {
     this.resource = resource;
   }
 
-  public void setLineAggregator(LineAggregator<T> lineAggregator) {
-    this.lineAggregator = lineAggregator;
-  }
-
-  public LineAggregator<T> getLineAggregator() {
-    return lineAggregator;
-  }
-
   public S3CompatibleStorage getStorage() {
     return storage;
   }
 
   @Override
   public void write(Chunk<? extends T> items) throws Exception {
+    var filename = resource.getFilename();
     var sb = new StringBuilder();
+    var header =new String(storage.readAllBytes(filename));
+    sb.append(header);
     for (T item : items) {
       sb.append(lineAggregator.aggregate(item)).append('\n');
     }
-    storage.append(resource.getFilename(), sb.toString().getBytes(StandardCharsets.UTF_8));
+    storage.write(filename, sb.toString().getBytes(StandardCharsets.UTF_8));
   }
 }

src/main/java/org/folio/dew/batch/CsvFileAssembler.java

@@ -10,7 +10,6 @@
 import org.springframework.stereotype.Component;
 
 import java.util.Collection;
-import java.util.stream.Collectors;
 
 @Component
 @Log4j2
@@ -24,28 +23,28 @@ public class CsvFileAssembler implements StepExecutionAggregator {
   public void aggregate(StepExecution stepExecution, Collection<StepExecution> finishedStepExecutions) {
     var csvFilePartObjectNames = finishedStepExecutions.stream()
         .map(e -> e.getExecutionContext().getString(JobParameterNames.TEMP_OUTPUT_FILE_PATH))
-        .collect(Collectors.toList());
+        .toList();
     var destCsvObject = FilenameUtils.getName(
         stepExecution.getJobExecution().getJobParameters().getString(JobParameterNames.TEMP_OUTPUT_FILE_PATH) + ".csv");
     try {
       if ("CIRCULATION_LOG".equals(stepExecution.getJobExecution().getJobInstance().getJobName())) {
-        var csvUrl = remoteFilesStorage.composeObject(destCsvObject, csvFilePartObjectNames, null, TEXT_CSV);
+        var csvUrl = remoteFilesStorage.compose(destCsvObject, csvFilePartObjectNames, null, TEXT_CSV);
         ExecutionContextUtils.addToJobExecutionContext(stepExecution, JobParameterNames.CIRCULATION_LOG_FILE_NAME, destCsvObject, ";");
         ExecutionContextUtils.addToJobExecutionContext(stepExecution, JobParameterNames.OUTPUT_FILES_IN_STORAGE, csvUrl, ";");
       } else {
         var prefix = stepExecution.getJobExecution().getJobParameters().getString(JobParameterNames.JOB_ID) + "/";
 
         destCsvObject = prefix + destCsvObject;
         var csvUrl = remoteFilesStorage.objectToPresignedObjectUrl(
-          remoteFilesStorage.composeObject(destCsvObject, csvFilePartObjectNames, null, TEXT_CSV));
+          remoteFilesStorage.compose(destCsvObject, csvFilePartObjectNames, null, TEXT_CSV));
 
         var jsonFilePartObjectNames = finishedStepExecutions.stream()
           .map(e -> e.getExecutionContext().getString(JobParameterNames.TEMP_OUTPUT_FILE_PATH) + ".json")
-          .collect(Collectors.toList());
+          .toList();
         var destJsonObject = prefix + FilenameUtils.getName(
           stepExecution.getJobExecution().getJobParameters().getString(JobParameterNames.TEMP_OUTPUT_FILE_PATH) + ".json");
         var jsonUrl = remoteFilesStorage.objectToPresignedObjectUrl(
-          remoteFilesStorage.composeObject(destJsonObject, jsonFilePartObjectNames, null, TEXT_CSV));
+          remoteFilesStorage.compose(destJsonObject, jsonFilePartObjectNames, null, TEXT_CSV));
 
         ExecutionContextUtils.addToJobExecutionContext(stepExecution, JobParameterNames.OUTPUT_FILES_IN_STORAGE, csvUrl + ";;" + jsonUrl, ";");
       }

src/main/java/org/folio/dew/batch/JobCompletionNotificationListener.java

@@ -84,7 +84,7 @@ private void processJobAfter(String jobId, JobParameters jobParameters) {
       return;
     }
     var files = localFilesStorage.walk(path)
-      .filter(name -> FilenameUtils.getName(name).startsWith(fileNameStart)).collect(Collectors.toList());
+      .filter(name -> FilenameUtils.getName(name).startsWith(fileNameStart)).toList();
     if (files.isEmpty()) {
       return;
     }

src/main/java/org/folio/dew/batch/authoritycontrol/AuthorityControlCsvFileWriter.java

@@ -74,7 +74,18 @@ protected String doWrite(Chunk<? extends AuthorityControlExportFormat> chunk) {
   }
 
   private void writeString(String str) throws IOException {
-    localFilesStorage.append(tempOutputFilePath, str.getBytes(StandardCharsets.UTF_8));
+
+    try {
+      var header = new String(localFilesStorage.readAllBytes(tempOutputFilePath));
+      if (header.charAt(header.length() - 1) != '\n') {
+        str = header + "\n" + str;
+      } else {
+        str = header + str;
+      }
+    } catch (IOException e) {
+      // Just ignore it if there's nothing to append to.
+    }
+    localFilesStorage.write(tempOutputFilePath, str.getBytes(StandardCharsets.UTF_8));
   }
 
   private void setResource(String tempOutputFilePath) {

src/main/java/org/folio/dew/batch/bursarfeesfines/service/BursarWriter.java

@@ -58,11 +58,10 @@ public void write(Chunk<? extends String> items) throws Exception {
       .map(token -> BursarTokenFormatter.formatHeaderFooterToken(token, items.size(), aggregateTotalAmount))
       .collect(Collectors.joining());
 
-    localFilesStorage.write(resource.getFilename(), header.getBytes(StandardCharsets.UTF_8));
+    String result = header
+        + lines
+        + footer;
 
-    localFilesStorage.append(resource.getFilename(), lines.toString()
-      .getBytes(StandardCharsets.UTF_8));
-
-    localFilesStorage.append(resource.getFilename(), footer.getBytes(StandardCharsets.UTF_8));
+    localFilesStorage.write(resource.getFilename(), result.getBytes(StandardCharsets.UTF_8));
   }
 }

src/main/java/org/folio/dew/batch/eholdings/EHoldingsCsvFileWriter.java

@@ -25,6 +25,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.folio.de.entity.EHoldingsPackage;
 import org.folio.dew.domain.dto.EHoldingsExportConfig;
 import org.folio.dew.domain.dto.eholdings.EHoldingsResourceExportFormat;
@@ -132,7 +133,13 @@ private void writePackage(Long jobExecutionId) throws IOException {
   }
 
   private void writeString(String str) throws IOException {
-    localFilesStorage.append(tempOutputFilePath, str.getBytes(StandardCharsets.UTF_8));
+    byte[] bytesHeader = new byte[0];
+    try {
+      bytesHeader = localFilesStorage.readAllBytes(tempOutputFilePath);
+    } catch (IOException e) {
+      // Just ignore it if there's nothing to append to.
+    }
+    localFilesStorage.write(tempOutputFilePath, ArrayUtils.addAll(bytesHeader, str.getBytes(StandardCharsets.UTF_8)));
   }
 
   private String getHeader(List<String> fieldNames) {

src/main/java/org/folio/dew/batch/eholdings/GetEHoldingsWriter.java

@@ -4,8 +4,6 @@
 import static org.folio.dew.batch.eholdings.EHoldingsJobConstants.CONTEXT_TOTAL_RESOURCES;
 
 import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
 
 import org.folio.dew.domain.dto.eholdings.EHoldingsResourceDTO;
 import org.folio.dew.repository.EHoldingsResourceRepository;
@@ -40,7 +38,7 @@ public void beforeStep(StepExecution stepExecution) {
 
   @Override
   public void write(Chunk<? extends EHoldingsResourceDTO> list) throws Exception {
-    var resources = list.getItems().stream().map(EHoldingsResourceMapper::convertToEntity).collect(Collectors.toList());
+    var resources = list.getItems().stream().map(EHoldingsResourceMapper::convertToEntity).toList();
     resources.forEach(r -> r.setJobExecutionId(jobId));
     repository.saveAll(resources);
     jobExecution.getExecutionContext().putInt(CONTEXT_TOTAL_RESOURCES,

src/main/java/org/folio/dew/batch/marc/DataExportCsvItemReader.java

@@ -6,7 +6,6 @@
 import org.folio.dew.repository.LocalFilesStorage;
 
 import java.util.List;
-import java.util.stream.Collectors;
 
 public class DataExportCsvItemReader extends CsvItemReader<ItemIdentifier> {
 
@@ -30,7 +29,7 @@ protected List<ItemIdentifier> getItems(int offset, int limit) {
           .skip(offset)
           .limit(limit)
           .map(ItemIdentifier::new)
-          .collect(Collectors.toList());
+          .toList();
       }
     } catch (Exception e) {
       throw new FileOperationException(e.getMessage());