[MODEXPW-582] Migrate from custom S3 library to folio-s3-client

Author: ncovercash

pom.xml

@@ -47,11 +47,11 @@
         <refresh-presigned-url.yaml.file>
           ${project.basedir}/src/main/resources/swagger.api/refresh-presigned-url.yaml
         </refresh-presigned-url.yaml.file>
+        <folio-s3-client.version>3.0.0-SNAPSHOT</folio-s3-client.version>
         <folio-spring-base.version>10.0.0-SNAPSHOT</folio-spring-base.version>
         <folio-service-tools.version>4.1.1</folio-service-tools.version>
         <folio-spring-cql.version>9.0.0</folio-spring-cql.version>
         <folio-module-descriptor-validator.version>1.0.0</folio-module-descriptor-validator.version>
-        <minio.version>8.5.15</minio.version>
         <openapi-generator.version>6.2.1</openapi-generator.version>
         <commons-collections4.version>4.5.0</commons-collections4.version>
         <hypersistence-utils-hibernate-63.version>3.7.3</hypersistence-utils-hibernate-63.version>
@@ -67,7 +67,6 @@
         <wiremock.version>2.27.2</wiremock.version>
         <marc4j.version>2.9.1</marc4j.version>
         <testcontainers.version>1.17.6</testcontainers.version>
-        <aws.sdk.version>2.29.47</aws.sdk.version>
         <hamcrest-all.version>1.3</hamcrest-all.version>
         <mockito-inline.version>5.2.0</mockito-inline.version>
 
@@ -105,6 +104,11 @@
         <artifactId>folio-spring-cql</artifactId>
         <version>${folio-spring-cql.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.folio</groupId>
+        <artifactId>folio-s3-client</artifactId>
+        <version>${folio-s3-client.version}</version>
+      </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.datatype</groupId>
         <artifactId>jackson-datatype-jsr310</artifactId>
@@ -155,11 +159,6 @@
         <artifactId>feign-jackson</artifactId>
         <version>${feign-jackson.version}</version>
       </dependency>
-      <dependency>
-          <groupId>io.minio</groupId>
-          <artifactId>minio</artifactId>
-          <version>${minio.version}</version>
-      </dependency>
 
       <dependency>
         <groupId>org.apache.sshd</groupId>
@@ -218,12 +217,6 @@
         <version>${marc4j.version}</version>
       </dependency>
 
-      <dependency>
-        <groupId>software.amazon.awssdk</groupId>
-        <artifactId>s3</artifactId>
-        <version>${aws.sdk.version}</version>
-      </dependency>
-
       <dependency>
         <groupId>org.hibernate.validator</groupId>
         <artifactId>hibernate-validator</artifactId>

src/main/java/org/folio/dew/batch/AbstractStorageStreamWriter.java

@@ -1,28 +1,23 @@
 package org.folio.dew.batch;
 
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.folio.dew.error.FileOperationException;
+import org.folio.dew.repository.AbstractFilesStorage;
 import org.folio.dew.repository.LocalFilesStorage;
 import org.folio.dew.repository.S3CompatibleResource;
-import org.folio.dew.repository.S3CompatibleStorage;
+import org.folio.s3.exception.S3ClientException;
 import org.springframework.batch.item.Chunk;
 import org.springframework.batch.item.ItemWriter;
 import org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor;
 import org.springframework.batch.item.file.transform.DelimitedLineAggregator;
 import org.springframework.batch.item.file.transform.LineAggregator;
 import org.springframework.core.io.WritableResource;
 
-import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.folio.dew.utils.Constants.LINE_SEPARATOR;
-import static org.folio.dew.utils.Constants.LINE_SEPARATOR_REPLACEMENT;
 
 @Slf4j
-public class AbstractStorageStreamWriter<T, S extends S3CompatibleStorage> implements ItemWriter<T> {
+public class AbstractStorageStreamWriter<T, S extends AbstractFilesStorage> implements ItemWriter<T> {
 
   private WritableResource resource;
   private S storage;
@@ -52,7 +47,7 @@ public AbstractStorageStreamWriter(String tempOutputFilePath, String columnHeade
     if (StringUtils.isNotBlank(columnHeaders)) {
       try {
         storage.write(tempOutputFilePath, (columnHeaders + '\n').getBytes(StandardCharsets.UTF_8));
-      } catch (IOException e) {
+      } catch (S3ClientException e) {
         throw new FileOperationException(e);
       }
     }
@@ -78,7 +73,7 @@ public LineAggregator<T> getLineAggregator() {
     return lineAggregator;
   }
 
-  public S3CompatibleStorage getStorage() {
+  public AbstractFilesStorage getStorage() {
     return storage;
   }
 

src/main/java/org/folio/dew/batch/CsvWriter.java

@@ -1,10 +1,10 @@
 package org.folio.dew.batch;
 
-import lombok.extern.log4j.Log4j2;
-import org.folio.dew.repository.S3CompatibleStorage;
+import org.folio.dew.repository.AbstractFilesStorage;
 
+import lombok.extern.log4j.Log4j2;
 @Log4j2
-public class CsvWriter<T, R extends S3CompatibleStorage> extends AbstractStorageStreamWriter<T, R> {
+public class CsvWriter<T, R extends AbstractFilesStorage> extends AbstractStorageStreamWriter<T, R> {
 
   public CsvWriter(String tempOutputFilePath, String columnHeaders, String[] extractedFieldNames, FieldProcessor fieldProcessor, R storage) {
     super(tempOutputFilePath, columnHeaders, extractedFieldNames, fieldProcessor, storage);

src/main/java/org/folio/dew/batch/JobCompletionNotificationListener.java

@@ -83,13 +83,13 @@ private void processJobAfter(String jobId, JobParameters jobParameters) {
     if (StringUtils.isBlank(path) || StringUtils.isBlank(fileNameStart)) {
       return;
     }
-    var files = localFilesStorage.walk(path)
+    var files = localFilesStorage.listRecursive(path).stream()
       .filter(name -> FilenameUtils.getName(name).startsWith(fileNameStart)).collect(Collectors.toList());
     if (files.isEmpty()) {
       return;
     }
     for (String f : files) {
-        localFilesStorage.delete(f);
+        localFilesStorage.removeRecursive(f);
     }
     log.info("Deleted temp files {} of job {}.", files, jobId);
   }

src/main/java/org/folio/dew/batch/marc/DataExportCsvItemReader.java

@@ -25,13 +25,10 @@ public DataExportCsvItemReader(String fileName, Long offset, Long limit, LocalFi
   @Override
   protected List<ItemIdentifier> getItems(int offset, int limit) {
     try {
-      try (var lines = localFilesStorage.lines(fileName)) {
-        return lines
-          .skip(offset)
-          .limit(limit)
-          .map(ItemIdentifier::new)
-          .collect(Collectors.toList());
-      }
+      return localFilesStorage.readLines(fileName, offset, limit)
+        .stream()
+        .map(ItemIdentifier::new)
+        .collect(Collectors.toList());
     } catch (Exception e) {
       throw new FileOperationException(e.getMessage());
     }

src/main/java/org/folio/dew/batch/marc/DataExportCsvPartitioner.java

@@ -3,6 +3,9 @@
 import lombok.extern.log4j.Log4j2;
 import org.folio.dew.batch.CsvPartitioner;
 import org.folio.dew.repository.LocalFilesStorage;
+import org.folio.s3.exception.S3ClientException;
+
+import java.io.IOException;
 
 @Log4j2
 public class DataExportCsvPartitioner extends CsvPartitioner {
@@ -19,9 +22,9 @@ public DataExportCsvPartitioner(Long offset, Long limit, String tempOutputFilePa
 
   @Override
   protected Long getLimit() {
-    try (var lines = localFilesStorage.lines(fileName)) {
-      return lines.count();
-    } catch (Exception e) {
+    try {
+      return localFilesStorage.numLines(fileName);
+    } catch (IOException | S3ClientException e) {
       log.error("Error reading file {}, reason: {}", fileName, e.getMessage());
       return 0L;
     }

src/main/java/org/folio/dew/config/properties/MinioClientProperties.java

@@ -1,6 +1,7 @@
 package org.folio.dew.config.properties;
 
 import lombok.Data;
+import org.folio.s3.client.S3ClientProperties;
 
 @Data
 public class MinioClientProperties {
@@ -50,4 +51,18 @@ public class MinioClientProperties {
    * Presigned url expiration time (in seconds).
    */
   private int urlExpirationTimeInSeconds;
+
+  public S3ClientProperties toS3ClientProperties() {
+    return S3ClientProperties
+      .builder()
+      .endpoint(endpoint)
+      .region(region)
+      .bucket(bucket)
+      .accessKey(accessKey)
+      .secretKey(secretKey)
+      .awsSdk(composeWithAwsSdk)
+      .forcePathStyle(forcePathStyle)
+      .subPath(subPath)
+      .build();
+  }
 }