优化Excel读取内存占用

app-builder/plugins/aipp-file-extract-excel/pom.xml

@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>modelengine.fit.jade</groupId>
+        <artifactId>app-builder-plugin-parent</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <groupId>modelengine.fit.jade.plugin</groupId>
+    <artifactId>aipp-file-extract-excel</artifactId>
+
+    <dependencies>
+        <!-- FIT -->
+        <dependency>
+            <groupId>org.fitframework</groupId>
+            <artifactId>fit-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.fitframework</groupId>
+            <artifactId>fit-util</artifactId>
+        </dependency>
+
+        <!-- fast excel -->
+        <dependency>
+            <groupId>cn.idev.excel</groupId>
+            <artifactId>fastexcel</artifactId>
+        </dependency>
+
+        <!-- Services -->
+        <dependency>
+            <groupId>modelengine.fit.jade</groupId>
+            <artifactId>aipp-file-extract-service</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>modelengine.fit.jade</groupId>
+            <artifactId>aipp-service</artifactId>
+        </dependency>
+
+        <!-- Tests -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.fitframework</groupId>
+            <artifactId>fit-test-framework</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.fitframework</groupId>
+                <artifactId>fit-build-maven-plugin</artifactId>
+                <version>${fit.version}</version>
+                <executions>
+                    <execution>
+                        <id>build-plugin</id>
+                        <goals>
+                            <goal>build-plugin</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>package-plugin</id>
+                        <goals>
+                            <goal>package-plugin</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>${maven.antrun.version}</version>
+                <executions>
+                    <execution>
+                        <phase>install</phase>
+                        <configuration>
+                            <target>
+                                <copy file="${project.build.directory}/${project.build.finalName}.jar"
+                                      todir="../../../build/plugins"/>
+                            </target>
+                        </configuration>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

app-builder/plugins/aipp-file-extract-excel/src/main/java/modelengine/fit/jade/aipp/file/extract/ExcelFileExtractor.java

@@ -0,0 +1,187 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
+ *  This file is a part of the ModelEngine Project.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+package modelengine.fit.jade.aipp.file.extract;
+
+import cn.idev.excel.ExcelReader;
+import cn.idev.excel.FastExcel;
+import cn.idev.excel.context.AnalysisContext;
+import cn.idev.excel.converters.Converter;
+import cn.idev.excel.enums.CellDataTypeEnum;
+import cn.idev.excel.metadata.GlobalConfiguration;
+import cn.idev.excel.metadata.data.DataFormatData;
+import cn.idev.excel.metadata.data.ReadCellData;
+import cn.idev.excel.metadata.property.ExcelContentProperty;
+import cn.idev.excel.read.listener.ReadListener;
+import cn.idev.excel.read.metadata.ReadSheet;
+import cn.idev.excel.util.DateUtils;
+import modelengine.fit.jober.aipp.service.OperatorService;
+import modelengine.fitframework.annotation.Component;
+import modelengine.fitframework.annotation.Fitable;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.math.BigDecimal;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Excel文件的提取器。
+ *
+ * @author 黄政炫
+ * @since 2025-09-06
+ */
+@Component
+public class ExcelFileExtractor implements FileExtraction {
+    /**
+     * 把单元格转换成格式化字符串。
+     *
+     * @param cell 表示单元格数据 {@link ReadCellData}。
+     * @return 转换后的内容 {@link String}。
+     */
+    private static String getCellValueAsString(ReadCellData<?> cell) {
+        switch (cell.getType()) {
+            case STRING:
+                return cell.getStringValue();
+            case NUMBER:
+                DataFormatData fmt = cell.getDataFormatData();
+                short formatIndex = fmt.getIndex();
+                String formatString = fmt.getFormat();
+                if (DateUtils.isADateFormat(formatIndex, formatString)) {
+                    double value = cell.getNumberValue().doubleValue();
+                    Date date = DateUtils.getJavaDate(value, true);
+                    return new SimpleDateFormat("yyyy-MM-dd").format(date);
+                } else {
+                    BigDecimal num = cell.getNumberValue();
+                    return num.stripTrailingZeros().toPlainString();
+                }
+            case BOOLEAN:
+                return Boolean.toString(cell.getBooleanValue());
+            default:
+                return "";
+        }
+    }
+
+    /**
+     * 该文件提取器支持EXCEL和CSV类型。
+     *
+     * @return 枚举常量类型集合 {@link List<String>}。
+     */
+    @Override
+    @Fitable(id = "get-fileType-excel")
+    public List<String> supportedFileType() {
+        return Arrays.asList(OperatorService.FileType.EXCEL.toString(), OperatorService.FileType.CSV.toString());
+    }
+
+    /**
+     * @param fileUrl 表示文件路径 {@link String}。
+     * @return 表示路径是否有效 {@link Boolean}。
+     */
+    private boolean isValidPath(String fileUrl) {
+        try {
+            Path path = Paths.get(fileUrl);
+            return Files.exists(path) && Files.isRegularFile(path);
+        } catch (InvalidPathException e) {
+            return false;
+        }
+    }
+
+    /**
+     * 从指定路径的 Excel 文件中提取内容，并返回为字符串形式。
+     *
+     * @param fileUrl 表示文件路径的 {@link String}。
+     * @return 表示文件内容的 {@link String}。
+     */
+    @Override
+    @Fitable(id = "extract-file-excel")
+    public String extractFile(String fileUrl) {
+        if (!isValidPath(fileUrl)) {
+            throw new IllegalArgumentException("无效的文件路径: " + fileUrl);
+        }
+        File file = Paths.get(fileUrl).toFile();
+        StringBuilder excelContent = new StringBuilder();
+        ExcelReadListener listener = new ExcelReadListener(excelContent);
+        ExcelReader reader = null;
+        try (InputStream is = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
+            reader = FastExcel.read(is, listener)
+                    .registerConverter(new CustomCellStringConverter())
+                    .headRowNumber(0)
+                    .build();
+
+            List<ReadSheet> sheets = reader.excelExecutor().sheetList();
+            for (ReadSheet meta : sheets) {
+                excelContent.append("Sheet ").append(meta.getSheetNo() + 1).append(':').append('\n');
+                ReadSheet readSheet = FastExcel.readSheet(meta.getSheetNo()).headRowNumber(0).build();
+                reader.read(readSheet);
+            }
+            excelContent.append('\n');
+        } catch (IOException e) {
+            throw new IllegalStateException("Excel文件读取失败", e);
+        } finally {
+            if (reader != null) {
+                reader.finish(); // 关闭资源
+            }
+        }
+        return excelContent.toString();
+    }
+
+    /**
+     * 读取监听器的内部类实现。
+     */
+    private class ExcelReadListener implements ReadListener<Map<Integer, String>> {
+        StringBuilder excelContent;
+
+        ExcelReadListener(StringBuilder excelContent) {
+            this.excelContent = excelContent;
+        }
+
+        @Override
+        public void invoke(Map<Integer, String> data, AnalysisContext context) {
+            String line = data.entrySet()
+                    .stream()
+                    .sorted(Map.Entry.comparingByKey())
+                    .map(e -> e.getValue() == null ? "" : e.getValue())
+                    .collect(Collectors.joining("\t"));
+            this.excelContent.append(line).append('\n');
+        }
+
+        @Override
+        public void doAfterAllAnalysed(AnalysisContext context) {
+        }
+    }
+
+    /**
+     * 自定义单元格数据转换器。
+     * 该转换器实现了能够处理单元格数据并将其转换为字符串形式。
+     */
+    public static class CustomCellStringConverter implements Converter<String> {
+        @Override
+        public Class<String> supportJavaTypeKey() {
+            return String.class;
+        }
+
+        @Override
+        public CellDataTypeEnum supportExcelTypeKey() {
+            return null;
+        }
+
+        @Override
+        public String convertToJavaData(ReadCellData<?> cellData, ExcelContentProperty contentProperty,
+                GlobalConfiguration globalConfiguration) {
+            return getCellValueAsString(cellData);
+        }
+    }
+}

app-builder/plugins/aipp-file-extract-excel/src/main/resources/application.yml

@@ -0,0 +1,4 @@
+fit:
+  beans:
+    packages:
+      - 'modelengine.fit.jade.aipp.file.extract'

app-builder/plugins/aipp-file-extract-excel/src/test/java/modelengine/fit/jade/aipp/file/extract/ExcelFileExtractorTest.java

@@ -0,0 +1,59 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
+ *  This file is a part of the ModelEngine Project.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+package modelengine.fit.jade.aipp.file.extract;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import modelengine.fit.jober.aipp.service.OperatorService;
+import modelengine.fitframework.annotation.Fit;
+import modelengine.fitframework.test.annotation.FitTestWithJunit;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * 表示{@link ExcelFileExtractor}的测试集。
+ *
+ * @author 黄政炫
+ * @since 2025-09-06
+ */
+@FitTestWithJunit(includeClasses = ExcelFileExtractor.class)
+@Disabled
+class ExcelFileExtractorTest {
+    @Fit
+    ExcelFileExtractor excelFileExtractor;
+
+    @Test
+    @DisplayName("测试获取支持文件类型")
+    void supportedFileType() {
+        List<String> supportedTypes =
+                Arrays.asList(OperatorService.FileType.EXCEL.toString(), OperatorService.FileType.CSV.toString());
+        assertThat(this.excelFileExtractor.supportedFileType()).isEqualTo(supportedTypes);
+    }
+
+    @Test
+    @DisplayName("测试能否捕获错误路径")
+    void validPath() {
+        assertThrows(IllegalArgumentException.class, () -> {
+            this.excelFileExtractor.extractFile("invalidPath.csv");
+        });
+    }
+
+    @Test
+    @DisplayName("测试 excel 文件提取成功")
+    void extractFile() {
+        File file = new File(this.getClass().getClassLoader().getResource("file/content.csv").getFile());
+        assertThat(this.excelFileExtractor.extractFile(file.getAbsolutePath())).isEqualTo(
+                "Sheet 1:\nThis is an excel test\n\n");
+    }
+}

app-builder/plugins/aipp-file-extract-excel/src/test/resources/file/content.csv

@@ -0,0 +1 @@
+This is an excel test

app-builder/plugins/aipp-plugin/pom.xml

@@ -143,6 +143,16 @@
             <groupId>org.redisson</groupId>
             <artifactId>redisson</artifactId>
         </dependency>
+        <!-- File-Extract-Service -->
+        <dependency>
+            <groupId>modelengine.fit.jade</groupId>
+            <artifactId>aipp-file-extract-service</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>modelengine.fit.jade.plugin</groupId>
+            <artifactId>aipp-file-extract-excel</artifactId>
+            <scope>test</scope>
+        </dependency>
 
         <!-- Test -->
         <dependency>