feat: 新增文件导入/导出
This commit is contained in:
zengxiaobo
parent
c5e235872b
commit
d4f01bb227
@ -0,0 +1,18 @@
|
|||||||
|
package cn.axzo.foundation.page;
|
||||||
|
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
public class PageReq implements IPageReq{
|
||||||
|
Integer page;
|
||||||
|
Integer pageSize;
|
||||||
|
List<String> sort;
|
||||||
|
}
|
||||||
44
excel-support-lib/pom.xml
Normal file
44
excel-support-lib/pom.xml
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>cn.axzo.foundation</groupId>
|
||||||
|
<artifactId>axzo-lib-box</artifactId>
|
||||||
|
<version>2.0.0-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<groupId>cn.axzo.maokai</groupId>
|
||||||
|
<artifactId>excel-support-lib</artifactId>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>cn.axzo.foundation</groupId>
|
||||||
|
<artifactId>web-support-lib</artifactId>
|
||||||
|
<version>2.0.0-SNAPSHOT</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.alibaba</groupId>
|
||||||
|
<artifactId>easyexcel</artifactId>
|
||||||
|
<version>3.3.4</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.opencsv</groupId>
|
||||||
|
<artifactId>opencsv</artifactId>
|
||||||
|
<version>5.9</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter-data-redis</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
</project>
|
||||||
@ -0,0 +1,918 @@
|
|||||||
|
package cn.axzo.foundation.excel.support;
|
||||||
|
|
||||||
|
import cn.axzo.foundation.exception.BusinessException;
|
||||||
|
import cn.axzo.foundation.page.IPageReq;
|
||||||
|
import cn.axzo.foundation.page.PageResp;
|
||||||
|
import cn.axzo.foundation.util.VarParamFormatter;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.alibaba.fastjson.JSONPath;
|
||||||
|
import com.alibaba.fastjson.parser.ParserConfig;
|
||||||
|
import com.alibaba.fastjson.util.TypeUtils;
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
|
import com.google.common.collect.ImmutableMap;
|
||||||
|
import com.google.common.collect.ImmutableSet;
|
||||||
|
import lombok.*;
|
||||||
|
import lombok.experimental.Accessors;
|
||||||
|
import lombok.experimental.SuperBuilder;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.commons.lang3.time.DateUtils;
|
||||||
|
import org.apache.poi.ss.SpreadsheetVersion;
|
||||||
|
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.RoundingMode;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.ZoneId;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.function.BiConsumer;
|
||||||
|
import java.util.function.BiFunction;
|
||||||
|
import java.util.function.Function;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
|
import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 用于发送sms & email验证码, 或者获取图片验证码. 并提供校验验证码是否正确的能力
|
||||||
|
*/
|
||||||
|
public interface DataSheetClient {
|
||||||
|
|
||||||
|
ExporterBuilder exportBuilder();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 数据导入的builder
|
||||||
|
*
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
ImporterBuilder importBuilder();
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Accessors(fluent = true)
|
||||||
|
abstract class ImporterBuilder {
|
||||||
|
/**
|
||||||
|
* 导入的场景名称, 必填
|
||||||
|
*/
|
||||||
|
@NonNull
|
||||||
|
private String scene;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导入格式
|
||||||
|
*/
|
||||||
|
@NonNull
|
||||||
|
private ImportFormat format;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* EXCEL格式的sheetName
|
||||||
|
*/
|
||||||
|
private String tableName;
|
||||||
|
|
||||||
|
private Meta meta;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否忽略导入文件表头的原meta
|
||||||
|
* 默认按false处理，即{@link #meta}为空时尝试使用原文件表头meta
|
||||||
|
*/
|
||||||
|
private Boolean ignoreHeaderMeta;
|
||||||
|
|
||||||
|
private boolean debugEnabled;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 忽略未知的列(cellMeta中未定义的列)
|
||||||
|
*/
|
||||||
|
private boolean ignoreUnknownColumn;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否自动清除表格内容的前后空白字符
|
||||||
|
* 默认true，即会清除前后空白字符
|
||||||
|
*/
|
||||||
|
private Boolean autoTrim;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解析结果ImportResp中的lines是否包含解析失败的行
|
||||||
|
* true: 解析失败的时候，不会抛异常，会在每行的JSONObject中添加
|
||||||
|
* "errors":[{"columnIndex":1,"columnKey":"","columnName":"","rawValue":"","errorCode":"","errorMsg":""}]
|
||||||
|
* false: 解析失败的时候，抛第一个异常
|
||||||
|
*/
|
||||||
|
private boolean includeLineErrors;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 允许最大的导入行数
|
||||||
|
*/
|
||||||
|
private Integer allowMaxLineCount;
|
||||||
|
|
||||||
|
/** 将excel中的表头转换为cellMeta中的表头 */
|
||||||
|
private Function<String, String> headerConverter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 操作人, 可选，用于导入统计报告
|
||||||
|
*/
|
||||||
|
private String operator;
|
||||||
|
|
||||||
|
public abstract Importer build();
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Importer {
|
||||||
|
/**
|
||||||
|
* 所有字段读取为String类型
|
||||||
|
*
|
||||||
|
* @param inputStream
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
ImportResp<JSONObject> readAll(InputStream inputStream);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Builder
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
class ImportResp<T> {
|
||||||
|
/**
|
||||||
|
* 导入的场景名称
|
||||||
|
*/
|
||||||
|
private String scene;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 模版对应的code以及版本
|
||||||
|
*/
|
||||||
|
private String templateCode;
|
||||||
|
private String version;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* headers
|
||||||
|
*/
|
||||||
|
private List<String> headers;
|
||||||
|
/**
|
||||||
|
* 每一行的数据
|
||||||
|
*/
|
||||||
|
private List<T> lines;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* header的行数
|
||||||
|
*/
|
||||||
|
private Integer headerRowCount;
|
||||||
|
/**
|
||||||
|
* 导入数据总行数, 不包含header
|
||||||
|
*/
|
||||||
|
private Integer rowCount;
|
||||||
|
/**
|
||||||
|
* 导入数据列数
|
||||||
|
*/
|
||||||
|
private Integer columnCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Meta数据
|
||||||
|
*/
|
||||||
|
private Meta meta;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 消耗的时间
|
||||||
|
*/
|
||||||
|
private Long elapsedMillis;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 操作人
|
||||||
|
*/
|
||||||
|
private String operator;
|
||||||
|
}
|
||||||
|
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Getter
|
||||||
|
enum ImportFormat {
|
||||||
|
EXCEL("xlsx"),
|
||||||
|
// TODO: 支持其他类型
|
||||||
|
// CSV("csv")
|
||||||
|
;
|
||||||
|
|
||||||
|
private String suffix;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@Accessors(fluent = true)
|
||||||
|
abstract class ExporterBuilder {
|
||||||
|
/**
|
||||||
|
* 导出的场景名称, 必填
|
||||||
|
*/
|
||||||
|
@NonNull
|
||||||
|
private String scene;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出格式
|
||||||
|
*/
|
||||||
|
@NonNull
|
||||||
|
private ExportFormat format;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 顶部提示内容，主要用于一些提示场景。如导出数据超范围等文案提示等，支持多行。
|
||||||
|
*/
|
||||||
|
private Function<PageResp<JSONObject>, List<String>> topHintsSupplier;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 提供分页获取数据的方法
|
||||||
|
*/
|
||||||
|
@NonNull
|
||||||
|
private Function<IPageReq, PageResp<JSONObject>> rowSupplier;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出的总数据量限制，不指定时，按照分页返回的结果全量导出
|
||||||
|
*/
|
||||||
|
private Long limit = Long.MAX_VALUE;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取数据的分页大小, 默认每页200条(MybatisPlus的PaginationInterceptor默认limit为500)
|
||||||
|
*/
|
||||||
|
private Long pageSize = 200L;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 表名，仅当导出格式为excel时有效
|
||||||
|
*/
|
||||||
|
private String sheetName;
|
||||||
|
|
||||||
|
@Deprecated
|
||||||
|
public ExporterBuilder tableName(String tableName) {
|
||||||
|
this.sheetName = tableName;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 列映射表, key=列名， value=获取字段值的方法, 方法参数1为当前row的JSONObject, 参数2为当前row的index，返回列字段值
|
||||||
|
*/
|
||||||
|
private ImmutableMap<String, BiFunction<JSONObject, Integer, Object>> columnMap;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 多行表头映射表, 当表头由多行数据组成时使用. key=表头, value=该表头对应的每行的内容
|
||||||
|
*/
|
||||||
|
private ImmutableMap<String, List<String>> multiLineHeadMap;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 行数据的转换器，外部可以通过它将原始的行数据根据需要做转换
|
||||||
|
*/
|
||||||
|
private Function<JSONObject, List<JSONObject>> rowConverter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 页数据的转换器，外部可以通过它将也的数据根据需要做转换
|
||||||
|
* 它会在 rowconvert() 执行完成
|
||||||
|
*/
|
||||||
|
private Function<List<JSONObject>, List<JSONObject>> pageConverter;
|
||||||
|
|
||||||
|
private BiConsumer<Integer, String> onProgress;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批注信息，可选
|
||||||
|
*/
|
||||||
|
private Meta meta;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否开启调试日志
|
||||||
|
*/
|
||||||
|
private Boolean debugEnabled = true;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出的fields. 优先使用fields来创建column&meta
|
||||||
|
*/
|
||||||
|
private List<ExportField> fields;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* meta的version
|
||||||
|
*/
|
||||||
|
private String version;
|
||||||
|
|
||||||
|
/** 文件名 */
|
||||||
|
private String fileName;
|
||||||
|
|
||||||
|
/** 操作人, 可选，用于异步导出 */
|
||||||
|
private String operator;
|
||||||
|
|
||||||
|
/** 拓展字段，可用于存放请求参数等，该字段会回传给FileLoaderParam */
|
||||||
|
private JSONObject ext;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 仅对导出excel生效，是否自动清除表格内容的前后空白字符
|
||||||
|
* 默认true，即会清除前后空白字符
|
||||||
|
*/
|
||||||
|
private Boolean autoTrim;
|
||||||
|
|
||||||
|
abstract public Exporter build();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
class ExportField {
|
||||||
|
private String column;
|
||||||
|
private String header;
|
||||||
|
private List<String> headerLines;
|
||||||
|
private Integer width;
|
||||||
|
private CellMeta.Type type;
|
||||||
|
private Boolean mandatory;
|
||||||
|
private Boolean wrapText;
|
||||||
|
|
||||||
|
private String lowerType;
|
||||||
|
private String upperType;
|
||||||
|
private String dateTimeFormat;
|
||||||
|
private List<Object> options;
|
||||||
|
|
||||||
|
private BiFunction<JSONObject, Integer, Object> reader;
|
||||||
|
|
||||||
|
@Builder
|
||||||
|
public ExportField(String column, String header, List<String> headerLines, Integer width, CellMeta.Type type,
|
||||||
|
Boolean mandatory, Boolean wrapText, Function<String, Object> resultConverter,
|
||||||
|
BiFunction<JSONObject, Integer, Object> reader, List<Object> options) {
|
||||||
|
Preconditions.checkArgument(!Strings.isNullOrEmpty(column), "column不能为空");
|
||||||
|
Preconditions.checkArgument(!Strings.isNullOrEmpty(header), "header不能为空");
|
||||||
|
Preconditions.checkArgument(!(resultConverter != null && reader != null), "reader & resultConverter不能同时存在");
|
||||||
|
|
||||||
|
this.column = column;
|
||||||
|
this.header = header;
|
||||||
|
this.headerLines = headerLines;
|
||||||
|
this.width = width;
|
||||||
|
this.type = Optional.ofNullable(type).orElse(CellMeta.Type.STRING);
|
||||||
|
this.mandatory = Optional.ofNullable(mandatory).orElse(Boolean.FALSE);
|
||||||
|
this.wrapText = wrapText;
|
||||||
|
if (reader != null) {
|
||||||
|
this.reader = reader;
|
||||||
|
} else {
|
||||||
|
this.reader = Optional.ofNullable(resultConverter)
|
||||||
|
.map(e -> (BiFunction<JSONObject, Integer, Object>) (row, integer) -> e.apply(Strings.nullToEmpty(row.getString(column))))
|
||||||
|
.orElseGet(() -> DataSheetClient.stringCellReader(column));
|
||||||
|
}
|
||||||
|
this.options = options;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CellMeta toCellMeta() {
|
||||||
|
JSONObject params = new JSONObject();
|
||||||
|
if (CellMeta.Type.RANGE == type) {
|
||||||
|
params.put(EXT_KEY_RANGE_LOWER_TYPE, lowerType);
|
||||||
|
params.put(EXT_KEY_RANGE_UPPER_TYPE, upperType);
|
||||||
|
}
|
||||||
|
if (CellMeta.Type.DATE == type || CellMeta.Type.DATETIME == type) {
|
||||||
|
params.put(EXT_KEY_DATETIME_FORMAT, dateTimeFormat);
|
||||||
|
}
|
||||||
|
|
||||||
|
return CellMeta.builder()
|
||||||
|
.key(column).name(header).width(width).mandatory(mandatory).type(type).wrapText(wrapText)
|
||||||
|
.params(params).options(options).build();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Exporter {
|
||||||
|
/**
|
||||||
|
* 导出到OutputStream中
|
||||||
|
*
|
||||||
|
* @param outputStream
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
ExportResp export(@NonNull OutputStream outputStream);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出到HttpServletResponse中
|
||||||
|
*
|
||||||
|
* @param response
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
ExportResp export(@NonNull HttpServletResponse response);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出到HttpServletResponse中，可指定导出的文件名
|
||||||
|
*
|
||||||
|
* @param response
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
ExportResp export(HttpServletResponse response, String filename);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface AsyncExporter {
|
||||||
|
/**
|
||||||
|
* 异步导出数据
|
||||||
|
*
|
||||||
|
* @param dataSheetExporter 实现导出逻辑的 exporter
|
||||||
|
* @return batchId
|
||||||
|
*/
|
||||||
|
String export(@NonNull ExporterBuilder dataSheetExporter);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 多表单数据异步导出至同一文件中, 仅支持导出格式是EXCEL
|
||||||
|
*
|
||||||
|
* @param fileName 导出文件名
|
||||||
|
* @param dataSheetExporters 实现导出逻辑的 exporter 列表
|
||||||
|
* @return batchId
|
||||||
|
*/
|
||||||
|
String export(String fileName, @NonNull List<ExporterBuilder> dataSheetExporters);
|
||||||
|
|
||||||
|
Progress getProgress(@NonNull String batchId);
|
||||||
|
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
class Progress {
|
||||||
|
String fileName;
|
||||||
|
int progress;
|
||||||
|
String errorMsg;
|
||||||
|
String downloadUrl;
|
||||||
|
|
||||||
|
public boolean isSuccess() {
|
||||||
|
return Strings.isNullOrEmpty(errorMsg);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isDone() {
|
||||||
|
return progress < 0 || progress >= 100;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
class ExportResp {
|
||||||
|
/**
|
||||||
|
* 导出场景
|
||||||
|
*/
|
||||||
|
private String scene;
|
||||||
|
/**
|
||||||
|
* 导出数据条数
|
||||||
|
*/
|
||||||
|
private Long exportedCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出数据总行数，包含了顶部提示行及表头行
|
||||||
|
*/
|
||||||
|
private Long rowCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出数据列数
|
||||||
|
*/
|
||||||
|
private Long columnCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出数据行数
|
||||||
|
*/
|
||||||
|
private Long dataRowCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 耗时
|
||||||
|
*/
|
||||||
|
private Long elapsedMillis;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 操作人
|
||||||
|
*/
|
||||||
|
private String operator;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Builder
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
class Meta {
|
||||||
|
public static final String PATTERN_VERSION_OPEN = "$V{";
|
||||||
|
public static final String PATTERN_CELL_OPEN = "$C{";
|
||||||
|
public static final String PATTERN_IGNORE_OPEN = "$I{";
|
||||||
|
public static final String PATTERN_EXTRA_OPEN = "$E{";
|
||||||
|
public static final String PATTERN_CLOSE = "}";
|
||||||
|
|
||||||
|
public static final String RANGE_TYPE_OPEN_OPEN = "1";
|
||||||
|
public static final String RANGE_TYPE_OPEN_CLOSED = "2";
|
||||||
|
public static final String RANGE_TYPE_CLOSED_OPEN = "3";
|
||||||
|
public static final String RANGE_TYPE_CLOSED_CLOSED = "4";
|
||||||
|
|
||||||
|
public static final String IGNORE_ROW_KEY = "r";
|
||||||
|
public static final String IGNORE_COLUMN_KEY = "c";
|
||||||
|
public static final String IGNORE_ROW_AND_COLUMN_KEY = "b";
|
||||||
|
|
||||||
|
private String templateCode;
|
||||||
|
private String version;
|
||||||
|
private List<CellMeta> cellMetas;
|
||||||
|
/**
|
||||||
|
* 忽略掉的行号
|
||||||
|
*/
|
||||||
|
private List<Integer> ignoreRowIndexes;
|
||||||
|
/**
|
||||||
|
* 忽略掉的列号
|
||||||
|
*/
|
||||||
|
private List<Integer> ignoreColumnIndexes;
|
||||||
|
/**
|
||||||
|
* 忽略掉的列名
|
||||||
|
* 当导入文件中有meta信息时，该ignore信息将会丢失
|
||||||
|
*/
|
||||||
|
private Set<String> ignoreColumnNames;
|
||||||
|
|
||||||
|
public List<Integer> getIgnoreRowIndexes() {
|
||||||
|
return Optional.ofNullable(ignoreRowIndexes).orElseGet(ImmutableList::of);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Integer> getIgnoreColumnIndexes() {
|
||||||
|
return Optional.ofNullable(ignoreColumnIndexes).orElseGet(ImmutableList::of);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Set<String> getIgnoreColumnNames() {
|
||||||
|
return Optional.ofNullable(ignoreColumnNames).orElseGet(ImmutableSet::of);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuperBuilder
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
class CellMeta {
|
||||||
|
public static final String EXT_KEY_DATETIME_FORMAT = "dateTimeFormat";
|
||||||
|
public static final String EXT_KEY_RANGE_LOWER_TYPE = "lowerType";
|
||||||
|
public static final String EXT_KEY_RANGE_UPPER_TYPE = "upperType";
|
||||||
|
|
||||||
|
public static final String RANGE_BOUND_TYPE_OPEN = "open";
|
||||||
|
public static final String RANGE_BOUND_TYPE_CLOSED = "closed";
|
||||||
|
public static final Set<String> RANGE_TYPES = ImmutableSet.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED);
|
||||||
|
|
||||||
|
private static final List<String> DEFAULT_DATE_FORMATS = ImmutableList.of(
|
||||||
|
"yyyy-MM-dd",
|
||||||
|
"yyyy-MM-dd HH:mm",
|
||||||
|
"yyyy-MM-dd HH:mm:ss",
|
||||||
|
"yyyy.MM.dd",
|
||||||
|
"yyyy.MM.dd HH:mm",
|
||||||
|
"yyyy.MM.dd HH:mm:ss",
|
||||||
|
"yyyy年MM月dd日",
|
||||||
|
"yyyy年MM月dd日 HH:mm",
|
||||||
|
"yyyy年MM月dd日 HH:mm:ss",
|
||||||
|
"yyyy年MM月dd日 HH时mm分",
|
||||||
|
"yyyy年MM月dd日 HH时mm分ss秒",
|
||||||
|
"yyyy/MM/dd",
|
||||||
|
"yyyy/MM/dd HH:mm",
|
||||||
|
"yyyy/MM/dd HH:mm:ss",
|
||||||
|
"yyyy-MM-dd'T'HH:mm:ss'Z'",
|
||||||
|
"yyyy-MM-dd'T'HH:mm:ssZ",
|
||||||
|
"yyyy-MM-dd'T'HH:mm:ssz",
|
||||||
|
"yyyy-MM-dd'T'HH:mm:ss",
|
||||||
|
"MM/dd/yyyy HH:mm:ss a",
|
||||||
|
"yyyyMMddHHmmss",
|
||||||
|
"yyyyMMdd"
|
||||||
|
);
|
||||||
|
|
||||||
|
private static final Map<String, Boolean> BOOLEAN_MAP = ImmutableMap.<String, Boolean>builder()
|
||||||
|
.put("是", Boolean.TRUE)
|
||||||
|
.put("yes", Boolean.TRUE)
|
||||||
|
.put("y", Boolean.TRUE)
|
||||||
|
.put("1", Boolean.TRUE)
|
||||||
|
.put("否", Boolean.FALSE)
|
||||||
|
.put("no", Boolean.FALSE)
|
||||||
|
.put("n", Boolean.FALSE)
|
||||||
|
.put("0", Boolean.FALSE)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* cell的key
|
||||||
|
*/
|
||||||
|
private String key;
|
||||||
|
/**
|
||||||
|
* cell的header，一般为中文
|
||||||
|
*/
|
||||||
|
private String name;
|
||||||
|
/**
|
||||||
|
* cell的宽度
|
||||||
|
*/
|
||||||
|
private Integer width;
|
||||||
|
/**
|
||||||
|
* 类型
|
||||||
|
*/
|
||||||
|
private Type type;
|
||||||
|
/**
|
||||||
|
* 是否必需, 为false时, 允许该列在Excel中不存在
|
||||||
|
*/
|
||||||
|
private Boolean mandatory;
|
||||||
|
/**
|
||||||
|
* 可选值
|
||||||
|
*/
|
||||||
|
private List<Object> options;
|
||||||
|
/**
|
||||||
|
* 存放一些和type相关的参数，例如DateTime的pattern格式，Range的开闭区间
|
||||||
|
*/
|
||||||
|
private JSONObject params;
|
||||||
|
/**
|
||||||
|
* 存放一些额外的信息
|
||||||
|
*/
|
||||||
|
private JSONObject ext;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* wrap the text automatically
|
||||||
|
*/
|
||||||
|
private Boolean wrapText;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导入的时候，将原始的String转换为特定的类型
|
||||||
|
*/
|
||||||
|
private Function<String, Object> importConverter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 字段值额外的校验方法，支持根据当前字段值及整行数据进行自定义校验
|
||||||
|
*/
|
||||||
|
private BiConsumer<Object, JSONObject> cellValidator;
|
||||||
|
|
||||||
|
public void validate() {
|
||||||
|
if (type == Type.RANGE
|
||||||
|
&& (!RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_LOWER_TYPE))
|
||||||
|
|| !RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_UPPER_TYPE)))) {
|
||||||
|
throw ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR.toException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Object convertType(String value) {
|
||||||
|
if (importConverter != null) {
|
||||||
|
return importConverter.apply(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Boolean.TRUE.equals(mandatory) && Strings.isNullOrEmpty(value)) {
|
||||||
|
throw ResultCode.IMPORT_CELL_META_MISSING_MANDATORY_VALUE.toException();
|
||||||
|
}
|
||||||
|
// 非必填的字段如果为空，直接返回
|
||||||
|
if (value == null) {
|
||||||
|
return type == Type.STRING ? StringUtils.EMPTY : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (type) {
|
||||||
|
case INT:
|
||||||
|
// 移除千分符
|
||||||
|
return Integer.valueOf(value.replace(",", ""));
|
||||||
|
case LONG:
|
||||||
|
// 移除千分符
|
||||||
|
return Long.valueOf(value.replace(",", ""));
|
||||||
|
case FLOAT:
|
||||||
|
// 移除千分符
|
||||||
|
return Float.valueOf(value.replace(",", ""));
|
||||||
|
case DOUBLE:
|
||||||
|
// 移除千分符
|
||||||
|
return Double.valueOf(value.replace(",", ""));
|
||||||
|
case BIG_DECIMAL:
|
||||||
|
// 移除千分符
|
||||||
|
return new BigDecimal(value.replace(",", ""));
|
||||||
|
case DATE:
|
||||||
|
case DATETIME:
|
||||||
|
return parseDate(value, DEFAULT_DATE_FORMATS);
|
||||||
|
case RANGE:
|
||||||
|
// 格式 100-200
|
||||||
|
List<Integer> values = Splitter.on("-").omitEmptyStrings().trimResults().splitToList(value)
|
||||||
|
.stream()
|
||||||
|
.map(Integer::valueOf)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
if (values.size() != 2) {
|
||||||
|
throw ResultCode.IMPORT_CELL_RANGE_FORMAT_ERROR.toException();
|
||||||
|
}
|
||||||
|
if (values.get(0) >= values.get(1)) {
|
||||||
|
throw ResultCode.IMPORT_CELL_RANGE_VALUE_ERROR.toException();
|
||||||
|
}
|
||||||
|
return new JSONObject()
|
||||||
|
.fluentPut("lower", values.get(0))
|
||||||
|
.fluentPut("upper", values.get(1))
|
||||||
|
.fluentPut(EXT_KEY_RANGE_LOWER_TYPE, params.getString(EXT_KEY_RANGE_LOWER_TYPE))
|
||||||
|
.fluentPut(EXT_KEY_RANGE_UPPER_TYPE, params.getString(EXT_KEY_RANGE_UPPER_TYPE));
|
||||||
|
case BOOLEAN:
|
||||||
|
return Optional.ofNullable(BOOLEAN_MAP.get(value.toLowerCase()))
|
||||||
|
.orElseThrow(ResultCode.IMPORT_CELL_BOOLEAN_VALUE_ERROR::toException);
|
||||||
|
default:
|
||||||
|
return Strings.nullToEmpty(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private LocalDateTime parseDate(String dateStr, List<String> formats) {
|
||||||
|
try {
|
||||||
|
Date date = DateUtils.parseDate(dateStr, formats.toArray(new String[0]));
|
||||||
|
return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw ResultCode.IMPORT_CELL_DATETIME_CONVERT_FAILED.toException("不支持的日期格式{}", dateStr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@AllArgsConstructor
|
||||||
|
public enum Type {
|
||||||
|
INT("00"),
|
||||||
|
LONG("01"),
|
||||||
|
FLOAT("02"),
|
||||||
|
DOUBLE("03"),
|
||||||
|
STRING("04"),
|
||||||
|
BIG_DECIMAL("05"),
|
||||||
|
DATE("06"),
|
||||||
|
DATETIME("07"),
|
||||||
|
RANGE("08"),
|
||||||
|
BOOLEAN("09");
|
||||||
|
|
||||||
|
@Getter
|
||||||
|
private String code;
|
||||||
|
|
||||||
|
private static Map<String, Type> map = Stream.of(Type.values())
|
||||||
|
.collect(Collectors.toMap(Type::getCode, Function.identity()));
|
||||||
|
|
||||||
|
public static Type from(String code) {
|
||||||
|
return map.get(code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Getter
|
||||||
|
enum ExportFormat {
|
||||||
|
EXCEL("application/vnd.ms-excel", ".xlsx"),
|
||||||
|
CSV("text/csv", ".csv");
|
||||||
|
|
||||||
|
private String contentType;
|
||||||
|
private String suffix;
|
||||||
|
}
|
||||||
|
|
||||||
|
static BiFunction<JSONObject, Integer, Object> indexCellReader() {
|
||||||
|
return (row, index) -> String.valueOf(index + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
static BiFunction<JSONObject, Integer, Object> stringCellReader(String columnName) {
|
||||||
|
// excel库poi会检查文本格式的单元格内容，当超过32767时，将会抛出IllegalArgumentException异常
|
||||||
|
// 参考 {@link org.apache.poi.xssf.streaming.SXSSFCell.setCellValue}
|
||||||
|
return (row, index) -> StringUtils.left(Strings.nullToEmpty(row.getString(columnName)),
|
||||||
|
SpreadsheetVersion.EXCEL2007.getMaxTextLength());
|
||||||
|
}
|
||||||
|
|
||||||
|
static BiFunction<JSONObject, Integer, Object> dateTimeCellReader(String columnName, String pattern) {
|
||||||
|
return (row, index) -> {
|
||||||
|
Long epochMillis = row.getLong(columnName);
|
||||||
|
if (epochMillis == null) {
|
||||||
|
return StringUtils.EMPTY;
|
||||||
|
}
|
||||||
|
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
|
||||||
|
Instant instant = Instant.ofEpochMilli(epochMillis);
|
||||||
|
LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
|
||||||
|
return formatter.format(localDateTime);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
static BiFunction<JSONObject, Integer, Object> bigDecimalCellReader(String columnName) {
|
||||||
|
return bigDecimalCellReader(columnName, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
static BiFunction<JSONObject, Integer, Object> bigDecimalCellReader(String columnName, int scale) {
|
||||||
|
return (row, index) -> {
|
||||||
|
BigDecimal number = row.getBigDecimal(columnName);
|
||||||
|
if (number == null) {
|
||||||
|
number = BigDecimal.ZERO;
|
||||||
|
}
|
||||||
|
|
||||||
|
return number.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
static BiFunction<JSONObject, Integer, Object> jsonPathCellReader(String jsonPath) {
|
||||||
|
return (row, index) -> {
|
||||||
|
Object obj = JSONPath.eval(row, jsonPath);
|
||||||
|
/**
|
||||||
|
* excel单元格对于数值最大只能保有15位，超过15位的数值(Long)会被置为0，比如在导出uid的场景，会导致数据错误
|
||||||
|
* https://docs.microsoft.com/en-us/office/troubleshoot/excel/last-digits-changed-to-zeros
|
||||||
|
* https://wenku.baidu.com/view/e1d2a497f624ccbff121dd36a32d7375a417c6ad.html
|
||||||
|
*/
|
||||||
|
if (obj instanceof Long && obj.toString().length() > 15) {
|
||||||
|
obj = obj.toString();
|
||||||
|
}
|
||||||
|
if (obj instanceof String) {
|
||||||
|
// excel库poi会检查文本格式的单元格内容，当超过32767时，将会抛出IllegalArgumentException异常
|
||||||
|
// 参考 {@link org.apache.poi.xssf.streaming.SXSSFCell.setCellValue}
|
||||||
|
return StringUtils.left(obj.toString(), SpreadsheetVersion.EXCEL2007.getMaxTextLength());
|
||||||
|
}
|
||||||
|
return obj;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
static BiFunction<JSONObject, Integer, Object> jsonPathDateTimeCellReader(String jsonPath, String pattern) {
|
||||||
|
return (row, index) -> {
|
||||||
|
Object val = JSONPath.eval(row, jsonPath);
|
||||||
|
Long epochMillis = TypeUtils.cast(val, Long.class, ParserConfig.getGlobalInstance());
|
||||||
|
if (epochMillis == null) {
|
||||||
|
return StringUtils.EMPTY;
|
||||||
|
}
|
||||||
|
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
|
||||||
|
Instant instant = Instant.ofEpochMilli(epochMillis);
|
||||||
|
LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
|
||||||
|
return formatter.format(localDateTime);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
static BiFunction<JSONObject, Integer, Object> jsonPathBigDecimalCellReader(String jsonPath) {
|
||||||
|
return jsonPathBigDecimalCellReader(jsonPath, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
static BiFunction<JSONObject, Integer, Object> jsonPathBigDecimalCellReader(String jsonPath, int scale) {
|
||||||
|
return (row, index) -> {
|
||||||
|
Object val = JSONPath.eval(row, jsonPath);
|
||||||
|
if (val == null) {
|
||||||
|
val = BigDecimal.ZERO;
|
||||||
|
}
|
||||||
|
BigDecimal bigDecimal = TypeUtils.cast(val, BigDecimal.class, ParserConfig.getGlobalInstance());
|
||||||
|
return bigDecimal.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
@Getter
|
||||||
|
class DataSheetException extends BusinessException {
|
||||||
|
private String subErrorCode;
|
||||||
|
/** 异常相关的行号，列号，0开始 */
|
||||||
|
private Integer rowIndex;
|
||||||
|
private Integer columnIndex;
|
||||||
|
/** 异常相关的列名称，中文 */
|
||||||
|
private List<String> columnNames;
|
||||||
|
|
||||||
|
public DataSheetException(ResultCode resultCode) {
|
||||||
|
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage());
|
||||||
|
this.subErrorCode = resultCode.getSubBizCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetException(ResultCode resultCode, Integer rowIndex, Integer columnIndex) {
|
||||||
|
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage());
|
||||||
|
this.subErrorCode = resultCode.getSubBizCode();
|
||||||
|
this.rowIndex = rowIndex;
|
||||||
|
this.columnIndex = columnIndex;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetException(ResultCode resultCode, List<String> columnNames) {
|
||||||
|
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage());
|
||||||
|
this.subErrorCode = resultCode.getSubBizCode();
|
||||||
|
this.columnNames = columnNames;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetException(ResultCode resultCode, String message) {
|
||||||
|
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), message);
|
||||||
|
this.subErrorCode = resultCode.getSubBizCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetException(String subErrorCode, String errorMsg, Integer rowIndex, Integer columnIndex) {
|
||||||
|
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), errorMsg);
|
||||||
|
this.subErrorCode = subErrorCode;
|
||||||
|
this.rowIndex = rowIndex;
|
||||||
|
this.columnIndex = columnIndex;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetException(String errorCode, String subErrorCode, String errorMsg) {
|
||||||
|
super(errorCode, errorMsg);
|
||||||
|
this.subErrorCode = subErrorCode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Getter
|
||||||
|
enum ResultCode {
|
||||||
|
/** 解析Excel的批注时，相关的errorCode */
|
||||||
|
IMPORT_PARSE_MISSING_VERSION("C00", "批注中没有找到版本信息"),
|
||||||
|
IMPORT_PARSE_VERSION_FORMAT_ERROR("C01", "批注中的版本格式不对"),
|
||||||
|
IMPORT_PARSE_CELL_META_MISSING_TYPE("C02", "批注中的字段没有找到类型信息"),
|
||||||
|
IMPORT_PARSE_CELL_META_FORMAT_ERROR("C03", "批注中的字段格式不对"),
|
||||||
|
IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR("C04", "批注中的范围类型字段的格式不对"),
|
||||||
|
IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR("C05", "批注中的范围类型字段的区间类型不对"),
|
||||||
|
|
||||||
|
/** 导入数据时，相关的errorCode */
|
||||||
|
IMPORT_LINES_REACHED_LIMIT("C20", "导入的数据超过了最大行数"),
|
||||||
|
IMPORT_COLUMN_RANGE_MISSING_TYPES("C21", "范围类型缺少区间类型的定义：开区间还是闭区间"),
|
||||||
|
IMPORT_COLUMN_DUPLICATED_NAME("C22", "列的名称不能重复"),
|
||||||
|
IMPORT_COLUMN_MISSING_CELL_META("C23", "字段缺少类型定义"),
|
||||||
|
IMPORT_COLUMN_NAME_NOT_MATCHED("C24", "字段的名称与类型定义的名称不一致"),
|
||||||
|
IMPORT_CELL_RANGE_FORMAT_ERROR("C25", "范围类型的格式不对"),
|
||||||
|
IMPORT_CELL_RANGE_VALUE_ERROR("C26", "范围类型的下限值必须小于上限值"),
|
||||||
|
IMPORT_CELL_DATETIME_CONVERT_FAILED("C27", "时间类型解析失败"),
|
||||||
|
IMPORT_CELL_CONVERT_FAILED("C28", "类型转换失败"),
|
||||||
|
IMPORT_CELL_META_MISSING_MANDATORY_VALUE("C29", "必填字段不能为空"),
|
||||||
|
IMPORT_CELL_BOOLEAN_VALUE_ERROR("C30", "布尔类型的值不支持"),
|
||||||
|
;
|
||||||
|
|
||||||
|
private String subBizCode;
|
||||||
|
private String message;
|
||||||
|
|
||||||
|
public DataSheetException toException() {
|
||||||
|
return new DataSheetException(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetException toException(String message) {
|
||||||
|
return new DataSheetException(this, message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetException toException(List<String> columnNames) {
|
||||||
|
return new DataSheetException(this, columnNames);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetException toException(Integer rowIndex, Integer columnIndex) {
|
||||||
|
return new DataSheetException(this, rowIndex, columnIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetException toException(String customMsg, Object... objects) {
|
||||||
|
if (objects == null) {
|
||||||
|
return toException(customMsg);
|
||||||
|
}
|
||||||
|
|
||||||
|
String msg = VarParamFormatter.format(customMsg, objects);
|
||||||
|
//如果最后一个参数是Throwable. 则将SimpleName附加到msg中
|
||||||
|
if (objects[objects.length - 1] instanceof Throwable) {
|
||||||
|
Throwable throwable = (Throwable) objects[objects.length - 1];
|
||||||
|
msg = String.format("%s (%s)", msg, throwable.getClass().getSimpleName());
|
||||||
|
}
|
||||||
|
|
||||||
|
return toException(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,326 @@
|
|||||||
|
package cn.axzo.foundation.excel.support.impl;
|
||||||
|
|
||||||
|
import cn.axzo.foundation.enums.AppEnvEnum;
|
||||||
|
import cn.axzo.foundation.excel.support.DataSheetClient;
|
||||||
|
import cn.axzo.foundation.util.UUIDBuilder;
|
||||||
|
import cn.axzo.foundation.web.support.AppRuntime;
|
||||||
|
import cn.axzo.foundation.web.support.utils.KeyBuilder;
|
||||||
|
import com.alibaba.excel.EasyExcel;
|
||||||
|
import com.alibaba.excel.ExcelWriter;
|
||||||
|
import com.alibaba.excel.context.AnalysisContext;
|
||||||
|
import com.alibaba.excel.event.AnalysisEventListener;
|
||||||
|
import com.alibaba.excel.util.IoUtils;
|
||||||
|
import com.alibaba.excel.util.StringUtils;
|
||||||
|
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||||
|
import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.NonNull;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.core.task.TaskExecutor;
|
||||||
|
import org.springframework.data.redis.core.RedisTemplate;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
import java.io.*;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.function.Function;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 异步数据导出.
|
||||||
|
* 将导出的进度防止redis 中，当导出完成后，将文件通过 fileUploader 上传到云存储。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class DataSheetAsyncExporter implements DataSheetClient.AsyncExporter {
|
||||||
|
|
||||||
|
private static final int PROGRESS_EXPIRE_DAYS = 3;
|
||||||
|
private RedisTemplate<String, String> redisTemplate;
|
||||||
|
private AppRuntime appRuntime;
|
||||||
|
private Function<FileUploadParam, FileUploadResult> fileUploader;
|
||||||
|
private TaskExecutor executor;
|
||||||
|
|
||||||
|
public DataSheetAsyncExporter(@NonNull RedisTemplate<String, String> redisTemplate,
|
||||||
|
@NonNull AppRuntime appRuntime,
|
||||||
|
@NonNull Function<FileUploadParam, FileUploadResult> fileUploader,
|
||||||
|
@NonNull TaskExecutor executor) {
|
||||||
|
this.redisTemplate = redisTemplate;
|
||||||
|
this.appRuntime = appRuntime;
|
||||||
|
this.fileUploader = fileUploader;
|
||||||
|
this.executor = executor;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@lombok.Builder
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
public static class FileUploadParam {
|
||||||
|
private byte[] bytes;
|
||||||
|
private String fileName;
|
||||||
|
private String operator;
|
||||||
|
private JSONObject ext;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@lombok.Builder
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
public static class FileUploadResult {
|
||||||
|
private String downloadUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String export(@NonNull DataSheetClient.ExporterBuilder dataSheetExporter) {
|
||||||
|
final String batchId = UUIDBuilder.generateLongUuid(false);
|
||||||
|
final String scene = dataSheetExporter.scene();
|
||||||
|
ExportContext context = ExportContext.builder().batchId(batchId).fileName(dataSheetExporter.fileName())
|
||||||
|
.operator(dataSheetExporter.operator()).ext(dataSheetExporter.ext()).build();
|
||||||
|
executor.execute(() -> doExport(context, scene, dataSheetExporter));
|
||||||
|
return batchId;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String export(String fileName, @NonNull List<DataSheetClient.ExporterBuilder> dataSheetExporters) {
|
||||||
|
checkMultiExporters(fileName, dataSheetExporters);
|
||||||
|
|
||||||
|
final String batchId = UUIDBuilder.generateLongUuid(false);
|
||||||
|
ExportContext context = ExportContext.builder()
|
||||||
|
.batchId(batchId)
|
||||||
|
.fileName(fileName)
|
||||||
|
.operator(dataSheetExporters.get(0).operator())
|
||||||
|
.writer(EasyExcel.write(new ByteArrayOutputStream())
|
||||||
|
.registerWriteHandler(new LongestMatchColumnWidthStyleStrategy())
|
||||||
|
.build())
|
||||||
|
.progressMap(dataSheetExporters.stream().collect(Collectors.toMap(e -> e.sheetName(), e -> 0)))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
executor.execute(() -> dataSheetExporters.forEach(e -> doExport(context, e.scene(), e)));
|
||||||
|
return batchId;
|
||||||
|
}
|
||||||
|
|
||||||
|
void doExport(final ExportContext context, final String scene, DataSheetClient.ExporterBuilder dataSheetExporter) {
|
||||||
|
try {
|
||||||
|
final File tempFile = File.createTempFile(context.getBatchId(), ".tmp");
|
||||||
|
tempFile.deleteOnExit();
|
||||||
|
log.info("DataSheetAsyncExporter export() scene={} context={}, tempFile->{}", scene, context, tempFile.getAbsolutePath());
|
||||||
|
try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) {
|
||||||
|
DataSheetClient.Exporter exporter = dataSheetExporter.onProgress((progress, errorMsg) -> {
|
||||||
|
if (context.isMultiSheet()) {
|
||||||
|
onMultiSheetProgress(progress, errorMsg, context, tempFile, dataSheetExporter.sheetName());
|
||||||
|
} else {
|
||||||
|
onProgress(progress, errorMsg, context, tempFile);
|
||||||
|
}
|
||||||
|
}).build();
|
||||||
|
|
||||||
|
Preconditions.checkState(exporter != null);
|
||||||
|
exporter.export(out);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("DataSheetAsyncExporter export() FAIL scene={} context={} ", scene, context, e);
|
||||||
|
} finally {
|
||||||
|
if (tempFile != null && tempFile.exists()) {
|
||||||
|
tempFile.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkMultiExporters(String fileName, List<DataSheetClient.ExporterBuilder> dataSheetExporters) {
|
||||||
|
Preconditions.checkArgument(!Strings.isNullOrEmpty(fileName), "导出文件名不能为空");
|
||||||
|
Preconditions.checkArgument(!CollectionUtils.isEmpty(dataSheetExporters), "导出参数不能为空");
|
||||||
|
dataSheetExporters.forEach(e -> Preconditions.checkArgument(e.format() == DataSheetClient.ExportFormat.EXCEL, "导出格式必须为EXCEL"));
|
||||||
|
dataSheetExporters.forEach(e -> Preconditions.checkArgument(!Strings.isNullOrEmpty(e.sheetName()), "表单名不能为空"));
|
||||||
|
|
||||||
|
long sheetCount = dataSheetExporters.stream().map(DataSheetClient.ExporterBuilder::sheetName).distinct().count();
|
||||||
|
Preconditions.checkArgument(sheetCount == dataSheetExporters.size(), "表单名不能重复");
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildRedisKey(String batchId) {
|
||||||
|
return KeyBuilder.build(appRuntime, "data_export", batchId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void onProgress(Integer progress, String errorMsg, ExportContext context, File tempFile) {
|
||||||
|
log.info("DataSheetAsyncExporter onProgress() progress={} errorMsg={} context={} tempFile={}", progress, errorMsg, context, tempFile.getAbsolutePath());
|
||||||
|
String downloadUrl = StringUtils.EMPTY;
|
||||||
|
if (Strings.isNullOrEmpty(errorMsg) && progress >= 100) {
|
||||||
|
try {
|
||||||
|
// 只有导出成功，并完成，才开始上传文件
|
||||||
|
byte[] bytes = IoUtils.toByteArray(new BufferedInputStream(new FileInputStream(tempFile)));
|
||||||
|
downloadUrl = fileUploader.apply(FileUploadParam.builder()
|
||||||
|
.bytes(bytes).fileName(context.getFileName())
|
||||||
|
.operator(context.getOperator()).ext(context.getExt()).build()).getDownloadUrl();
|
||||||
|
log.info("DataSheetAsyncExporter onProgress() DONE context={}, downloadUrl={}", context, downloadUrl);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 将导出的状态放入 redis 中。后续通过getProgress()来获取。
|
||||||
|
String jsonString = JSONObject.toJSONString(Progress.builder().fileName(context.getFileName()).progress(progress)
|
||||||
|
.downloadUrl(downloadUrl).errorMsg(errorMsg).build());
|
||||||
|
redisTemplate.opsForValue().set(buildRedisKey(context.getBatchId()), jsonString, PROGRESS_EXPIRE_DAYS, TimeUnit.DAYS);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void onMultiSheetProgress(Integer progress, String errorMsg, ExportContext context, File tempFile, String sheetName) {
|
||||||
|
log.info("DataSheetAsyncExporter onMultiSheetProgress() context={}, progress={} errorMsg={} tempFile={} sheetName={}", context, progress, errorMsg, tempFile.getAbsolutePath(), sheetName);
|
||||||
|
context.getProgressMap().put(sheetName, progress);
|
||||||
|
Progress totalProgress = getMultiSheetProgress(context, errorMsg);
|
||||||
|
|
||||||
|
if (!Strings.isNullOrEmpty(errorMsg) && progress < 0) {
|
||||||
|
context.getWriter().write(ImmutableList.of(ImmutableList.of(errorMsg)),
|
||||||
|
EasyExcel.writerSheet(sheetName).build());
|
||||||
|
} else if (progress >= 100) {
|
||||||
|
EasyExcel.read(tempFile, new ReadAndWriteListener(context, sheetName)).sheet(sheetName).doRead();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (totalProgress.isDone()) {
|
||||||
|
// 只有所有表单都导出完成，才开始上传文件
|
||||||
|
context.getWriter().finish();
|
||||||
|
ByteArrayOutputStream outputStream = (ByteArrayOutputStream) context.getWriter().writeContext()
|
||||||
|
.writeWorkbookHolder().getOutputStream();
|
||||||
|
String downloadUrl = fileUploader.apply(FileUploadParam.builder()
|
||||||
|
.bytes(outputStream.toByteArray()).fileName(context.getFileName())
|
||||||
|
.operator(context.getOperator()).build()).getDownloadUrl();
|
||||||
|
log.info("DataSheetAsyncExporter onMultiSheetProgress() DONE context={}, downloadUrl={}", context, downloadUrl);
|
||||||
|
totalProgress.setDownloadUrl(downloadUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 将导出的状态放入 redis 中。后续通过getProgress()来获取。
|
||||||
|
String jsonString = JSONObject.toJSONString(totalProgress);
|
||||||
|
redisTemplate.opsForValue().set(buildRedisKey(context.getBatchId()), jsonString, PROGRESS_EXPIRE_DAYS, TimeUnit.DAYS);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Progress getMultiSheetProgress(ExportContext context, String errorMsg) {
|
||||||
|
Map<String, Integer> progressMap = context.getProgressMap();
|
||||||
|
int totalProgress = progressMap.values().stream().mapToInt(a -> a).sum() / progressMap.size();
|
||||||
|
boolean isDone = progressMap.values().stream().allMatch(a -> a < 0 || a >= 100);
|
||||||
|
if (isDone && totalProgress != 100) {
|
||||||
|
// 所有导出都已完成，但其中有失败，将progress设为-1
|
||||||
|
totalProgress = -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Progress.builder()
|
||||||
|
.fileName(context.getFileName())
|
||||||
|
.progress(totalProgress)
|
||||||
|
.errorMsg(errorMsg)
|
||||||
|
.downloadUrl(StringUtils.EMPTY)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用在export()返回的 batchId 来查询导出进度。
|
||||||
|
*
|
||||||
|
* @param batchId
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public Progress getProgress(String batchId) {
|
||||||
|
String res = redisTemplate.opsForValue().get(buildRedisKey(batchId));
|
||||||
|
if (Strings.isNullOrEmpty(res)) {
|
||||||
|
return Progress.builder().fileName(StringUtils.EMPTY)
|
||||||
|
.progress(0).errorMsg(StringUtils.EMPTY).downloadUrl(StringUtils.EMPTY).build();
|
||||||
|
}
|
||||||
|
return JSONObject.parseObject(res, Progress.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public static class Builder {
|
||||||
|
private RedisTemplate<String, String> redisTemplate;
|
||||||
|
private AppRuntime appRuntime;
|
||||||
|
private Function<FileUploadParam, FileUploadResult> fileUploader;
|
||||||
|
private TaskExecutor executor;
|
||||||
|
|
||||||
|
public Builder redisTemplate(RedisTemplate redisTemplate) {
|
||||||
|
this.redisTemplate = redisTemplate;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder appRuntime(AppRuntime appRuntime) {
|
||||||
|
this.appRuntime = appRuntime;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder fileUploader(Function<FileUploadParam, FileUploadResult> fileUploader) {
|
||||||
|
this.fileUploader = fileUploader;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder executor(TaskExecutor executor) {
|
||||||
|
this.executor = executor;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataSheetClient.AsyncExporter build() {
|
||||||
|
if (appRuntime.getEnv() == AppEnvEnum.unittest) {
|
||||||
|
return new DataSheetClient.AsyncExporter() {
|
||||||
|
@Override
|
||||||
|
public String export(DataSheetClient.@NonNull ExporterBuilder dataSheetExporter) {
|
||||||
|
return "8888";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String export(String fileName, @NonNull List<DataSheetClient.ExporterBuilder> dataSheetExporters) {
|
||||||
|
return "8888";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Progress getProgress(String batchId) {
|
||||||
|
return Progress.builder().progress(100).downloadUrl("http://test/test.jpg").build();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return new DataSheetAsyncExporter(redisTemplate, this.appRuntime, this.fileUploader, this.executor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class ReadAndWriteListener extends AnalysisEventListener<Map<Integer, String>> {
|
||||||
|
private ExportContext context;
|
||||||
|
private WriteSheet sheet;
|
||||||
|
|
||||||
|
public ReadAndWriteListener(ExportContext context, String sheetName) {
|
||||||
|
this.context = context;
|
||||||
|
this.sheet = EasyExcel.writerSheet(sheetName).build();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void invoke(Map<Integer, String> data, AnalysisContext context) {
|
||||||
|
List<String> row = data.entrySet().stream().sorted(Map.Entry.comparingByKey())
|
||||||
|
.map(Map.Entry::getValue).collect(Collectors.toList());
|
||||||
|
this.context.getWriter().write(ImmutableList.of(row), sheet);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
|
||||||
|
List<String> row = headMap.entrySet().stream().sorted(Map.Entry.comparingByKey())
|
||||||
|
.map(Map.Entry::getValue).collect(Collectors.toList());
|
||||||
|
this.context.getWriter().write(ImmutableList.of(row), sheet);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@lombok.Builder
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
static class ExportContext {
|
||||||
|
private String batchId;
|
||||||
|
private String fileName;
|
||||||
|
private String operator;
|
||||||
|
private ExcelWriter writer;
|
||||||
|
private Map<String, Integer> progressMap;
|
||||||
|
private JSONObject ext;
|
||||||
|
|
||||||
|
private boolean isMultiSheet() {
|
||||||
|
return writer != null && !CollectionUtils.isEmpty(progressMap);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,90 @@
|
|||||||
|
package cn.axzo.foundation.excel.support.impl;
|
||||||
|
|
||||||
|
import cn.axzo.foundation.excel.support.DataSheetClient;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.cglib.beans.BeanMap;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class DataSheetClientImpl implements DataSheetClient {
|
||||||
|
|
||||||
|
private final static String UNKNOWN_OPERATOR = "unknown@fiture.com";
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ExporterBuilder exportBuilder() {
|
||||||
|
DataSheetExporter exporter = new DataSheetExporter();
|
||||||
|
exporter.setOnCompleted(this::reportExport);
|
||||||
|
return exporter;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ImporterBuilder importBuilder() {
|
||||||
|
DataSheetImporter importer = new DataSheetImporter();
|
||||||
|
importer.setOnCompleted(this::reportImport);
|
||||||
|
return importer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void reportExport(DataSheetClient.ExportResp resp) {
|
||||||
|
String operator = resp.getOperator();
|
||||||
|
|
||||||
|
ReportReq req = ReportReq.builder()
|
||||||
|
.scene(resp.getScene())
|
||||||
|
.action(ReportReq.Action.EXPORT)
|
||||||
|
.resp(BeanMap.create(resp))
|
||||||
|
.rowCount(resp.getRowCount())
|
||||||
|
.elapsedMillis(resp.getElapsedMillis())
|
||||||
|
.operatorId(operator)
|
||||||
|
.operatorName(operator)
|
||||||
|
.build();
|
||||||
|
report(req);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void reportImport(DataSheetClient.ImportResp resp) {
|
||||||
|
String operator = resp.getOperator();
|
||||||
|
|
||||||
|
JSONObject jsonObject = new JSONObject(new HashMap<>(BeanMap.create(resp)));
|
||||||
|
ReportReq req = ReportReq.builder()
|
||||||
|
.scene(resp.getScene())
|
||||||
|
.action(ReportReq.Action.IMPORT)
|
||||||
|
.resp(jsonObject.fluentRemove("lines").fluentRemove("meta"))
|
||||||
|
.rowCount(resp.getRowCount().longValue())
|
||||||
|
.elapsedMillis(resp.getElapsedMillis())
|
||||||
|
.operatorId(operator)
|
||||||
|
.operatorName(operator)
|
||||||
|
.build();
|
||||||
|
report(req);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean report(ReportReq req) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@lombok.Builder
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
private static class ReportReq {
|
||||||
|
String appName;
|
||||||
|
String scene;
|
||||||
|
Action action;
|
||||||
|
Map resp;
|
||||||
|
Long rowCount;
|
||||||
|
Long elapsedMillis;
|
||||||
|
String filePath;
|
||||||
|
String operatorId;
|
||||||
|
String operatorName;
|
||||||
|
String operatorTenantId;
|
||||||
|
|
||||||
|
public enum Action {
|
||||||
|
IMPORT,
|
||||||
|
EXPORT;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,698 @@
|
|||||||
|
package cn.axzo.foundation.excel.support.impl;
|
||||||
|
|
||||||
|
import cn.axzo.foundation.excel.support.DataSheetClient;
|
||||||
|
import cn.axzo.foundation.exception.BusinessException;
|
||||||
|
import cn.axzo.foundation.page.PageReq;
|
||||||
|
import cn.axzo.foundation.page.PageResp;
|
||||||
|
import cn.axzo.foundation.result.ResultCode;
|
||||||
|
import com.alibaba.excel.EasyExcel;
|
||||||
|
import com.alibaba.excel.ExcelWriter;
|
||||||
|
import com.alibaba.excel.metadata.Head;
|
||||||
|
import com.alibaba.excel.metadata.data.WriteCellData;
|
||||||
|
import com.alibaba.excel.util.StringUtils;
|
||||||
|
import com.alibaba.excel.write.builder.ExcelWriterBuilder;
|
||||||
|
import com.alibaba.excel.write.handler.AbstractCellWriteHandler;
|
||||||
|
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||||
|
import com.alibaba.excel.write.metadata.holder.WriteSheetHolder;
|
||||||
|
import com.alibaba.excel.write.metadata.holder.WriteTableHolder;
|
||||||
|
import com.alibaba.excel.write.style.AbstractCellStyleStrategy;
|
||||||
|
import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy;
|
||||||
|
import com.alibaba.excel.write.style.column.SimpleColumnWidthStyleStrategy;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.google.common.base.Joiner;
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
import com.google.common.base.Stopwatch;
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import com.google.common.collect.*;
|
||||||
|
import com.opencsv.CSVWriter;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.codec.CharEncoding;
|
||||||
|
import org.apache.commons.lang3.BooleanUtils;
|
||||||
|
import org.apache.poi.ss.usermodel.*;
|
||||||
|
import org.apache.poi.ss.util.CellRangeAddressList;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFClientAnchor;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
import java.net.URLEncoder;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.function.BiFunction;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import java.util.stream.IntStream;
|
||||||
|
|
||||||
|
import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*;
|
||||||
|
import static cn.axzo.foundation.excel.support.DataSheetClient.Meta.*;
|
||||||
|
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class DataSheetExporter extends DataSheetClient.ExporterBuilder implements DataSheetClient.Exporter {
|
||||||
|
/** 完成后处理事件. */
|
||||||
|
@Setter
|
||||||
|
private Consumer<DataSheetClient.ExportResp> onCompleted;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataSheetClient.Exporter build() {
|
||||||
|
|
||||||
|
if (!CollectionUtils.isEmpty(this.fields())) {
|
||||||
|
|
||||||
|
if (this.columnMap() != null) {
|
||||||
|
throw new RuntimeException("column map is present, please remove column or fields");
|
||||||
|
}
|
||||||
|
if (this.meta() != null) {
|
||||||
|
throw new RuntimeException("meta is present, please remove meta or fields");
|
||||||
|
}
|
||||||
|
|
||||||
|
ImmutableMap<String, BiFunction<JSONObject, Integer, Object>> columns = this.fields().stream()
|
||||||
|
.collect(ImmutableMap.toImmutableMap(DataSheetClient.ExportField::getHeader, DataSheetClient.ExportField::getReader));
|
||||||
|
this.columnMap(columns);
|
||||||
|
|
||||||
|
ImmutableMap<String, List<String>> multiLineHeadMap = this.fields().stream()
|
||||||
|
.filter(f -> !CollectionUtils.isEmpty(f.getHeaderLines()))
|
||||||
|
.collect(ImmutableMap.toImmutableMap(DataSheetClient.ExportField::getHeader, DataSheetClient.ExportField::getHeaderLines));
|
||||||
|
this.multiLineHeadMap(multiLineHeadMap);
|
||||||
|
|
||||||
|
DataSheetClient.Meta meta = DataSheetClient.Meta.builder()
|
||||||
|
.templateCode(this.scene())
|
||||||
|
.version(Optional.ofNullable(this.version()).orElse("00"))
|
||||||
|
.cellMetas(this.fields().stream().map(DataSheetClient.ExportField::toCellMeta).collect(Collectors.toList()))
|
||||||
|
.build();
|
||||||
|
this.meta(meta);
|
||||||
|
}
|
||||||
|
|
||||||
|
Preconditions.checkArgument(this.scene() != null, "scene不能为空");
|
||||||
|
Preconditions.checkArgument(this.format() != null, "format不能为空");
|
||||||
|
Preconditions.checkArgument(this.rowSupplier() != null, "rowSupplier不能为空");
|
||||||
|
Preconditions.checkArgument(this.columnMap() != null, "columnMap不能为空");
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataSheetClient.ExportResp export(OutputStream outputStream) {
|
||||||
|
Preconditions.checkArgument(outputStream != null, "outputStream不能为空");
|
||||||
|
|
||||||
|
long exportedCount = 0;
|
||||||
|
Exporter exporter = null;
|
||||||
|
Stopwatch stopwatch = Stopwatch.createStarted();
|
||||||
|
String errorMsg = "";
|
||||||
|
|
||||||
|
try {
|
||||||
|
exporter = getDataExporter(outputStream);
|
||||||
|
|
||||||
|
long pageNumber = 1;
|
||||||
|
while (limit() > exportedCount) {
|
||||||
|
// 每页最多允许数量
|
||||||
|
long pageMaxSize = Math.min(limit() - exportedCount, pageSize());
|
||||||
|
if (pageMaxSize <= 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
PageReq pageParam = PageReq.builder()
|
||||||
|
.page((int) pageNumber)
|
||||||
|
// 这里使用pageSize，而不是pageMaxSize，否则会导致最后一次查询的数据和前面的查询数据重复
|
||||||
|
.pageSize(pageSize().intValue())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
if (debugEnabled()) {
|
||||||
|
log.info("-------ExportClient[{}]------, request data, pageParam={}", scene(), pageParam);
|
||||||
|
}
|
||||||
|
PageResp<JSONObject> result = rowSupplier().apply(pageParam);
|
||||||
|
|
||||||
|
// nextPageMaxSize少于查询的pageSize，说明达到了最大导出数，只提取允许的数量
|
||||||
|
if (result.getData().size() > pageMaxSize && pageMaxSize < pageSize()) {
|
||||||
|
result = PageResp.<JSONObject>builder().current(result.getCurrent())
|
||||||
|
.size(result.getSize()).total(result.getTotal())
|
||||||
|
.data(result.getData().subList(0, (int) pageMaxSize))
|
||||||
|
.build();
|
||||||
|
if (debugEnabled()) {
|
||||||
|
log.info("-------ExportClient[{}]------, result data, and split result size to pageMaxSize={}, size={}",
|
||||||
|
scene(), pageMaxSize, result.getData().size());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (debugEnabled()) {
|
||||||
|
log.info("-------ExportClient[{}]------, result data, size={}",
|
||||||
|
scene(), result.getData().size());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pageNumber == 1) {
|
||||||
|
if (!CollectionUtils.isEmpty(multiLineHeadMap())) {
|
||||||
|
List<List<String>> multiLineHeads = columnMap().keySet().stream()
|
||||||
|
.map(headName -> multiLineHeadMap().getOrDefault(headName, ImmutableList.of(headName)))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
exporter.writeMultiLineHeads(multiLineHeads);
|
||||||
|
} else {
|
||||||
|
exporter.writeHead(Lists.newArrayList(columnMap().keySet()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 第一页数据需要写顶部提示内容及表头
|
||||||
|
if (topHintsSupplier() != null) {
|
||||||
|
exporter.writeTopHints(topHintsSupplier().apply(result));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeData(exporter, result);
|
||||||
|
|
||||||
|
exportedCount += result.getData().size();
|
||||||
|
|
||||||
|
if (onProgress() != null) {
|
||||||
|
//XXX： 不知道最大的页数，这里模拟下进度，保证不超过 100%
|
||||||
|
int progress = Math.min((int) pageNumber, 90);
|
||||||
|
onProgress().accept(progress, "");
|
||||||
|
if (debugEnabled()) {
|
||||||
|
log.info("-------ExportClient[{}]------, progress={}", scene(), progress);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.getData().size() < result.getSize()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
pageNumber += 1;
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
errorMsg = e.getMessage();
|
||||||
|
if (!StringUtils.isEmpty(errorMsg) && onProgress() != null) {
|
||||||
|
onProgress().accept(-1, errorMsg);
|
||||||
|
log.error("-------ExportClient ERROR [{}]------, progress={}", scene(), -1);
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
|
} finally {
|
||||||
|
if (exporter != null) {
|
||||||
|
exporter.finish();
|
||||||
|
if (StringUtils.isEmpty(errorMsg) && onProgress() != null) {
|
||||||
|
onProgress().accept(100, "");
|
||||||
|
if (debugEnabled()) {
|
||||||
|
log.info("-------ExportClient[{}]------, progress={}", scene(), 100);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stopwatch.stop();
|
||||||
|
DataSheetClient.ExportResp exportResp = DataSheetClient.ExportResp.builder()
|
||||||
|
.scene(scene())
|
||||||
|
.exportedCount(exportedCount)
|
||||||
|
.rowCount(exporter.getRowCount())
|
||||||
|
.columnCount(exporter.getColumnCount())
|
||||||
|
.dataRowCount(exporter.getDataRowCount())
|
||||||
|
.elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS))
|
||||||
|
.operator(operator())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
if (onCompleted != null) {
|
||||||
|
onCompleted.accept(exportResp);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("-------ExportClient[{}]------, finish export, exportResp={}", scene(), exportResp);
|
||||||
|
return exportResp;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataSheetClient.ExportResp export(HttpServletResponse response) {
|
||||||
|
return export(response, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataSheetClient.ExportResp export(HttpServletResponse response, String filename) {
|
||||||
|
Preconditions.checkArgument(response != null, "response不能为空");
|
||||||
|
|
||||||
|
if (filename == null) {
|
||||||
|
filename = getFilename(format().getSuffix());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
filename = URLEncoder.encode(filename, CharEncoding.UTF_8);
|
||||||
|
} catch (UnsupportedEncodingException e) {
|
||||||
|
throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出失败，请稍后重试");
|
||||||
|
}
|
||||||
|
|
||||||
|
response.setCharacterEncoding(CharEncoding.UTF_8);
|
||||||
|
response.setContentType(format().getContentType());
|
||||||
|
|
||||||
|
try {
|
||||||
|
response.setHeader("Content-Disposition", "attachment;filename=" + filename);
|
||||||
|
|
||||||
|
log.info("-------ExportClient[{}]------, write to servlet response, contentType={}, filename={}",
|
||||||
|
scene(), format().getContentType(), filename);
|
||||||
|
|
||||||
|
return export(response.getOutputStream());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.error("导出数据异常，文件名：" + filename, ex);
|
||||||
|
throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出失败，请稍后重试");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Exporter getDataExporter(OutputStream outputStream) {
|
||||||
|
return format() == DataSheetClient.ExportFormat.CSV ?
|
||||||
|
new CsvExporter(outputStream, scene(), debugEnabled()) :
|
||||||
|
new ExcelExporter(sheetName(), outputStream, scene(), meta(), debugEnabled(), autoTrim());
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getFilename(String suffix) {
|
||||||
|
if (fileName() != null) {
|
||||||
|
return fileName();
|
||||||
|
}
|
||||||
|
String formatDatetime = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").format(LocalDateTime.now());
|
||||||
|
return (Strings.isNullOrEmpty(sheetName()) ? "" : sheetName() + "-" + formatDatetime) + suffix;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeData(Exporter exporter, PageResp<JSONObject> page) {
|
||||||
|
if (CollectionUtils.isEmpty(page.getData())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<JSONObject> rowPage = page.getData();
|
||||||
|
if (rowConverter() != null) {
|
||||||
|
rowPage = page.getData().stream()
|
||||||
|
.map(row -> {
|
||||||
|
List<JSONObject> converted = rowConverter().apply(row);
|
||||||
|
|
||||||
|
if (debugEnabled()) {
|
||||||
|
log.info("-------ExportClient[{}]------, row converted, row={}, converted={}",
|
||||||
|
scene(), row, converted);
|
||||||
|
}
|
||||||
|
|
||||||
|
return converted;
|
||||||
|
})
|
||||||
|
.flatMap(Collection::stream)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
if (pageConverter() != null) {
|
||||||
|
final List<JSONObject> originRowPage = rowPage;
|
||||||
|
rowPage = pageConverter().apply(rowPage);
|
||||||
|
if (debugEnabled()) {
|
||||||
|
log.info("-------ExportClient[{}]------, page converted, old={}, converted={}",
|
||||||
|
scene(), originRowPage, rowPage);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int rowIndex = (int) exporter.getDataRowCount();
|
||||||
|
List<JSONObject> finalRowPage = rowPage;
|
||||||
|
List<List<Object>> rows = IntStream.range(0, finalRowPage.size())
|
||||||
|
.mapToObj(index -> columnMap().values().stream()
|
||||||
|
.map(v -> v.apply(finalRowPage.get(index), rowIndex + index))
|
||||||
|
.collect(Collectors.toList()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
rows.forEach(exporter::writeRow);
|
||||||
|
}
|
||||||
|
|
||||||
|
private class CsvExporter extends RowColumnCounter implements Exporter {
|
||||||
|
|
||||||
|
private CSVWriter writer;
|
||||||
|
private String scene;
|
||||||
|
private boolean debugEnabled;
|
||||||
|
|
||||||
|
private CsvExporter(OutputStream outputStream, String scene, boolean debugEnabled) {
|
||||||
|
try {
|
||||||
|
// encoding
|
||||||
|
outputStream.write(0xef);
|
||||||
|
outputStream.write(0xbb);
|
||||||
|
outputStream.write(0xbf);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出数据失败");
|
||||||
|
}
|
||||||
|
|
||||||
|
writer = new CSVWriter(new OutputStreamWriter(outputStream));
|
||||||
|
this.scene = scene;
|
||||||
|
this.debugEnabled = debugEnabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void writeTopHints(List<String> topHints) {
|
||||||
|
if (CollectionUtils.isEmpty(topHints)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
topHints.forEach(t -> writer.writeNext(new String[]{t}));
|
||||||
|
countTopHints(topHints.size());
|
||||||
|
|
||||||
|
if (debugEnabled) {
|
||||||
|
log.info("-------ExportClient[{}]------, write top={}", scene, topHints);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void writeHead(List<String> head) {
|
||||||
|
writer.writeNext(head.toArray(new String[0]));
|
||||||
|
countHead(head.size());
|
||||||
|
|
||||||
|
if (debugEnabled) {
|
||||||
|
log.info("-------ExportClient[{}]------, write head={}", scene, head);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void writeMultiLineHeads(List<List<String>> multiLineHeads) {
|
||||||
|
int lineNum = multiLineHeads.stream().mapToInt(List::size).max().orElse(0);
|
||||||
|
IntStream.range(0, lineNum).forEach(i -> {
|
||||||
|
List<String> heads = multiLineHeads.stream().map(head -> {
|
||||||
|
// 当表头的行数小于最大行数时，最上面的行中表头内容写入空字符串
|
||||||
|
int line = i - lineNum + head.size();
|
||||||
|
return line < 0 ? StringUtils.EMPTY : head.get(line);
|
||||||
|
}).collect(Collectors.toList());
|
||||||
|
writeHead(heads);
|
||||||
|
});
|
||||||
|
if (debugEnabled) {
|
||||||
|
log.info("-------ExportClient[{}]------, write multi line heads={}", scene, multiLineHeads);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void writeRow(List<Object> row) {
|
||||||
|
String[] arrayContents = row.stream()
|
||||||
|
.map(c -> c == null ? StringUtils.EMPTY : c.toString())
|
||||||
|
.toArray(String[]::new);
|
||||||
|
writer.writeNext(arrayContents);
|
||||||
|
countRow(row.size());
|
||||||
|
|
||||||
|
if (debugEnabled) {
|
||||||
|
log.info("-------ExportClient[{}]------, write row={}", scene, row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void finish() {
|
||||||
|
try {
|
||||||
|
writer.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出数据失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class ExcelExporter extends RowColumnCounter implements Exporter {
|
||||||
|
|
||||||
|
private final Integer DEFAULT_WIDTH = 6;
|
||||||
|
|
||||||
|
private ExcelWriter writer;
|
||||||
|
private WriteSheet sheet;
|
||||||
|
private String scene;
|
||||||
|
private boolean debugEnabled;
|
||||||
|
private HeaderCommentWriteHandler commentWriteHandler;
|
||||||
|
private DataValidationWriteHandler validationWriteHandler;
|
||||||
|
private int topHintsRowCount = 0;
|
||||||
|
|
||||||
|
private ExcelExporter(String sheetName, OutputStream outputStream, String scene, DataSheetClient.Meta meta,
|
||||||
|
boolean debugEnabled, Boolean autoTrim) {
|
||||||
|
|
||||||
|
ExcelWriterBuilder excelWriterBuilder = EasyExcel.write(outputStream);
|
||||||
|
if (Objects.nonNull(meta) && !CollectionUtils.isEmpty(meta.getCellMetas())) {
|
||||||
|
Integer width = meta.getCellMetas().get(0).getWidth();
|
||||||
|
excelWriterBuilder.registerWriteHandler(new SimpleColumnWidthStyleStrategy(Objects.nonNull(width) ?
|
||||||
|
width : DEFAULT_WIDTH));
|
||||||
|
} else {
|
||||||
|
excelWriterBuilder.registerWriteHandler(new LongestMatchColumnWidthStyleStrategy());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (meta != null) {
|
||||||
|
excelWriterBuilder.registerWriteHandler(new AbstractCellStyleStrategy() {
|
||||||
|
private CellStyle wrapStyle;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void setHeadCellStyle(Cell cell, Head head, Integer relativeRowIndex) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void setContentCellStyle(Cell cell, Head head, Integer relativeRowIndex) {
|
||||||
|
if (cell.getColumnIndex() >= meta.getCellMetas().size()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
DataSheetClient.CellMeta cellMeta = meta.getCellMetas().get(cell.getColumnIndex());
|
||||||
|
if (cellMeta != null && BooleanUtils.isTrue(cellMeta.getWrapText())) {
|
||||||
|
cell.setCellStyle(wrapStyle);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
commentWriteHandler = new HeaderCommentWriteHandler(meta);
|
||||||
|
validationWriteHandler = new DataValidationWriteHandler(meta);
|
||||||
|
|
||||||
|
excelWriterBuilder.registerWriteHandler(new AbstractCellWriteHandler() {
|
||||||
|
@Override
|
||||||
|
public void afterCellDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, List<WriteCellData<?>> cellDataList, Cell cell, Head head, Integer relativeRowIndex, Boolean isHead) {
|
||||||
|
boolean isHeadRow = cell.getRowIndex() == topHintsRowCount;
|
||||||
|
commentWriteHandler.afterCellDispose(cell, isHeadRow);
|
||||||
|
validationWriteHandler.afterCellDispose(cell, isHeadRow);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
writer = excelWriterBuilder.autoTrim(autoTrim).build();
|
||||||
|
sheet = EasyExcel.writerSheet(sheetName).build();
|
||||||
|
this.scene = scene;
|
||||||
|
this.debugEnabled = debugEnabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void writeTopHints(List<String> topHints) {
|
||||||
|
if (CollectionUtils.isEmpty(topHints)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
countTopHints(topHints.size());
|
||||||
|
topHintsRowCount += topHints.size();
|
||||||
|
writer.write(topHints.stream().map(ImmutableList::of).collect(Collectors.toList()), sheet);
|
||||||
|
|
||||||
|
if (debugEnabled) {
|
||||||
|
log.info("-------ExportClient[{}]------, write top={}", scene, topHints);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void writeHead(List<String> head) {
|
||||||
|
sheet.setHead(head.stream().map(Lists::newArrayList).collect(Collectors.toList()));
|
||||||
|
countHead(head.size());
|
||||||
|
|
||||||
|
// XXX 写入head后需要调用一次write()确保excel中有数据，否则数据为空时将出现内容异常
|
||||||
|
writer.write(ImmutableList.of(), sheet);
|
||||||
|
|
||||||
|
if (debugEnabled) {
|
||||||
|
log.info("-------ExportClient[{}]------, write head={}", scene, head);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void writeMultiLineHeads(List<List<String>> multiLineHeads) {
|
||||||
|
// XXX setHead()方法有副作用，内部逻辑会改变传入的list, 因此需要确保传入的list可操作
|
||||||
|
sheet.setHead(multiLineHeads.stream().map(Lists::newArrayList).collect(Collectors.toList()));
|
||||||
|
int lineNum = multiLineHeads.stream().mapToInt(List::size).max().orElse(0);
|
||||||
|
IntStream.range(0, lineNum).forEach(i -> countHead(multiLineHeads.size()));
|
||||||
|
|
||||||
|
// XXX 写入head后需要调用一次write()确保excel中有数据，否则数据为空时将出现内容异常
|
||||||
|
writer.write(ImmutableList.of(), sheet);
|
||||||
|
|
||||||
|
if (debugEnabled) {
|
||||||
|
log.info("-------ExportClient[{}]------, write multi line heads={}", scene, multiLineHeads);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void writeRow(List<Object> row) {
|
||||||
|
writer.write(ImmutableList.of(row), sheet);
|
||||||
|
countRow(row.size());
|
||||||
|
|
||||||
|
if (debugEnabled) {
|
||||||
|
log.info("-------ExportClient[{}]------, write row={}", scene, row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void finish() {
|
||||||
|
writer.finish();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Excel添加头部的批注
|
||||||
|
* XXX: 此处不能继承AbstractCellWriteHandler 会导致 mybatis typehandler 扫描时被加载
|
||||||
|
* 导致老项目没有使用 easyexcel2.2 的项目启动失败。
|
||||||
|
*/
|
||||||
|
private static class HeaderCommentWriteHandler {
|
||||||
|
private static final Map<List<String>, String> rangeBoundTypes = ImmutableMap.of(
|
||||||
|
ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN), RANGE_TYPE_OPEN_OPEN,
|
||||||
|
ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED), RANGE_TYPE_OPEN_CLOSED,
|
||||||
|
ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN), RANGE_TYPE_CLOSED_OPEN,
|
||||||
|
ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED), RANGE_TYPE_CLOSED_CLOSED);
|
||||||
|
|
||||||
|
private String templateCode;
|
||||||
|
private String version;
|
||||||
|
private Map<String, DataSheetClient.CellMeta> cellMetaMap;
|
||||||
|
|
||||||
|
private Set<DataSheetClient.CellMeta.Type> typeWithParams = ImmutableSet.of(
|
||||||
|
DataSheetClient.CellMeta.Type.RANGE,
|
||||||
|
DataSheetClient.CellMeta.Type.DATE,
|
||||||
|
DataSheetClient.CellMeta.Type.DATETIME
|
||||||
|
);
|
||||||
|
|
||||||
|
private HeaderCommentWriteHandler(DataSheetClient.Meta meta) {
|
||||||
|
this.templateCode = meta.getTemplateCode();
|
||||||
|
this.version = meta.getVersion();
|
||||||
|
|
||||||
|
this.cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), DataSheetClient.CellMeta::getName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void afterCellDispose(Cell cell, boolean isHead) {
|
||||||
|
// 只添加表头行的批注
|
||||||
|
if (!isHead) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> comments = Lists.newArrayList();
|
||||||
|
if (cell.getColumnIndex() == 0) {
|
||||||
|
String content = Joiner.on("_").join(ImmutableList.of(templateCode, version));
|
||||||
|
comments.add(buildComment(PATTERN_VERSION_OPEN, content));
|
||||||
|
}
|
||||||
|
|
||||||
|
DataSheetClient.CellMeta cellMeta = cellMetaMap.get(cell.getStringCellValue());
|
||||||
|
if (cellMeta == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// 格式： key_type_mandatory
|
||||||
|
String content = Joiner.on("_").join(ImmutableList.of(
|
||||||
|
cellMeta.getKey(),
|
||||||
|
cellMeta.getType().getCode(),
|
||||||
|
Boolean.TRUE.equals(cellMeta.getMandatory()) ? "1" : "0"));
|
||||||
|
|
||||||
|
if (typeWithParams.contains(cellMeta.getType())) {
|
||||||
|
content = content.concat("_").concat(getParamsByType(cellMeta));
|
||||||
|
}
|
||||||
|
comments.add(buildComment(PATTERN_CELL_OPEN, content));
|
||||||
|
|
||||||
|
// 一个Cell只能写入一次Comment
|
||||||
|
writeComment(cell, Joiner.on("\n").join(comments));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getParamsByType(DataSheetClient.CellMeta cellMeta) {
|
||||||
|
switch (cellMeta.getType()) {
|
||||||
|
case RANGE:
|
||||||
|
String lowerType = cellMeta.getParams().getString(EXT_KEY_RANGE_LOWER_TYPE);
|
||||||
|
String upperType = cellMeta.getParams().getString(EXT_KEY_RANGE_UPPER_TYPE);
|
||||||
|
return rangeBoundTypes.get(ImmutableList.of(lowerType, upperType));
|
||||||
|
case DATE:
|
||||||
|
case DATETIME:
|
||||||
|
return cellMeta.getParams().getString(EXT_KEY_DATETIME_FORMAT);
|
||||||
|
default:
|
||||||
|
return StringUtils.EMPTY;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeComment(Cell cell, String comment) {
|
||||||
|
int columnIndex = cell.getColumnIndex();
|
||||||
|
int rowIndex = cell.getRowIndex();
|
||||||
|
Drawing<?> drawing = cell.getSheet().createDrawingPatriarch();
|
||||||
|
Comment cellComment = drawing.createCellComment(new XSSFClientAnchor(0, 0, 0, 0,
|
||||||
|
// 控制批注的位置
|
||||||
|
columnIndex, rowIndex, columnIndex + 2, rowIndex + 2));
|
||||||
|
cellComment.setString(new XSSFRichTextString(comment));
|
||||||
|
cell.setCellComment(cellComment);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildComment(String patternOpen, String content) {
|
||||||
|
return patternOpen.concat(content).concat(PATTERN_CLOSE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Excel添加校验数据
|
||||||
|
* XXX: 此处不能继承AbstractCellWriteHandler 会导致 mybatis typehandler 扫描时被加载
|
||||||
|
* 导致老项目没有使用 easyexcel2.2 的项目启动失败。
|
||||||
|
*/
|
||||||
|
private static class DataValidationWriteHandler {
|
||||||
|
private Map<String, DataSheetClient.CellMeta> cellMetaMap;
|
||||||
|
|
||||||
|
private DataValidationWriteHandler(DataSheetClient.Meta meta) {
|
||||||
|
this.cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), DataSheetClient.CellMeta::getName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void afterCellDispose(Cell cell, boolean isHead) {
|
||||||
|
// 只在写入表头数据时才写入校验数据
|
||||||
|
if (!isHead) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
DataSheetClient.CellMeta cellMeta = cellMetaMap.get(cell.getStringCellValue());
|
||||||
|
// 目前仅支持将可选值作为下拉选项的校验数据
|
||||||
|
if (cellMeta == null || CollectionUtils.isEmpty(cellMeta.getOptions())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String[] options = cellMeta.getOptions().stream().map(String::valueOf).toArray(String[]::new);
|
||||||
|
|
||||||
|
Sheet currentSheet = cell.getSheet();
|
||||||
|
Workbook workbook = currentSheet.getWorkbook();
|
||||||
|
// 直接将校验数据作为列表项的方式有数据大小限制(不超过256个字符), 因此需要将校验数据放在一个新增的隐藏表单中
|
||||||
|
// 以表头的名字来命名新增的表单
|
||||||
|
String sheetName = cell.getStringCellValue();
|
||||||
|
Sheet validationSheet = workbook.createSheet(sheetName);
|
||||||
|
workbook.setSheetHidden(workbook.getSheetIndex(validationSheet), true);
|
||||||
|
|
||||||
|
// 将可选项放在新增的隐藏表单里
|
||||||
|
for (int i = 0; i < options.length; i++) {
|
||||||
|
Row rowInValidationSheet = validationSheet.createRow(i);
|
||||||
|
Cell cellInValidationSheet = rowInValidationSheet.createCell(0);
|
||||||
|
cellInValidationSheet.setCellValue(options[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 设置列表校验数据的公式 [表名!开始列开始行:结束列结束行]
|
||||||
|
String listFormula = sheetName + "!$A$1:$A$" + options.length;
|
||||||
|
DataValidationHelper dataValidationHelper = currentSheet.getDataValidationHelper();
|
||||||
|
DataValidationConstraint constraint = dataValidationHelper.createFormulaListConstraint(listFormula);
|
||||||
|
|
||||||
|
// 从下一行开始10000行数据都增加校验数据
|
||||||
|
CellRangeAddressList addressList = new CellRangeAddressList(cell.getRowIndex() + 1,
|
||||||
|
cell.getRowIndex() + 10001, cell.getColumnIndex(), cell.getColumnIndex());
|
||||||
|
DataValidation dataValidation = dataValidationHelper.createValidation(constraint, addressList);
|
||||||
|
dataValidation.setSuppressDropDownArrow(true);
|
||||||
|
dataValidation.setShowErrorBox(true);
|
||||||
|
|
||||||
|
currentSheet.addValidationData(dataValidation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class RowColumnCounter {
|
||||||
|
@Getter
|
||||||
|
private long rowCount = 0;
|
||||||
|
@Getter
|
||||||
|
private long columnCount = 0;
|
||||||
|
@Getter
|
||||||
|
private long dataRowCount = 0;
|
||||||
|
|
||||||
|
protected void countTopHints(int topHintsRowCount) {
|
||||||
|
rowCount += topHintsRowCount;
|
||||||
|
columnCount = Math.max(columnCount, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void countHead(int headColumnCount) {
|
||||||
|
rowCount += 1;
|
||||||
|
columnCount = Math.max(columnCount, headColumnCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void countRow(int columnCount) {
|
||||||
|
rowCount += 1;
|
||||||
|
this.columnCount = Math.max(this.columnCount, columnCount);
|
||||||
|
dataRowCount += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private interface Exporter {
|
||||||
|
void writeTopHints(List<String> topHints);
|
||||||
|
|
||||||
|
void writeHead(List<String> head);
|
||||||
|
|
||||||
|
void writeMultiLineHeads(List<List<String>> multiLineHeads);
|
||||||
|
|
||||||
|
void writeRow(List<Object> row);
|
||||||
|
|
||||||
|
void finish();
|
||||||
|
|
||||||
|
long getRowCount();
|
||||||
|
|
||||||
|
long getColumnCount();
|
||||||
|
|
||||||
|
long getDataRowCount();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,595 @@
|
|||||||
|
package cn.axzo.foundation.excel.support.impl;
|
||||||
|
|
||||||
|
import cn.axzo.foundation.excel.support.DataSheetClient;
|
||||||
|
import cn.axzo.foundation.exception.BusinessException;
|
||||||
|
import cn.axzo.foundation.util.Regex;
|
||||||
|
import com.alibaba.excel.EasyExcel;
|
||||||
|
import com.alibaba.excel.context.AnalysisContext;
|
||||||
|
import com.alibaba.excel.enums.CellExtraTypeEnum;
|
||||||
|
import com.alibaba.excel.event.AnalysisEventListener;
|
||||||
|
import com.alibaba.excel.metadata.CellExtra;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
import com.google.common.base.Stopwatch;
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import com.google.common.collect.*;
|
||||||
|
import lombok.*;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.BooleanUtils;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
import java.util.function.Function;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import java.util.stream.IntStream;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
|
import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*;
|
||||||
|
import static cn.axzo.foundation.excel.support.DataSheetClient.Meta.*;
|
||||||
|
import static cn.axzo.foundation.excel.support.DataSheetClient.ResultCode.IMPORT_CELL_CONVERT_FAILED;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用ali的EasyExcel来读入excel/csv文件
|
||||||
|
*
|
||||||
|
* @author yuanyi
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class DataSheetImporter extends DataSheetClient.ImporterBuilder {
|
||||||
|
|
||||||
|
private static final String LINE_KEY_ERRORS = "errors";
|
||||||
|
private static final String LINE_KEY_ROW_INDEX = "rowIndex";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 完成后处理事件.
|
||||||
|
*/
|
||||||
|
@Setter
|
||||||
|
private Consumer<DataSheetClient.ImportResp> onCompleted;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataSheetClient.Importer build() {
|
||||||
|
Preconditions.checkArgument(this.scene() != null, "scene不能为空");
|
||||||
|
Preconditions.checkArgument(this.format() != null, "format不能为空");
|
||||||
|
|
||||||
|
// TODO: 当支持更多format的时候，需要生成对应的Importer实例
|
||||||
|
return ExcelImporter.builder()
|
||||||
|
.scene(scene())
|
||||||
|
.tableName(tableName())
|
||||||
|
.meta(meta())
|
||||||
|
.ignoreHeaderMeta(ignoreHeaderMeta())
|
||||||
|
.debugEnabled(debugEnabled())
|
||||||
|
.allowMaxLineCount(allowMaxLineCount())
|
||||||
|
.headerConverter(headerConverter())
|
||||||
|
.includeLineErrors(includeLineErrors())
|
||||||
|
.ignoreUnknownColumn(ignoreUnknownColumn())
|
||||||
|
.autoTrim(autoTrim())
|
||||||
|
.onCompleted(onCompleted)
|
||||||
|
.operator(operator())
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Builder
|
||||||
|
private static class ExcelImporter implements DataSheetClient.Importer {
|
||||||
|
|
||||||
|
private static final Map<String, List<String>> rangeBoundTypes = ImmutableMap.of(
|
||||||
|
RANGE_TYPE_OPEN_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN),
|
||||||
|
RANGE_TYPE_OPEN_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED),
|
||||||
|
RANGE_TYPE_CLOSED_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN),
|
||||||
|
RANGE_TYPE_CLOSED_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED));
|
||||||
|
|
||||||
|
private String scene;
|
||||||
|
private String tableName;
|
||||||
|
private DataSheetClient.Meta meta;
|
||||||
|
private Boolean ignoreHeaderMeta;
|
||||||
|
private boolean debugEnabled;
|
||||||
|
private Integer allowMaxLineCount;
|
||||||
|
private boolean includeLineErrors;
|
||||||
|
private boolean ignoreUnknownColumn;
|
||||||
|
private Boolean autoTrim;
|
||||||
|
private Function<String, String> headerConverter;
|
||||||
|
private Consumer<DataSheetClient.ImportResp> onCompleted;
|
||||||
|
private String operator;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataSheetClient.ImportResp<JSONObject> readAll(InputStream inputStream) {
|
||||||
|
Stopwatch stopwatch = Stopwatch.createStarted();
|
||||||
|
|
||||||
|
NoModelDataListener dataListener = new NoModelDataListener(headerConverter, autoTrim);
|
||||||
|
EasyExcel.read(inputStream, dataListener)
|
||||||
|
.autoTrim(autoTrim)
|
||||||
|
.extraRead(CellExtraTypeEnum.COMMENT).sheet(tableName).doRead();
|
||||||
|
if (allowMaxLineCount != null && dataListener.lines.size() > allowMaxLineCount) {
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_LINES_REACHED_LIMIT
|
||||||
|
.toException("导入的数据超过了最大行数" + allowMaxLineCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (BooleanUtils.isNotTrue(this.ignoreHeaderMeta)) {
|
||||||
|
// 聚合来自入参和文件批注的meta信息；如果都存在，使用入参的meta覆盖文件批注的meta
|
||||||
|
this.meta = mergeMeta(this.meta, parseMetaFromData(dataListener));
|
||||||
|
}
|
||||||
|
filterHeadMap(dataListener);
|
||||||
|
filterLines(dataListener);
|
||||||
|
|
||||||
|
validateHeaders(dataListener);
|
||||||
|
validateMeta(dataListener);
|
||||||
|
|
||||||
|
List<String> headers = parseHeaders(dataListener);
|
||||||
|
List<JSONObject> lines = parseLines(dataListener);
|
||||||
|
if (!includeLineErrors) {
|
||||||
|
// 没有设置includeLineErrors时，抛第一个发现的异常
|
||||||
|
Optional<JSONObject> errorLine = lines.stream()
|
||||||
|
.filter(line -> line.containsKey(LINE_KEY_ERRORS))
|
||||||
|
.findFirst();
|
||||||
|
if (errorLine.isPresent()) {
|
||||||
|
JSONObject error = errorLine.get().getJSONArray(LINE_KEY_ERRORS).getJSONObject(0);
|
||||||
|
Integer rowIndex = lines.indexOf(errorLine.get());
|
||||||
|
Integer columnIndex = error.getInteger("columnIndex");
|
||||||
|
|
||||||
|
String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1,
|
||||||
|
error.getString("columnName"), error.getString("errorMsg"));
|
||||||
|
throw new DataSheetClient.DataSheetException(error.getString("subErrorCode"), errMsg, rowIndex, columnIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DataSheetClient.ImportResp resp = DataSheetClient.ImportResp.<JSONObject>builder()
|
||||||
|
.scene(scene)
|
||||||
|
.templateCode(Optional.ofNullable(meta).map(DataSheetClient.Meta::getTemplateCode).orElse(null))
|
||||||
|
.version(Optional.ofNullable(meta).map(DataSheetClient.Meta::getVersion).orElse(null))
|
||||||
|
.headers(headers)
|
||||||
|
.lines(lines)
|
||||||
|
.headerRowCount(1)
|
||||||
|
.rowCount(lines.size())
|
||||||
|
.columnCount(headers.size())
|
||||||
|
.meta(meta)
|
||||||
|
.elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS))
|
||||||
|
.operator(operator)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
if (null != onCompleted) {
|
||||||
|
onCompleted.accept(resp);
|
||||||
|
}
|
||||||
|
return resp;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void filterHeadMap(NoModelDataListener dataListener) {
|
||||||
|
if (meta == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<Integer, String> headMap = dataListener.getHeadMap();
|
||||||
|
|
||||||
|
Set<String> ignoreColumnNames = meta.getIgnoreColumnNames();
|
||||||
|
// 收集需要过滤的列号
|
||||||
|
Set<Integer> ignoreColumnIndexes = headMap.entrySet().stream()
|
||||||
|
.filter(entry -> ignoreColumnNames.contains(entry.getValue()))
|
||||||
|
.map(Map.Entry::getKey).collect(Collectors.toSet());
|
||||||
|
// 加上指定过滤的列号
|
||||||
|
ignoreColumnIndexes.addAll(meta.getIgnoreColumnIndexes());
|
||||||
|
|
||||||
|
ignoreColumnIndexes.forEach(headMap::remove);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void filterLines(NoModelDataListener dataListener) {
|
||||||
|
if (meta == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Set<Integer> toRemoveLines = ImmutableSet.copyOf(meta.getIgnoreRowIndexes());
|
||||||
|
List<Map<Integer, String>> lines = dataListener.getLines();
|
||||||
|
dataListener.setLines(IntStream.range(0, lines.size())
|
||||||
|
// 这里要加1，是因为removeIndex是整个文档的行数来计算，包括了header
|
||||||
|
// 但是lines的数据，已经排除了header
|
||||||
|
.filter(index -> !toRemoveLines.contains(index + 1))
|
||||||
|
.mapToObj(lines::get)
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private DataSheetClient.Meta parseMetaFromData(NoModelDataListener dataListener) {
|
||||||
|
Map<Integer, String> headMap = dataListener.getHeadMap();
|
||||||
|
List<CellExtra> headComments = dataListener.getCellComments().stream()
|
||||||
|
// 获取第一行Head的批注信息
|
||||||
|
.filter(cellExtra -> cellExtra.getRowIndex() == 0)
|
||||||
|
// 过滤带有Cell参数配置的批注信息
|
||||||
|
.filter(cellExtra -> !Strings.isNullOrEmpty(StringUtils.substringBetween(cellExtra.getText(),
|
||||||
|
PATTERN_CELL_OPEN, PATTERN_CLOSE)))
|
||||||
|
// 排序，方便找到第一列，获取templateCode与version
|
||||||
|
.sorted(Comparator.comparing(CellExtra::getColumnIndex))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
// 没有批注信息，直接返回null；会以String来解析值
|
||||||
|
if (headComments.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 从文件中解析meta信息
|
||||||
|
JSONObject codeAndVersion = parseTemplateCodeAndVersion(headComments);
|
||||||
|
List<DataSheetClient.CellMeta> cellMetas = headComments.stream()
|
||||||
|
.map(cellExtra -> parseCellMeta(cellExtra.getText(), headMap.get(cellExtra.getColumnIndex())))
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
if (cellMetas.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return DataSheetClient.Meta.builder()
|
||||||
|
.templateCode(codeAndVersion.getString("templateCode"))
|
||||||
|
.version(codeAndVersion.getString("version"))
|
||||||
|
.cellMetas(cellMetas)
|
||||||
|
.ignoreColumnIndexes(parseIgnoreColumns(dataListener))
|
||||||
|
.ignoreRowIndexes(parseIgnoreRows(dataListener))
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DataSheetClient.Meta mergeMeta(DataSheetClient.Meta metaFromParam, DataSheetClient.Meta metaFromHeader) {
|
||||||
|
if (metaFromHeader == null || metaFromParam == null) {
|
||||||
|
return metaFromHeader == null ? metaFromParam : metaFromHeader;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 在两者都存在的时候，以文件的meta为准；仅仅将入参的CellMetas覆盖文件的CellMetas
|
||||||
|
if (metaFromHeader.getCellMetas() == null) {
|
||||||
|
metaFromHeader.setCellMetas(metaFromParam.getCellMetas());
|
||||||
|
return metaFromHeader;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 使用入参中定义的列信息，覆盖文件的cellMetas
|
||||||
|
Map<String, DataSheetClient.CellMeta> cellMetaMap =
|
||||||
|
Maps.uniqueIndex(metaFromParam.getCellMetas(), DataSheetClient.CellMeta::getKey);
|
||||||
|
List<DataSheetClient.CellMeta> cellMetas = metaFromHeader.getCellMetas().stream()
|
||||||
|
.map(cellMeta -> cellMetaMap.getOrDefault(cellMeta.getKey(), cellMeta))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
metaFromHeader.setCellMetas(cellMetas);
|
||||||
|
return metaFromHeader;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Integer> parseIgnoreRows(NoModelDataListener dataListener) {
|
||||||
|
return dataListener.getCellComments().stream()
|
||||||
|
.map(cellExtra -> {
|
||||||
|
String ignoreKey = StringUtils.substringBetween(cellExtra.getText(),
|
||||||
|
PATTERN_IGNORE_OPEN, PATTERN_CLOSE);
|
||||||
|
if (IGNORE_ROW_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) {
|
||||||
|
return cellExtra.getRowIndex();
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
})
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Integer> parseIgnoreColumns(NoModelDataListener dataListener) {
|
||||||
|
return dataListener.getCellComments().stream()
|
||||||
|
.map(cellExtra -> {
|
||||||
|
String ignoreKey = StringUtils.substringBetween(cellExtra.getText(),
|
||||||
|
PATTERN_IGNORE_OPEN, PATTERN_CLOSE);
|
||||||
|
if (IGNORE_COLUMN_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) {
|
||||||
|
return cellExtra.getColumnIndex();
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
})
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
public JSONObject parseTemplateCodeAndVersion(List<CellExtra> headComments) {
|
||||||
|
// 从整个头部的批注，获取模版code和版本信息
|
||||||
|
Optional<String> parsedText = headComments.stream()
|
||||||
|
.map(comment -> StringUtils.substringBetween(comment.getText(), PATTERN_VERSION_OPEN, PATTERN_CLOSE))
|
||||||
|
.filter(str -> !Strings.isNullOrEmpty(str))
|
||||||
|
.findFirst();
|
||||||
|
if (!parsedText.isPresent()) {
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_PARSE_MISSING_VERSION.toException();
|
||||||
|
}
|
||||||
|
// 格式为"CODE_VERSION"
|
||||||
|
List<String> values = Splitter.on("_").splitToList(parsedText.get());
|
||||||
|
if (values.size() != 2) {
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_PARSE_VERSION_FORMAT_ERROR.toException();
|
||||||
|
}
|
||||||
|
return new JSONObject()
|
||||||
|
.fluentPut("templateCode", values.get(0))
|
||||||
|
.fluentPut("version", values.get(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
private DataSheetClient.CellMeta parseCellMeta(String text, String name) {
|
||||||
|
String value = StringUtils.substringBetween(text, PATTERN_CELL_OPEN, PATTERN_CLOSE);
|
||||||
|
if (Strings.isNullOrEmpty(value)) {
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_MISSING_TYPE
|
||||||
|
.toException("没有找到列[{}]的类型信息", name);
|
||||||
|
}
|
||||||
|
List<String> values = Splitter.on("_").splitToList(value);
|
||||||
|
// 格式： key_type_mandatory_params...
|
||||||
|
if (values.size() < 3) {
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_FORMAT_ERROR
|
||||||
|
.toException("列[{}]类型的批注格式不对[{}]", name, text);
|
||||||
|
}
|
||||||
|
|
||||||
|
DataSheetClient.CellMeta.Type type = DataSheetClient.CellMeta.Type.from(values.get(1));
|
||||||
|
JSONObject params = new JSONObject();
|
||||||
|
if (type == DataSheetClient.CellMeta.Type.RANGE) {
|
||||||
|
if (values.size() != 4 || Strings.isNullOrEmpty(values.get(3))) {
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR
|
||||||
|
.toException("列[{}]范围类型的批注格式不对[{}]", name, text);
|
||||||
|
}
|
||||||
|
List<String> boundType = rangeBoundTypes.get(values.get(3));
|
||||||
|
if (boundType == null) {
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR
|
||||||
|
.toException(String.format("列[{}]范围类型的值不对", name));
|
||||||
|
}
|
||||||
|
params.put(EXT_KEY_RANGE_LOWER_TYPE, boundType.get(0));
|
||||||
|
params.put(EXT_KEY_RANGE_UPPER_TYPE, boundType.get(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
JSONObject ext = new JSONObject();
|
||||||
|
value = StringUtils.substringBetween(text, PATTERN_EXTRA_OPEN, PATTERN_CLOSE);
|
||||||
|
if (!Strings.isNullOrEmpty(value)) {
|
||||||
|
ext.putAll(Splitter.on("&").withKeyValueSeparator("=").split(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
return DataSheetClient.CellMeta.builder()
|
||||||
|
.key(values.get(0))
|
||||||
|
.name(name)
|
||||||
|
.type(type)
|
||||||
|
.mandatory("1".equals(values.get(2)))
|
||||||
|
.params(params)
|
||||||
|
.ext(ext)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<String> parseHeaders(NoModelDataListener dataListener) {
|
||||||
|
return dataListener.getHeadMap().keySet().stream()
|
||||||
|
.sorted()
|
||||||
|
.map(key -> Strings.nullToEmpty(dataListener.getHeadMap().get(key)))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<JSONObject> parseLines(NoModelDataListener dataListener) {
|
||||||
|
// 如果没有找到meta信息，按照key=header(很可能是中文), 类型就为String
|
||||||
|
if (this.meta == null) {
|
||||||
|
Map<Integer, String> headerMap = dataListener.getHeadMap();
|
||||||
|
return dataListener.getLines().stream()
|
||||||
|
.map(line -> new JSONObject().fluentPutAll(line.entrySet().stream()
|
||||||
|
.map(entry -> Pair.of(headerMap.get(entry.getKey()), Strings.nullToEmpty(entry.getValue())))
|
||||||
|
.collect(Collectors.toMap(Pair::getKey, Pair::getValue))))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
// 根据meta来校验cell的类型
|
||||||
|
Map<Integer, String> headerMap = dataListener.getHeadMap();
|
||||||
|
Map<String, DataSheetClient.CellMeta> cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(),
|
||||||
|
DataSheetClient.CellMeta::getName);
|
||||||
|
List<Map<Integer, String>> lines = dataListener.getLines();
|
||||||
|
return IntStream.range(0, lines.size())
|
||||||
|
.mapToObj(lineIndex -> {
|
||||||
|
Map<Integer, String> line = lines.get(lineIndex);
|
||||||
|
// 收集每一行每一列的转换结果
|
||||||
|
List<ColumnConvertResp> convertRespList = headerMap.entrySet().stream()
|
||||||
|
.filter(e -> {
|
||||||
|
//XXX 当支持忽略列的时候, 只处理定义了cellMeta的数据
|
||||||
|
if (ignoreUnknownColumn) {
|
||||||
|
return cellMetaMap.containsKey(e.getValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
})
|
||||||
|
.map(entry -> {
|
||||||
|
Integer columnIndex = entry.getKey();
|
||||||
|
String header = entry.getValue();
|
||||||
|
DataSheetClient.CellMeta cellMeta = cellMetaMap.get(header);
|
||||||
|
return convertType(cellMeta, line.get(columnIndex), lineIndex, columnIndex);
|
||||||
|
})
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
JSONObject row = new JSONObject()
|
||||||
|
.fluentPutAll(convertRespList.stream()
|
||||||
|
.filter(ColumnConvertResp::getSuccess)
|
||||||
|
// convertValue可能为null, 有非必需的字段
|
||||||
|
.collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll));
|
||||||
|
convertRespList.stream()
|
||||||
|
.filter(r -> BooleanUtils.isTrue(r.getSuccess()))
|
||||||
|
.filter(r -> r.getCellMeta().getCellValidator() != null)
|
||||||
|
.forEach(r -> {
|
||||||
|
try {
|
||||||
|
r.getCellMeta().getCellValidator().accept(r.getConvertedValue(), row);
|
||||||
|
} catch (BusinessException e) {
|
||||||
|
log.warn("failed to validate cell, resp={}", r, e);
|
||||||
|
String subErrorCode = null;
|
||||||
|
if (e instanceof DataSheetClient.DataSheetException) {
|
||||||
|
subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode();
|
||||||
|
}
|
||||||
|
r.setSuccess(false);
|
||||||
|
r.setErrorCode(e.getErrorCode());
|
||||||
|
r.setErrorMsg(e.getErrorMsg());
|
||||||
|
r.setSubErrorCode(subErrorCode);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Map<Boolean, List<ColumnConvertResp>> convertRespMap = convertRespList.stream()
|
||||||
|
.collect(Collectors.groupingBy(ColumnConvertResp::getSuccess));
|
||||||
|
|
||||||
|
JSONObject convertedLine = new JSONObject()
|
||||||
|
.fluentPutAll(convertRespMap.getOrDefault(Boolean.TRUE, ImmutableList.of()).stream()
|
||||||
|
// convertValue可能为null, 有非必需的字段
|
||||||
|
.collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll));
|
||||||
|
if (convertRespMap.get(Boolean.FALSE) != null) {
|
||||||
|
// 转换失败的，将失败信息放到errors字段中
|
||||||
|
convertedLine.put(LINE_KEY_ERRORS, convertRespMap.get(Boolean.FALSE).stream()
|
||||||
|
.map(ColumnConvertResp::getError)
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
convertedLine.put(LINE_KEY_ROW_INDEX, lineIndex);
|
||||||
|
return convertedLine;
|
||||||
|
})
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
private ColumnConvertResp convertType(DataSheetClient.CellMeta cellMeta, String rawValue, int rowIndex, int columnIndex) {
|
||||||
|
try {
|
||||||
|
return ColumnConvertResp.builder()
|
||||||
|
.success(true).cellMeta(cellMeta)
|
||||||
|
.rawValue(rawValue).convertedValue(cellMeta.convertType(rawValue))
|
||||||
|
.build();
|
||||||
|
} catch (BusinessException e) {
|
||||||
|
log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}",
|
||||||
|
cellMeta, rawValue, rowIndex, columnIndex, e);
|
||||||
|
String subErrorCode = null;
|
||||||
|
if (e instanceof DataSheetClient.DataSheetException) {
|
||||||
|
subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode();
|
||||||
|
}
|
||||||
|
return ColumnConvertResp.builder()
|
||||||
|
.success(false).cellMeta(cellMeta).rawValue(rawValue)
|
||||||
|
.columnIndex(columnIndex)
|
||||||
|
.errorCode(e.getErrorCode())
|
||||||
|
.errorMsg(e.getErrorMsg())
|
||||||
|
.subErrorCode(subErrorCode)
|
||||||
|
.build();
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}",
|
||||||
|
cellMeta, rawValue, rowIndex, columnIndex, e);
|
||||||
|
String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1,
|
||||||
|
cellMeta.getName(), Optional.ofNullable(e.getMessage())
|
||||||
|
.orElse(IMPORT_CELL_CONVERT_FAILED.getMessage()));
|
||||||
|
throw new DataSheetClient.DataSheetException(IMPORT_CELL_CONVERT_FAILED.getSubBizCode(), errMsg, rowIndex, columnIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateHeaders(NoModelDataListener dataListener) {
|
||||||
|
Map<Integer, String> headMap = dataListener.getHeadMap();
|
||||||
|
Set<String> headerNames = ImmutableSet.copyOf(headMap.values());
|
||||||
|
if (headerNames.size() != headMap.size()) {
|
||||||
|
List<String> columnNames = headMap.values().stream()
|
||||||
|
.collect(Collectors.groupingBy(Function.identity()))
|
||||||
|
.entrySet().stream()
|
||||||
|
.filter(grouped -> grouped.getValue().size() > 1)
|
||||||
|
.map(Map.Entry::getKey)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_COLUMN_DUPLICATED_NAME.toException(columnNames);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateMeta(NoModelDataListener dataListener) {
|
||||||
|
if (this.meta == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (meta.getCellMetas().size() != dataListener.getHeadMap().size()) {
|
||||||
|
if (dataListener.getHeadMap().size() > meta.getCellMetas().size()) {
|
||||||
|
if (!ignoreUnknownColumn) {
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
long mandatoryColumnSize = meta.getCellMetas().stream()
|
||||||
|
.filter(cm -> BooleanUtils.isTrue(cm.getMandatory()))
|
||||||
|
.count();
|
||||||
|
if (dataListener.getHeadMap().size() < mandatoryColumnSize) {
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Set<String> headerNames = ImmutableSet.copyOf(dataListener.getHeadMap().values());
|
||||||
|
Set<String> cellNames = meta.getCellMetas().stream()
|
||||||
|
.map(DataSheetClient.CellMeta::getName)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
if (!ignoreUnknownColumn && !headerNames.equals(cellNames)) {
|
||||||
|
Set<String> missingNames = Sets.difference(cellNames, headerNames);
|
||||||
|
Set<String> redundantNames = Sets.difference(headerNames, cellNames);
|
||||||
|
List<String> columnNames = Stream.of(missingNames, redundantNames)
|
||||||
|
.flatMap(Set::stream)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
throw DataSheetClient.ResultCode.IMPORT_COLUMN_NAME_NOT_MATCHED.toException(columnNames);
|
||||||
|
}
|
||||||
|
meta.getCellMetas().forEach(DataSheetClient.CellMeta::validate);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@Data
|
||||||
|
private static class NoModelDataListener extends AnalysisEventListener<Map<Integer, String>> {
|
||||||
|
private Map<Integer, String> headMap = Maps.newHashMap();
|
||||||
|
private List<Map<Integer, String>> lines = Lists.newArrayList();
|
||||||
|
private List<CellExtra> cellComments = Lists.newArrayList();
|
||||||
|
|
||||||
|
private Function<String, String> headerConverter;
|
||||||
|
private Boolean autoTrim;
|
||||||
|
|
||||||
|
public NoModelDataListener(Function<String, String> headerConverter, Boolean autoTrim) {
|
||||||
|
this.headerConverter = headerConverter;
|
||||||
|
this.autoTrim = autoTrim;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void invoke(Map<Integer, String> data, AnalysisContext context) {
|
||||||
|
lines.add(strip(data));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
|
||||||
|
Map<Integer, String> convertedMap = headMap;
|
||||||
|
if (headerConverter != null) {
|
||||||
|
convertedMap = headMap.entrySet().stream()
|
||||||
|
.collect(Collectors.toMap(Map.Entry::getKey, e -> headerConverter.apply(e.getValue())));
|
||||||
|
}
|
||||||
|
this.headMap.putAll(strip(convertedMap));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<Integer, String> strip(Map<Integer, String> data) {
|
||||||
|
if (BooleanUtils.isFalse(autoTrim)) {
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
return data.entrySet().stream()
|
||||||
|
.map(entry -> Maps.immutableEntry(entry.getKey(), StringUtils.strip(entry.getValue(), Regex.WHITESPACE_CHARS)))
|
||||||
|
// value有可能为null，不能直接用Collectors.toMap
|
||||||
|
.collect(Maps::newHashMap, (m, v) -> m.put(v.getKey(), v.getValue()), HashMap::putAll);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* extra是在整个文件被解析后才会调用
|
||||||
|
*
|
||||||
|
* @param extra
|
||||||
|
* @param context
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void extra(CellExtra extra, AnalysisContext context) {
|
||||||
|
if (extra.getType() == CellExtraTypeEnum.COMMENT) {
|
||||||
|
cellComments.add(extra);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Builder
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
private static class ColumnConvertResp {
|
||||||
|
private Boolean success;
|
||||||
|
private DataSheetClient.CellMeta cellMeta;
|
||||||
|
private String rawValue;
|
||||||
|
private Object convertedValue;
|
||||||
|
|
||||||
|
private Integer columnIndex;
|
||||||
|
private String errorCode;
|
||||||
|
private String errorMsg;
|
||||||
|
private String subErrorCode;
|
||||||
|
|
||||||
|
public String getKey() {
|
||||||
|
return cellMeta.getKey();
|
||||||
|
}
|
||||||
|
|
||||||
|
public JSONObject getError() {
|
||||||
|
return new JSONObject()
|
||||||
|
.fluentPut("columnIndex", columnIndex)
|
||||||
|
.fluentPut("columnKey", cellMeta.getKey())
|
||||||
|
.fluentPut("columnName", cellMeta.getName())
|
||||||
|
.fluentPut("rawValue", rawValue)
|
||||||
|
.fluentPut("errorCode", errorCode)
|
||||||
|
.fluentPut("errorMsg", errorMsg)
|
||||||
|
.fluentPut("subErrorCode", subErrorCode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
1
pom.xml
1
pom.xml
@ -33,6 +33,7 @@
|
|||||||
<module>gateway-support-lib</module>
|
<module>gateway-support-lib</module>
|
||||||
<module>event-support-lib</module>
|
<module>event-support-lib</module>
|
||||||
<module>redis-support-lib</module>
|
<module>redis-support-lib</module>
|
||||||
|
<module>excel-support-lib</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user