From d4f01bb227ad884775d043e2c3b081c891eabc0d Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 5 Jul 2024 16:17:28 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=AF=BC=E5=85=A5/=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/axzo/foundation/page/PageReq.java | 18 + excel-support-lib/pom.xml | 44 + .../excel/support/DataSheetClient.java | 918 ++++++++++++++++++ .../support/impl/DataSheetAsyncExporter.java | 326 +++++++ .../support/impl/DataSheetClientImpl.java | 90 ++ .../excel/support/impl/DataSheetExporter.java | 698 +++++++++++++ .../excel/support/impl/DataSheetImporter.java | 595 ++++++++++++ pom.xml | 1 + 8 files changed, 2690 insertions(+) create mode 100644 common-lib/src/main/java/cn/axzo/foundation/page/PageReq.java create mode 100644 excel-support-lib/pom.xml create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/DataSheetClient.java create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetAsyncExporter.java create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetClientImpl.java create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetExporter.java create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetImporter.java diff --git a/common-lib/src/main/java/cn/axzo/foundation/page/PageReq.java b/common-lib/src/main/java/cn/axzo/foundation/page/PageReq.java new file mode 100644 index 0000000..7dfbc4c --- /dev/null +++ b/common-lib/src/main/java/cn/axzo/foundation/page/PageReq.java @@ -0,0 +1,18 @@ +package cn.axzo.foundation.page; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PageReq implements IPageReq{ + Integer page; + Integer pageSize; + List sort; +} diff --git a/excel-support-lib/pom.xml b/excel-support-lib/pom.xml new file mode 100644 index 0000000..62c6fac --- /dev/null +++ b/excel-support-lib/pom.xml @@ -0,0 +1,44 @@ + + + 4.0.0 + + cn.axzo.foundation + axzo-lib-box + 2.0.0-SNAPSHOT + + + cn.axzo.maokai + excel-support-lib + + + 8 + 8 + UTF-8 + + + + + cn.axzo.foundation + web-support-lib + 2.0.0-SNAPSHOT + + + com.alibaba + easyexcel + 3.3.4 + + + com.opencsv + opencsv + 5.9 + + + org.springframework.boot + spring-boot-starter-data-redis + provided + + + + \ No newline at end of file diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/DataSheetClient.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/DataSheetClient.java new file mode 100644 index 0000000..989c923 --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/DataSheetClient.java @@ -0,0 +1,918 @@ +package cn.axzo.foundation.excel.support; + +import cn.axzo.foundation.exception.BusinessException; +import cn.axzo.foundation.page.IPageReq; +import cn.axzo.foundation.page.PageResp; +import cn.axzo.foundation.util.VarParamFormatter; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSONPath; +import com.alibaba.fastjson.parser.ParserConfig; +import com.alibaba.fastjson.util.TypeUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import lombok.*; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.poi.ss.SpreadsheetVersion; + +import javax.servlet.http.HttpServletResponse; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*; + +/** + * 用于发送sms & email验证码, 或者获取图片验证码. 并提供校验验证码是否正确的能力 + */ +public interface DataSheetClient { + + ExporterBuilder exportBuilder(); + + /** + * 数据导入的builder + * + * @return + */ + ImporterBuilder importBuilder(); + + @Data + @NoArgsConstructor + @AllArgsConstructor + @Accessors(fluent = true) + abstract class ImporterBuilder { + /** + * 导入的场景名称, 必填 + */ + @NonNull + private String scene; + + /** + * 导入格式 + */ + @NonNull + private ImportFormat format; + + /** + * EXCEL格式的sheetName + */ + private String tableName; + + private Meta meta; + + /** + * 是否忽略导入文件表头的原meta + * 默认按false处理，即{@link #meta}为空时尝试使用原文件表头meta + */ + private Boolean ignoreHeaderMeta; + + private boolean debugEnabled; + + /** + * 忽略未知的列(cellMeta中未定义的列) + */ + private boolean ignoreUnknownColumn; + + /** + * 是否自动清除表格内容的前后空白字符 + * 默认true，即会清除前后空白字符 + */ + private Boolean autoTrim; + + /** + * 解析结果ImportResp中的lines是否包含解析失败的行 + * true: 解析失败的时候，不会抛异常，会在每行的JSONObject中添加 + * "errors":[{"columnIndex":1,"columnKey":"","columnName":"","rawValue":"","errorCode":"","errorMsg":""}] + * false: 解析失败的时候，抛第一个异常 + */ + private boolean includeLineErrors; + + /** + * 允许最大的导入行数 + */ + private Integer allowMaxLineCount; + + /** 将excel中的表头转换为cellMeta中的表头 */ + private Function headerConverter; + + /** + * 操作人, 可选，用于导入统计报告 + */ + private String operator; + + public abstract Importer build(); + } + + interface Importer { + /** + * 所有字段读取为String类型 + * + * @param inputStream + * @return + */ + ImportResp readAll(InputStream inputStream); + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class ImportResp { + /** + * 导入的场景名称 + */ + private String scene; + + /** + * 模版对应的code以及版本 + */ + private String templateCode; + private String version; + + /** + * headers + */ + private List headers; + /** + * 每一行的数据 + */ + private List lines; + + /** + * header的行数 + */ + private Integer headerRowCount; + /** + * 导入数据总行数, 不包含header + */ + private Integer rowCount; + /** + * 导入数据列数 + */ + private Integer columnCount; + + /** + * Meta数据 + */ + private Meta meta; + + /** + * 消耗的时间 + */ + private Long elapsedMillis; + + /** + * 操作人 + */ + private String operator; + } + + @AllArgsConstructor + @Getter + enum ImportFormat { + EXCEL("xlsx"), + // TODO: 支持其他类型 + // CSV("csv") + ; + + private String suffix; + } + + @Data + @NoArgsConstructor + @Accessors(fluent = true) + abstract class ExporterBuilder { + /** + * 导出的场景名称, 必填 + */ + @NonNull + private String scene; + + /** + * 导出格式 + */ + @NonNull + private ExportFormat format; + + /** + * 顶部提示内容，主要用于一些提示场景。如导出数据超范围等文案提示等，支持多行。 + */ + private Function, List> topHintsSupplier; + + /** + * 提供分页获取数据的方法 + */ + @NonNull + private Function> rowSupplier; + + /** + * 导出的总数据量限制，不指定时，按照分页返回的结果全量导出 + */ + private Long limit = Long.MAX_VALUE; + + /** + * 获取数据的分页大小, 默认每页200条(MybatisPlus的PaginationInterceptor默认limit为500) + */ + private Long pageSize = 200L; + + /** + * 表名，仅当导出格式为excel时有效 + */ + private String sheetName; + + @Deprecated + public ExporterBuilder tableName(String tableName) { + this.sheetName = tableName; + return this; + } + + /** + * 列映射表, key=列名， value=获取字段值的方法, 方法参数1为当前row的JSONObject, 参数2为当前row的index，返回列字段值 + */ + private ImmutableMap> columnMap; + + /** + * 多行表头映射表, 当表头由多行数据组成时使用. key=表头, value=该表头对应的每行的内容 + */ + private ImmutableMap> multiLineHeadMap; + + /** + * 行数据的转换器，外部可以通过它将原始的行数据根据需要做转换 + */ + private Function> rowConverter; + + /** + * 页数据的转换器，外部可以通过它将也的数据根据需要做转换 + * 它会在 rowconvert() 执行完成 + */ + private Function, List> pageConverter; + + private BiConsumer onProgress; + + /** + * 批注信息，可选 + */ + private Meta meta; + + /** + * 是否开启调试日志 + */ + private Boolean debugEnabled = true; + + /** + * 导出的fields. 优先使用fields来创建column&meta + */ + private List fields; + + /** + * meta的version + */ + private String version; + + /** 文件名 */ + private String fileName; + + /** 操作人, 可选，用于异步导出 */ + private String operator; + + /** 拓展字段，可用于存放请求参数等，该字段会回传给FileLoaderParam */ + private JSONObject ext; + + /** + * 仅对导出excel生效，是否自动清除表格内容的前后空白字符 + * 默认true，即会清除前后空白字符 + */ + private Boolean autoTrim; + + abstract public Exporter build(); + } + + @Data + class ExportField { + private String column; + private String header; + private List headerLines; + private Integer width; + private CellMeta.Type type; + private Boolean mandatory; + private Boolean wrapText; + + private String lowerType; + private String upperType; + private String dateTimeFormat; + private List options; + + private BiFunction reader; + + @Builder + public ExportField(String column, String header, List headerLines, Integer width, CellMeta.Type type, + Boolean mandatory, Boolean wrapText, Function resultConverter, + BiFunction reader, List options) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(column), "column不能为空"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(header), "header不能为空"); + Preconditions.checkArgument(!(resultConverter != null && reader != null), "reader & resultConverter不能同时存在"); + + this.column = column; + this.header = header; + this.headerLines = headerLines; + this.width = width; + this.type = Optional.ofNullable(type).orElse(CellMeta.Type.STRING); + this.mandatory = Optional.ofNullable(mandatory).orElse(Boolean.FALSE); + this.wrapText = wrapText; + if (reader != null) { + this.reader = reader; + } else { + this.reader = Optional.ofNullable(resultConverter) + .map(e -> (BiFunction) (row, integer) -> e.apply(Strings.nullToEmpty(row.getString(column)))) + .orElseGet(() -> DataSheetClient.stringCellReader(column)); + } + this.options = options; + } + + public CellMeta toCellMeta() { + JSONObject params = new JSONObject(); + if (CellMeta.Type.RANGE == type) { + params.put(EXT_KEY_RANGE_LOWER_TYPE, lowerType); + params.put(EXT_KEY_RANGE_UPPER_TYPE, upperType); + } + if (CellMeta.Type.DATE == type || CellMeta.Type.DATETIME == type) { + params.put(EXT_KEY_DATETIME_FORMAT, dateTimeFormat); + } + + return CellMeta.builder() + .key(column).name(header).width(width).mandatory(mandatory).type(type).wrapText(wrapText) + .params(params).options(options).build(); + } + } + + interface Exporter { + /** + * 导出到OutputStream中 + * + * @param outputStream + * @return + */ + ExportResp export(@NonNull OutputStream outputStream); + + /** + * 导出到HttpServletResponse中 + * + * @param response + * @return + */ + ExportResp export(@NonNull HttpServletResponse response); + + /** + * 导出到HttpServletResponse中，可指定导出的文件名 + * + * @param response + * @return + */ + ExportResp export(HttpServletResponse response, String filename); + } + + interface AsyncExporter { + /** + * 异步导出数据 + * + * @param dataSheetExporter 实现导出逻辑的 exporter + * @return batchId + */ + String export(@NonNull ExporterBuilder dataSheetExporter); + + /** + * 多表单数据异步导出至同一文件中, 仅支持导出格式是EXCEL + * + * @param fileName 导出文件名 + * @param dataSheetExporters 实现导出逻辑的 exporter 列表 + * @return batchId + */ + String export(String fileName, @NonNull List dataSheetExporters); + + Progress getProgress(@NonNull String batchId); + + @NoArgsConstructor + @AllArgsConstructor + @Data + @Builder + class Progress { + String fileName; + int progress; + String errorMsg; + String downloadUrl; + + public boolean isSuccess() { + return Strings.isNullOrEmpty(errorMsg); + } + + public boolean isDone() { + return progress < 0 || progress >= 100; + } + } + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + class ExportResp { + /** + * 导出场景 + */ + private String scene; + /** + * 导出数据条数 + */ + private Long exportedCount; + + /** + * 导出数据总行数，包含了顶部提示行及表头行 + */ + private Long rowCount; + + /** + * 导出数据列数 + */ + private Long columnCount; + + /** + * 导出数据行数 + */ + private Long dataRowCount; + + /** + * 耗时 + */ + private Long elapsedMillis; + + /** + * 操作人 + */ + private String operator; + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class Meta { + public static final String PATTERN_VERSION_OPEN = "$V{"; + public static final String PATTERN_CELL_OPEN = "$C{"; + public static final String PATTERN_IGNORE_OPEN = "$I{"; + public static final String PATTERN_EXTRA_OPEN = "$E{"; + public static final String PATTERN_CLOSE = "}"; + + public static final String RANGE_TYPE_OPEN_OPEN = "1"; + public static final String RANGE_TYPE_OPEN_CLOSED = "2"; + public static final String RANGE_TYPE_CLOSED_OPEN = "3"; + public static final String RANGE_TYPE_CLOSED_CLOSED = "4"; + + public static final String IGNORE_ROW_KEY = "r"; + public static final String IGNORE_COLUMN_KEY = "c"; + public static final String IGNORE_ROW_AND_COLUMN_KEY = "b"; + + private String templateCode; + private String version; + private List cellMetas; + /** + * 忽略掉的行号 + */ + private List ignoreRowIndexes; + /** + * 忽略掉的列号 + */ + private List ignoreColumnIndexes; + /** + * 忽略掉的列名 + * 当导入文件中有meta信息时，该ignore信息将会丢失 + */ + private Set ignoreColumnNames; + + public List getIgnoreRowIndexes() { + return Optional.ofNullable(ignoreRowIndexes).orElseGet(ImmutableList::of); + } + + public List getIgnoreColumnIndexes() { + return Optional.ofNullable(ignoreColumnIndexes).orElseGet(ImmutableList::of); + } + + public Set getIgnoreColumnNames() { + return Optional.ofNullable(ignoreColumnNames).orElseGet(ImmutableSet::of); + } + } + + @SuperBuilder + @Data + @NoArgsConstructor + @AllArgsConstructor + class CellMeta { + public static final String EXT_KEY_DATETIME_FORMAT = "dateTimeFormat"; + public static final String EXT_KEY_RANGE_LOWER_TYPE = "lowerType"; + public static final String EXT_KEY_RANGE_UPPER_TYPE = "upperType"; + + public static final String RANGE_BOUND_TYPE_OPEN = "open"; + public static final String RANGE_BOUND_TYPE_CLOSED = "closed"; + public static final Set RANGE_TYPES = ImmutableSet.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED); + + private static final List DEFAULT_DATE_FORMATS = ImmutableList.of( + "yyyy-MM-dd", + "yyyy-MM-dd HH:mm", + "yyyy-MM-dd HH:mm:ss", + "yyyy.MM.dd", + "yyyy.MM.dd HH:mm", + "yyyy.MM.dd HH:mm:ss", + "yyyy年MM月dd日", + "yyyy年MM月dd日 HH:mm", + "yyyy年MM月dd日 HH:mm:ss", + "yyyy年MM月dd日 HH时mm分", + "yyyy年MM月dd日 HH时mm分ss秒", + "yyyy/MM/dd", + "yyyy/MM/dd HH:mm", + "yyyy/MM/dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss'Z'", + "yyyy-MM-dd'T'HH:mm:ssZ", + "yyyy-MM-dd'T'HH:mm:ssz", + "yyyy-MM-dd'T'HH:mm:ss", + "MM/dd/yyyy HH:mm:ss a", + "yyyyMMddHHmmss", + "yyyyMMdd" + ); + + private static final Map BOOLEAN_MAP = ImmutableMap.builder() + .put("是", Boolean.TRUE) + .put("yes", Boolean.TRUE) + .put("y", Boolean.TRUE) + .put("1", Boolean.TRUE) + .put("否", Boolean.FALSE) + .put("no", Boolean.FALSE) + .put("n", Boolean.FALSE) + .put("0", Boolean.FALSE) + .build(); + + /** + * cell的key + */ + private String key; + /** + * cell的header，一般为中文 + */ + private String name; + /** + * cell的宽度 + */ + private Integer width; + /** + * 类型 + */ + private Type type; + /** + * 是否必需, 为false时, 允许该列在Excel中不存在 + */ + private Boolean mandatory; + /** + * 可选值 + */ + private List options; + /** + * 存放一些和type相关的参数，例如DateTime的pattern格式，Range的开闭区间 + */ + private JSONObject params; + /** + * 存放一些额外的信息 + */ + private JSONObject ext; + + /** + * wrap the text automatically + */ + private Boolean wrapText; + + /** + * 导入的时候，将原始的String转换为特定的类型 + */ + private Function importConverter; + + /** + * 字段值额外的校验方法，支持根据当前字段值及整行数据进行自定义校验 + */ + private BiConsumer cellValidator; + + public void validate() { + if (type == Type.RANGE + && (!RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_LOWER_TYPE)) + || !RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_UPPER_TYPE)))) { + throw ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR.toException(); + } + } + + public Object convertType(String value) { + if (importConverter != null) { + return importConverter.apply(value); + } + + if (Boolean.TRUE.equals(mandatory) && Strings.isNullOrEmpty(value)) { + throw ResultCode.IMPORT_CELL_META_MISSING_MANDATORY_VALUE.toException(); + } + // 非必填的字段如果为空，直接返回 + if (value == null) { + return type == Type.STRING ? StringUtils.EMPTY : null; + } + + switch (type) { + case INT: + // 移除千分符 + return Integer.valueOf(value.replace(",", "")); + case LONG: + // 移除千分符 + return Long.valueOf(value.replace(",", "")); + case FLOAT: + // 移除千分符 + return Float.valueOf(value.replace(",", "")); + case DOUBLE: + // 移除千分符 + return Double.valueOf(value.replace(",", "")); + case BIG_DECIMAL: + // 移除千分符 + return new BigDecimal(value.replace(",", "")); + case DATE: + case DATETIME: + return parseDate(value, DEFAULT_DATE_FORMATS); + case RANGE: + // 格式 100-200 + List values = Splitter.on("-").omitEmptyStrings().trimResults().splitToList(value) + .stream() + .map(Integer::valueOf) + .collect(Collectors.toList()); + if (values.size() != 2) { + throw ResultCode.IMPORT_CELL_RANGE_FORMAT_ERROR.toException(); + } + if (values.get(0) >= values.get(1)) { + throw ResultCode.IMPORT_CELL_RANGE_VALUE_ERROR.toException(); + } + return new JSONObject() + .fluentPut("lower", values.get(0)) + .fluentPut("upper", values.get(1)) + .fluentPut(EXT_KEY_RANGE_LOWER_TYPE, params.getString(EXT_KEY_RANGE_LOWER_TYPE)) + .fluentPut(EXT_KEY_RANGE_UPPER_TYPE, params.getString(EXT_KEY_RANGE_UPPER_TYPE)); + case BOOLEAN: + return Optional.ofNullable(BOOLEAN_MAP.get(value.toLowerCase())) + .orElseThrow(ResultCode.IMPORT_CELL_BOOLEAN_VALUE_ERROR::toException); + default: + return Strings.nullToEmpty(value); + } + } + + private LocalDateTime parseDate(String dateStr, List formats) { + try { + Date date = DateUtils.parseDate(dateStr, formats.toArray(new String[0])); + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } catch (Exception e) { + throw ResultCode.IMPORT_CELL_DATETIME_CONVERT_FAILED.toException("不支持的日期格式{}", dateStr); + } + } + + @AllArgsConstructor + public enum Type { + INT("00"), + LONG("01"), + FLOAT("02"), + DOUBLE("03"), + STRING("04"), + BIG_DECIMAL("05"), + DATE("06"), + DATETIME("07"), + RANGE("08"), + BOOLEAN("09"); + + @Getter + private String code; + + private static Map map = Stream.of(Type.values()) + .collect(Collectors.toMap(Type::getCode, Function.identity())); + + public static Type from(String code) { + return map.get(code); + } + } + } + + @AllArgsConstructor + @Getter + enum ExportFormat { + EXCEL("application/vnd.ms-excel", ".xlsx"), + CSV("text/csv", ".csv"); + + private String contentType; + private String suffix; + } + + static BiFunction indexCellReader() { + return (row, index) -> String.valueOf(index + 1); + } + + static BiFunction stringCellReader(String columnName) { + // excel库poi会检查文本格式的单元格内容，当超过32767时，将会抛出IllegalArgumentException异常 + // 参考 {@link org.apache.poi.xssf.streaming.SXSSFCell.setCellValue} + return (row, index) -> StringUtils.left(Strings.nullToEmpty(row.getString(columnName)), + SpreadsheetVersion.EXCEL2007.getMaxTextLength()); + } + + static BiFunction dateTimeCellReader(String columnName, String pattern) { + return (row, index) -> { + Long epochMillis = row.getLong(columnName); + if (epochMillis == null) { + return StringUtils.EMPTY; + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); + Instant instant = Instant.ofEpochMilli(epochMillis); + LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); + return formatter.format(localDateTime); + }; + } + + static BiFunction bigDecimalCellReader(String columnName) { + return bigDecimalCellReader(columnName, 2); + } + + static BiFunction bigDecimalCellReader(String columnName, int scale) { + return (row, index) -> { + BigDecimal number = row.getBigDecimal(columnName); + if (number == null) { + number = BigDecimal.ZERO; + } + + return number.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString(); + }; + } + + static BiFunction jsonPathCellReader(String jsonPath) { + return (row, index) -> { + Object obj = JSONPath.eval(row, jsonPath); + /** + * excel单元格对于数值最大只能保有15位，超过15位的数值(Long)会被置为0，比如在导出uid的场景，会导致数据错误 + * https://docs.microsoft.com/en-us/office/troubleshoot/excel/last-digits-changed-to-zeros + * https://wenku.baidu.com/view/e1d2a497f624ccbff121dd36a32d7375a417c6ad.html + */ + if (obj instanceof Long && obj.toString().length() > 15) { + obj = obj.toString(); + } + if (obj instanceof String) { + // excel库poi会检查文本格式的单元格内容，当超过32767时，将会抛出IllegalArgumentException异常 + // 参考 {@link org.apache.poi.xssf.streaming.SXSSFCell.setCellValue} + return StringUtils.left(obj.toString(), SpreadsheetVersion.EXCEL2007.getMaxTextLength()); + } + return obj; + }; + } + + static BiFunction jsonPathDateTimeCellReader(String jsonPath, String pattern) { + return (row, index) -> { + Object val = JSONPath.eval(row, jsonPath); + Long epochMillis = TypeUtils.cast(val, Long.class, ParserConfig.getGlobalInstance()); + if (epochMillis == null) { + return StringUtils.EMPTY; + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); + Instant instant = Instant.ofEpochMilli(epochMillis); + LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); + return formatter.format(localDateTime); + }; + } + + static BiFunction jsonPathBigDecimalCellReader(String jsonPath) { + return jsonPathBigDecimalCellReader(jsonPath, 2); + } + + static BiFunction jsonPathBigDecimalCellReader(String jsonPath, int scale) { + return (row, index) -> { + Object val = JSONPath.eval(row, jsonPath); + if (val == null) { + val = BigDecimal.ZERO; + } + BigDecimal bigDecimal = TypeUtils.cast(val, BigDecimal.class, ParserConfig.getGlobalInstance()); + return bigDecimal.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString(); + }; + } + + @Getter + class DataSheetException extends BusinessException { + private String subErrorCode; + /** 异常相关的行号，列号，0开始 */ + private Integer rowIndex; + private Integer columnIndex; + /** 异常相关的列名称，中文 */ + private List columnNames; + + public DataSheetException(ResultCode resultCode) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + } + + public DataSheetException(ResultCode resultCode, Integer rowIndex, Integer columnIndex) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + this.rowIndex = rowIndex; + this.columnIndex = columnIndex; + } + + public DataSheetException(ResultCode resultCode, List columnNames) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + this.columnNames = columnNames; + } + + public DataSheetException(ResultCode resultCode, String message) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), message); + this.subErrorCode = resultCode.getSubBizCode(); + } + + public DataSheetException(String subErrorCode, String errorMsg, Integer rowIndex, Integer columnIndex) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), errorMsg); + this.subErrorCode = subErrorCode; + this.rowIndex = rowIndex; + this.columnIndex = columnIndex; + } + + public DataSheetException(String errorCode, String subErrorCode, String errorMsg) { + super(errorCode, errorMsg); + this.subErrorCode = subErrorCode; + } + } + + @AllArgsConstructor + @Getter + enum ResultCode { + /** 解析Excel的批注时，相关的errorCode */ + IMPORT_PARSE_MISSING_VERSION("C00", "批注中没有找到版本信息"), + IMPORT_PARSE_VERSION_FORMAT_ERROR("C01", "批注中的版本格式不对"), + IMPORT_PARSE_CELL_META_MISSING_TYPE("C02", "批注中的字段没有找到类型信息"), + IMPORT_PARSE_CELL_META_FORMAT_ERROR("C03", "批注中的字段格式不对"), + IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR("C04", "批注中的范围类型字段的格式不对"), + IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR("C05", "批注中的范围类型字段的区间类型不对"), + + /** 导入数据时，相关的errorCode */ + IMPORT_LINES_REACHED_LIMIT("C20", "导入的数据超过了最大行数"), + IMPORT_COLUMN_RANGE_MISSING_TYPES("C21", "范围类型缺少区间类型的定义：开区间还是闭区间"), + IMPORT_COLUMN_DUPLICATED_NAME("C22", "列的名称不能重复"), + IMPORT_COLUMN_MISSING_CELL_META("C23", "字段缺少类型定义"), + IMPORT_COLUMN_NAME_NOT_MATCHED("C24", "字段的名称与类型定义的名称不一致"), + IMPORT_CELL_RANGE_FORMAT_ERROR("C25", "范围类型的格式不对"), + IMPORT_CELL_RANGE_VALUE_ERROR("C26", "范围类型的下限值必须小于上限值"), + IMPORT_CELL_DATETIME_CONVERT_FAILED("C27", "时间类型解析失败"), + IMPORT_CELL_CONVERT_FAILED("C28", "类型转换失败"), + IMPORT_CELL_META_MISSING_MANDATORY_VALUE("C29", "必填字段不能为空"), + IMPORT_CELL_BOOLEAN_VALUE_ERROR("C30", "布尔类型的值不支持"), + ; + + private String subBizCode; + private String message; + + public DataSheetException toException() { + return new DataSheetException(this); + } + + public DataSheetException toException(String message) { + return new DataSheetException(this, message); + } + + public DataSheetException toException(List columnNames) { + return new DataSheetException(this, columnNames); + } + + public DataSheetException toException(Integer rowIndex, Integer columnIndex) { + return new DataSheetException(this, rowIndex, columnIndex); + } + + public DataSheetException toException(String customMsg, Object... objects) { + if (objects == null) { + return toException(customMsg); + } + + String msg = VarParamFormatter.format(customMsg, objects); + //如果最后一个参数是Throwable. 则将SimpleName附加到msg中 + if (objects[objects.length - 1] instanceof Throwable) { + Throwable throwable = (Throwable) objects[objects.length - 1]; + msg = String.format("%s (%s)", msg, throwable.getClass().getSimpleName()); + } + + return toException(msg); + } + } +} diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetAsyncExporter.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetAsyncExporter.java new file mode 100644 index 0000000..7a2f0b3 --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetAsyncExporter.java @@ -0,0 +1,326 @@ +package cn.axzo.foundation.excel.support.impl; + +import cn.axzo.foundation.enums.AppEnvEnum; +import cn.axzo.foundation.excel.support.DataSheetClient; +import cn.axzo.foundation.util.UUIDBuilder; +import cn.axzo.foundation.web.support.AppRuntime; +import cn.axzo.foundation.web.support.utils.KeyBuilder; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.excel.util.IoUtils; +import com.alibaba.excel.util.StringUtils; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.task.TaskExecutor; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.util.CollectionUtils; + +import java.io.*; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * 异步数据导出. + * 将导出的进度防止redis 中，当导出完成后，将文件通过 fileUploader 上传到云存储。 + */ +@Slf4j +public class DataSheetAsyncExporter implements DataSheetClient.AsyncExporter { + + private static final int PROGRESS_EXPIRE_DAYS = 3; + private RedisTemplate redisTemplate; + private AppRuntime appRuntime; + private Function fileUploader; + private TaskExecutor executor; + + public DataSheetAsyncExporter(@NonNull RedisTemplate redisTemplate, + @NonNull AppRuntime appRuntime, + @NonNull Function fileUploader, + @NonNull TaskExecutor executor) { + this.redisTemplate = redisTemplate; + this.appRuntime = appRuntime; + this.fileUploader = fileUploader; + this.executor = executor; + } + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + public static class FileUploadParam { + private byte[] bytes; + private String fileName; + private String operator; + private JSONObject ext; + } + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + public static class FileUploadResult { + private String downloadUrl; + } + + @Override + public String export(@NonNull DataSheetClient.ExporterBuilder dataSheetExporter) { + final String batchId = UUIDBuilder.generateLongUuid(false); + final String scene = dataSheetExporter.scene(); + ExportContext context = ExportContext.builder().batchId(batchId).fileName(dataSheetExporter.fileName()) + .operator(dataSheetExporter.operator()).ext(dataSheetExporter.ext()).build(); + executor.execute(() -> doExport(context, scene, dataSheetExporter)); + return batchId; + } + + @Override + public String export(String fileName, @NonNull List dataSheetExporters) { + checkMultiExporters(fileName, dataSheetExporters); + + final String batchId = UUIDBuilder.generateLongUuid(false); + ExportContext context = ExportContext.builder() + .batchId(batchId) + .fileName(fileName) + .operator(dataSheetExporters.get(0).operator()) + .writer(EasyExcel.write(new ByteArrayOutputStream()) + .registerWriteHandler(new LongestMatchColumnWidthStyleStrategy()) + .build()) + .progressMap(dataSheetExporters.stream().collect(Collectors.toMap(e -> e.sheetName(), e -> 0))) + .build(); + + executor.execute(() -> dataSheetExporters.forEach(e -> doExport(context, e.scene(), e))); + return batchId; + } + + void doExport(final ExportContext context, final String scene, DataSheetClient.ExporterBuilder dataSheetExporter) { + try { + final File tempFile = File.createTempFile(context.getBatchId(), ".tmp"); + tempFile.deleteOnExit(); + log.info("DataSheetAsyncExporter export() scene={} context={}, tempFile->{}", scene, context, tempFile.getAbsolutePath()); + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) { + DataSheetClient.Exporter exporter = dataSheetExporter.onProgress((progress, errorMsg) -> { + if (context.isMultiSheet()) { + onMultiSheetProgress(progress, errorMsg, context, tempFile, dataSheetExporter.sheetName()); + } else { + onProgress(progress, errorMsg, context, tempFile); + } + }).build(); + + Preconditions.checkState(exporter != null); + exporter.export(out); + } catch (Exception e) { + log.error("DataSheetAsyncExporter export() FAIL scene={} context={} ", scene, context, e); + } finally { + if (tempFile != null && tempFile.exists()) { + tempFile.delete(); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void checkMultiExporters(String fileName, List dataSheetExporters) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(fileName), "导出文件名不能为空"); + Preconditions.checkArgument(!CollectionUtils.isEmpty(dataSheetExporters), "导出参数不能为空"); + dataSheetExporters.forEach(e -> Preconditions.checkArgument(e.format() == DataSheetClient.ExportFormat.EXCEL, "导出格式必须为EXCEL")); + dataSheetExporters.forEach(e -> Preconditions.checkArgument(!Strings.isNullOrEmpty(e.sheetName()), "表单名不能为空")); + + long sheetCount = dataSheetExporters.stream().map(DataSheetClient.ExporterBuilder::sheetName).distinct().count(); + Preconditions.checkArgument(sheetCount == dataSheetExporters.size(), "表单名不能重复"); + } + + private String buildRedisKey(String batchId) { + return KeyBuilder.build(appRuntime, "data_export", batchId); + } + + private void onProgress(Integer progress, String errorMsg, ExportContext context, File tempFile) { + log.info("DataSheetAsyncExporter onProgress() progress={} errorMsg={} context={} tempFile={}", progress, errorMsg, context, tempFile.getAbsolutePath()); + String downloadUrl = StringUtils.EMPTY; + if (Strings.isNullOrEmpty(errorMsg) && progress >= 100) { + try { + // 只有导出成功，并完成，才开始上传文件 + byte[] bytes = IoUtils.toByteArray(new BufferedInputStream(new FileInputStream(tempFile))); + downloadUrl = fileUploader.apply(FileUploadParam.builder() + .bytes(bytes).fileName(context.getFileName()) + .operator(context.getOperator()).ext(context.getExt()).build()).getDownloadUrl(); + log.info("DataSheetAsyncExporter onProgress() DONE context={}, downloadUrl={}", context, downloadUrl); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + // 将导出的状态放入 redis 中。后续通过getProgress()来获取。 + String jsonString = JSONObject.toJSONString(Progress.builder().fileName(context.getFileName()).progress(progress) + .downloadUrl(downloadUrl).errorMsg(errorMsg).build()); + redisTemplate.opsForValue().set(buildRedisKey(context.getBatchId()), jsonString, PROGRESS_EXPIRE_DAYS, TimeUnit.DAYS); + } + + private void onMultiSheetProgress(Integer progress, String errorMsg, ExportContext context, File tempFile, String sheetName) { + log.info("DataSheetAsyncExporter onMultiSheetProgress() context={}, progress={} errorMsg={} tempFile={} sheetName={}", context, progress, errorMsg, tempFile.getAbsolutePath(), sheetName); + context.getProgressMap().put(sheetName, progress); + Progress totalProgress = getMultiSheetProgress(context, errorMsg); + + if (!Strings.isNullOrEmpty(errorMsg) && progress < 0) { + context.getWriter().write(ImmutableList.of(ImmutableList.of(errorMsg)), + EasyExcel.writerSheet(sheetName).build()); + } else if (progress >= 100) { + EasyExcel.read(tempFile, new ReadAndWriteListener(context, sheetName)).sheet(sheetName).doRead(); + } + + if (totalProgress.isDone()) { + // 只有所有表单都导出完成，才开始上传文件 + context.getWriter().finish(); + ByteArrayOutputStream outputStream = (ByteArrayOutputStream) context.getWriter().writeContext() + .writeWorkbookHolder().getOutputStream(); + String downloadUrl = fileUploader.apply(FileUploadParam.builder() + .bytes(outputStream.toByteArray()).fileName(context.getFileName()) + .operator(context.getOperator()).build()).getDownloadUrl(); + log.info("DataSheetAsyncExporter onMultiSheetProgress() DONE context={}, downloadUrl={}", context, downloadUrl); + totalProgress.setDownloadUrl(downloadUrl); + } + + // 将导出的状态放入 redis 中。后续通过getProgress()来获取。 + String jsonString = JSONObject.toJSONString(totalProgress); + redisTemplate.opsForValue().set(buildRedisKey(context.getBatchId()), jsonString, PROGRESS_EXPIRE_DAYS, TimeUnit.DAYS); + } + + private Progress getMultiSheetProgress(ExportContext context, String errorMsg) { + Map progressMap = context.getProgressMap(); + int totalProgress = progressMap.values().stream().mapToInt(a -> a).sum() / progressMap.size(); + boolean isDone = progressMap.values().stream().allMatch(a -> a < 0 || a >= 100); + if (isDone && totalProgress != 100) { + // 所有导出都已完成，但其中有失败，将progress设为-1 + totalProgress = -1; + } + + return Progress.builder() + .fileName(context.getFileName()) + .progress(totalProgress) + .errorMsg(errorMsg) + .downloadUrl(StringUtils.EMPTY) + .build(); + } + + /** + * 使用在export()返回的 batchId 来查询导出进度。 + * + * @param batchId + * @return + */ + @Override + public Progress getProgress(String batchId) { + String res = redisTemplate.opsForValue().get(buildRedisKey(batchId)); + if (Strings.isNullOrEmpty(res)) { + return Progress.builder().fileName(StringUtils.EMPTY) + .progress(0).errorMsg(StringUtils.EMPTY).downloadUrl(StringUtils.EMPTY).build(); + } + return JSONObject.parseObject(res, Progress.class); + } + + @Data + public static class Builder { + private RedisTemplate redisTemplate; + private AppRuntime appRuntime; + private Function fileUploader; + private TaskExecutor executor; + + public Builder redisTemplate(RedisTemplate redisTemplate) { + this.redisTemplate = redisTemplate; + return this; + } + + public Builder appRuntime(AppRuntime appRuntime) { + this.appRuntime = appRuntime; + return this; + } + + public Builder fileUploader(Function fileUploader) { + this.fileUploader = fileUploader; + return this; + } + + public Builder executor(TaskExecutor executor) { + this.executor = executor; + return this; + } + + public DataSheetClient.AsyncExporter build() { + if (appRuntime.getEnv() == AppEnvEnum.unittest) { + return new DataSheetClient.AsyncExporter() { + @Override + public String export(DataSheetClient.@NonNull ExporterBuilder dataSheetExporter) { + return "8888"; + } + + @Override + public String export(String fileName, @NonNull List dataSheetExporters) { + return "8888"; + } + + @Override + public Progress getProgress(String batchId) { + return Progress.builder().progress(100).downloadUrl("http://test/test.jpg").build(); + } + }; + } + return new DataSheetAsyncExporter(redisTemplate, this.appRuntime, this.fileUploader, this.executor); + } + } + + private static class ReadAndWriteListener extends AnalysisEventListener> { + private ExportContext context; + private WriteSheet sheet; + + public ReadAndWriteListener(ExportContext context, String sheetName) { + this.context = context; + this.sheet = EasyExcel.writerSheet(sheetName).build(); + } + + @Override + public void invoke(Map data, AnalysisContext context) { + List row = data.entrySet().stream().sorted(Map.Entry.comparingByKey()) + .map(Map.Entry::getValue).collect(Collectors.toList()); + this.context.getWriter().write(ImmutableList.of(row), sheet); + } + + @Override + public void invokeHeadMap(Map headMap, AnalysisContext context) { + List row = headMap.entrySet().stream().sorted(Map.Entry.comparingByKey()) + .map(Map.Entry::getValue).collect(Collectors.toList()); + this.context.getWriter().write(ImmutableList.of(row), sheet); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + } + } + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + static class ExportContext { + private String batchId; + private String fileName; + private String operator; + private ExcelWriter writer; + private Map progressMap; + private JSONObject ext; + + private boolean isMultiSheet() { + return writer != null && !CollectionUtils.isEmpty(progressMap); + } + } +} diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetClientImpl.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetClientImpl.java new file mode 100644 index 0000000..12b6fcd --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetClientImpl.java @@ -0,0 +1,90 @@ +package cn.axzo.foundation.excel.support.impl; + +import cn.axzo.foundation.excel.support.DataSheetClient; +import com.alibaba.fastjson.JSONObject; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cglib.beans.BeanMap; + +import java.util.HashMap; +import java.util.Map; + +@Slf4j +public class DataSheetClientImpl implements DataSheetClient { + + private final static String UNKNOWN_OPERATOR = "unknown@fiture.com"; + + @Override + public ExporterBuilder exportBuilder() { + DataSheetExporter exporter = new DataSheetExporter(); + exporter.setOnCompleted(this::reportExport); + return exporter; + } + + @Override + public ImporterBuilder importBuilder() { + DataSheetImporter importer = new DataSheetImporter(); + importer.setOnCompleted(this::reportImport); + return importer; + } + + public void reportExport(DataSheetClient.ExportResp resp) { + String operator = resp.getOperator(); + + ReportReq req = ReportReq.builder() + .scene(resp.getScene()) + .action(ReportReq.Action.EXPORT) + .resp(BeanMap.create(resp)) + .rowCount(resp.getRowCount()) + .elapsedMillis(resp.getElapsedMillis()) + .operatorId(operator) + .operatorName(operator) + .build(); + report(req); + } + + public void reportImport(DataSheetClient.ImportResp resp) { + String operator = resp.getOperator(); + + JSONObject jsonObject = new JSONObject(new HashMap<>(BeanMap.create(resp))); + ReportReq req = ReportReq.builder() + .scene(resp.getScene()) + .action(ReportReq.Action.IMPORT) + .resp(jsonObject.fluentRemove("lines").fluentRemove("meta")) + .rowCount(resp.getRowCount().longValue()) + .elapsedMillis(resp.getElapsedMillis()) + .operatorId(operator) + .operatorName(operator) + .build(); + report(req); + } + + private boolean report(ReportReq req) { + return true; + } + + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + private static class ReportReq { + String appName; + String scene; + Action action; + Map resp; + Long rowCount; + Long elapsedMillis; + String filePath; + String operatorId; + String operatorName; + String operatorTenantId; + + public enum Action { + IMPORT, + EXPORT; + } + } +} diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetExporter.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetExporter.java new file mode 100644 index 0000000..5e3cf7d --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetExporter.java @@ -0,0 +1,698 @@ +package cn.axzo.foundation.excel.support.impl; + +import cn.axzo.foundation.excel.support.DataSheetClient; +import cn.axzo.foundation.exception.BusinessException; +import cn.axzo.foundation.page.PageReq; +import cn.axzo.foundation.page.PageResp; +import cn.axzo.foundation.result.ResultCode; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.metadata.Head; +import com.alibaba.excel.metadata.data.WriteCellData; +import com.alibaba.excel.util.StringUtils; +import com.alibaba.excel.write.builder.ExcelWriterBuilder; +import com.alibaba.excel.write.handler.AbstractCellWriteHandler; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.excel.write.metadata.holder.WriteSheetHolder; +import com.alibaba.excel.write.metadata.holder.WriteTableHolder; +import com.alibaba.excel.write.style.AbstractCellStyleStrategy; +import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy; +import com.alibaba.excel.write.style.column.SimpleColumnWidthStyleStrategy; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; +import com.google.common.collect.*; +import com.opencsv.CSVWriter; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.CharEncoding; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.ss.util.CellRangeAddressList; +import org.apache.poi.xssf.usermodel.XSSFClientAnchor; +import org.apache.poi.xssf.usermodel.XSSFRichTextString; +import org.springframework.util.CollectionUtils; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*; +import static cn.axzo.foundation.excel.support.DataSheetClient.Meta.*; + + +@Slf4j +public class DataSheetExporter extends DataSheetClient.ExporterBuilder implements DataSheetClient.Exporter { + /** 完成后处理事件. */ + @Setter + private Consumer onCompleted; + + @Override + public DataSheetClient.Exporter build() { + + if (!CollectionUtils.isEmpty(this.fields())) { + + if (this.columnMap() != null) { + throw new RuntimeException("column map is present, please remove column or fields"); + } + if (this.meta() != null) { + throw new RuntimeException("meta is present, please remove meta or fields"); + } + + ImmutableMap> columns = this.fields().stream() + .collect(ImmutableMap.toImmutableMap(DataSheetClient.ExportField::getHeader, DataSheetClient.ExportField::getReader)); + this.columnMap(columns); + + ImmutableMap> multiLineHeadMap = this.fields().stream() + .filter(f -> !CollectionUtils.isEmpty(f.getHeaderLines())) + .collect(ImmutableMap.toImmutableMap(DataSheetClient.ExportField::getHeader, DataSheetClient.ExportField::getHeaderLines)); + this.multiLineHeadMap(multiLineHeadMap); + + DataSheetClient.Meta meta = DataSheetClient.Meta.builder() + .templateCode(this.scene()) + .version(Optional.ofNullable(this.version()).orElse("00")) + .cellMetas(this.fields().stream().map(DataSheetClient.ExportField::toCellMeta).collect(Collectors.toList())) + .build(); + this.meta(meta); + } + + Preconditions.checkArgument(this.scene() != null, "scene不能为空"); + Preconditions.checkArgument(this.format() != null, "format不能为空"); + Preconditions.checkArgument(this.rowSupplier() != null, "rowSupplier不能为空"); + Preconditions.checkArgument(this.columnMap() != null, "columnMap不能为空"); + return this; + } + + @Override + public DataSheetClient.ExportResp export(OutputStream outputStream) { + Preconditions.checkArgument(outputStream != null, "outputStream不能为空"); + + long exportedCount = 0; + Exporter exporter = null; + Stopwatch stopwatch = Stopwatch.createStarted(); + String errorMsg = ""; + + try { + exporter = getDataExporter(outputStream); + + long pageNumber = 1; + while (limit() > exportedCount) { + // 每页最多允许数量 + long pageMaxSize = Math.min(limit() - exportedCount, pageSize()); + if (pageMaxSize <= 0) { + break; + } + PageReq pageParam = PageReq.builder() + .page((int) pageNumber) + // 这里使用pageSize，而不是pageMaxSize，否则会导致最后一次查询的数据和前面的查询数据重复 + .pageSize(pageSize().intValue()) + .build(); + + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, request data, pageParam={}", scene(), pageParam); + } + PageResp result = rowSupplier().apply(pageParam); + + // nextPageMaxSize少于查询的pageSize，说明达到了最大导出数，只提取允许的数量 + if (result.getData().size() > pageMaxSize && pageMaxSize < pageSize()) { + result = PageResp.builder().current(result.getCurrent()) + .size(result.getSize()).total(result.getTotal()) + .data(result.getData().subList(0, (int) pageMaxSize)) + .build(); + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, result data, and split result size to pageMaxSize={}, size={}", + scene(), pageMaxSize, result.getData().size()); + } + } else { + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, result data, size={}", + scene(), result.getData().size()); + } + } + + if (pageNumber == 1) { + if (!CollectionUtils.isEmpty(multiLineHeadMap())) { + List> multiLineHeads = columnMap().keySet().stream() + .map(headName -> multiLineHeadMap().getOrDefault(headName, ImmutableList.of(headName))) + .collect(Collectors.toList()); + exporter.writeMultiLineHeads(multiLineHeads); + } else { + exporter.writeHead(Lists.newArrayList(columnMap().keySet())); + } + + // 第一页数据需要写顶部提示内容及表头 + if (topHintsSupplier() != null) { + exporter.writeTopHints(topHintsSupplier().apply(result)); + } + } + writeData(exporter, result); + + exportedCount += result.getData().size(); + + if (onProgress() != null) { + //XXX： 不知道最大的页数，这里模拟下进度，保证不超过 100% + int progress = Math.min((int) pageNumber, 90); + onProgress().accept(progress, ""); + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, progress={}", scene(), progress); + } + } + + if (result.getData().size() < result.getSize()) { + break; + } + pageNumber += 1; + } + } catch (Exception e) { + errorMsg = e.getMessage(); + if (!StringUtils.isEmpty(errorMsg) && onProgress() != null) { + onProgress().accept(-1, errorMsg); + log.error("-------ExportClient ERROR [{}]------, progress={}", scene(), -1); + } + throw e; + } finally { + if (exporter != null) { + exporter.finish(); + if (StringUtils.isEmpty(errorMsg) && onProgress() != null) { + onProgress().accept(100, ""); + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, progress={}", scene(), 100); + } + } + } + } + + stopwatch.stop(); + DataSheetClient.ExportResp exportResp = DataSheetClient.ExportResp.builder() + .scene(scene()) + .exportedCount(exportedCount) + .rowCount(exporter.getRowCount()) + .columnCount(exporter.getColumnCount()) + .dataRowCount(exporter.getDataRowCount()) + .elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS)) + .operator(operator()) + .build(); + + if (onCompleted != null) { + onCompleted.accept(exportResp); + } + + log.info("-------ExportClient[{}]------, finish export, exportResp={}", scene(), exportResp); + return exportResp; + } + + @Override + public DataSheetClient.ExportResp export(HttpServletResponse response) { + return export(response, null); + } + + @Override + public DataSheetClient.ExportResp export(HttpServletResponse response, String filename) { + Preconditions.checkArgument(response != null, "response不能为空"); + + if (filename == null) { + filename = getFilename(format().getSuffix()); + } + + try { + filename = URLEncoder.encode(filename, CharEncoding.UTF_8); + } catch (UnsupportedEncodingException e) { + throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出失败，请稍后重试"); + } + + response.setCharacterEncoding(CharEncoding.UTF_8); + response.setContentType(format().getContentType()); + + try { + response.setHeader("Content-Disposition", "attachment;filename=" + filename); + + log.info("-------ExportClient[{}]------, write to servlet response, contentType={}, filename={}", + scene(), format().getContentType(), filename); + + return export(response.getOutputStream()); + } catch (Exception ex) { + log.error("导出数据异常，文件名：" + filename, ex); + throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出失败，请稍后重试"); + } + } + + private Exporter getDataExporter(OutputStream outputStream) { + return format() == DataSheetClient.ExportFormat.CSV ? + new CsvExporter(outputStream, scene(), debugEnabled()) : + new ExcelExporter(sheetName(), outputStream, scene(), meta(), debugEnabled(), autoTrim()); + } + + private String getFilename(String suffix) { + if (fileName() != null) { + return fileName(); + } + String formatDatetime = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").format(LocalDateTime.now()); + return (Strings.isNullOrEmpty(sheetName()) ? "" : sheetName() + "-" + formatDatetime) + suffix; + } + + private void writeData(Exporter exporter, PageResp page) { + if (CollectionUtils.isEmpty(page.getData())) { + return; + } + + List rowPage = page.getData(); + if (rowConverter() != null) { + rowPage = page.getData().stream() + .map(row -> { + List converted = rowConverter().apply(row); + + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, row converted, row={}, converted={}", + scene(), row, converted); + } + + return converted; + }) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } + if (pageConverter() != null) { + final List originRowPage = rowPage; + rowPage = pageConverter().apply(rowPage); + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, page converted, old={}, converted={}", + scene(), originRowPage, rowPage); + } + } + + int rowIndex = (int) exporter.getDataRowCount(); + List finalRowPage = rowPage; + List> rows = IntStream.range(0, finalRowPage.size()) + .mapToObj(index -> columnMap().values().stream() + .map(v -> v.apply(finalRowPage.get(index), rowIndex + index)) + .collect(Collectors.toList())) + .collect(Collectors.toList()); + rows.forEach(exporter::writeRow); + } + + private class CsvExporter extends RowColumnCounter implements Exporter { + + private CSVWriter writer; + private String scene; + private boolean debugEnabled; + + private CsvExporter(OutputStream outputStream, String scene, boolean debugEnabled) { + try { + // encoding + outputStream.write(0xef); + outputStream.write(0xbb); + outputStream.write(0xbf); + } catch (IOException e) { + throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出数据失败"); + } + + writer = new CSVWriter(new OutputStreamWriter(outputStream)); + this.scene = scene; + this.debugEnabled = debugEnabled; + } + + @Override + public void writeTopHints(List topHints) { + if (CollectionUtils.isEmpty(topHints)) { + return; + } + topHints.forEach(t -> writer.writeNext(new String[]{t})); + countTopHints(topHints.size()); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write top={}", scene, topHints); + } + } + + @Override + public void writeHead(List head) { + writer.writeNext(head.toArray(new String[0])); + countHead(head.size()); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write head={}", scene, head); + } + } + + @Override + public void writeMultiLineHeads(List> multiLineHeads) { + int lineNum = multiLineHeads.stream().mapToInt(List::size).max().orElse(0); + IntStream.range(0, lineNum).forEach(i -> { + List heads = multiLineHeads.stream().map(head -> { + // 当表头的行数小于最大行数时，最上面的行中表头内容写入空字符串 + int line = i - lineNum + head.size(); + return line < 0 ? StringUtils.EMPTY : head.get(line); + }).collect(Collectors.toList()); + writeHead(heads); + }); + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write multi line heads={}", scene, multiLineHeads); + } + } + + @Override + public void writeRow(List row) { + String[] arrayContents = row.stream() + .map(c -> c == null ? StringUtils.EMPTY : c.toString()) + .toArray(String[]::new); + writer.writeNext(arrayContents); + countRow(row.size()); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write row={}", scene, row); + } + } + + @Override + public void finish() { + try { + writer.close(); + } catch (IOException e) { + throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出数据失败"); + } + } + } + + private class ExcelExporter extends RowColumnCounter implements Exporter { + + private final Integer DEFAULT_WIDTH = 6; + + private ExcelWriter writer; + private WriteSheet sheet; + private String scene; + private boolean debugEnabled; + private HeaderCommentWriteHandler commentWriteHandler; + private DataValidationWriteHandler validationWriteHandler; + private int topHintsRowCount = 0; + + private ExcelExporter(String sheetName, OutputStream outputStream, String scene, DataSheetClient.Meta meta, + boolean debugEnabled, Boolean autoTrim) { + + ExcelWriterBuilder excelWriterBuilder = EasyExcel.write(outputStream); + if (Objects.nonNull(meta) && !CollectionUtils.isEmpty(meta.getCellMetas())) { + Integer width = meta.getCellMetas().get(0).getWidth(); + excelWriterBuilder.registerWriteHandler(new SimpleColumnWidthStyleStrategy(Objects.nonNull(width) ? + width : DEFAULT_WIDTH)); + } else { + excelWriterBuilder.registerWriteHandler(new LongestMatchColumnWidthStyleStrategy()); + } + + if (meta != null) { + excelWriterBuilder.registerWriteHandler(new AbstractCellStyleStrategy() { + private CellStyle wrapStyle; + + @Override + protected void setHeadCellStyle(Cell cell, Head head, Integer relativeRowIndex) { + } + + @Override + protected void setContentCellStyle(Cell cell, Head head, Integer relativeRowIndex) { + if (cell.getColumnIndex() >= meta.getCellMetas().size()) { + return; + } + DataSheetClient.CellMeta cellMeta = meta.getCellMetas().get(cell.getColumnIndex()); + if (cellMeta != null && BooleanUtils.isTrue(cellMeta.getWrapText())) { + cell.setCellStyle(wrapStyle); + } + } + }); + commentWriteHandler = new HeaderCommentWriteHandler(meta); + validationWriteHandler = new DataValidationWriteHandler(meta); + + excelWriterBuilder.registerWriteHandler(new AbstractCellWriteHandler() { + @Override + public void afterCellDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, List> cellDataList, Cell cell, Head head, Integer relativeRowIndex, Boolean isHead) { + boolean isHeadRow = cell.getRowIndex() == topHintsRowCount; + commentWriteHandler.afterCellDispose(cell, isHeadRow); + validationWriteHandler.afterCellDispose(cell, isHeadRow); + } + }); + } + + writer = excelWriterBuilder.autoTrim(autoTrim).build(); + sheet = EasyExcel.writerSheet(sheetName).build(); + this.scene = scene; + this.debugEnabled = debugEnabled; + } + + @Override + public void writeTopHints(List topHints) { + if (CollectionUtils.isEmpty(topHints)) { + return; + } + countTopHints(topHints.size()); + topHintsRowCount += topHints.size(); + writer.write(topHints.stream().map(ImmutableList::of).collect(Collectors.toList()), sheet); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write top={}", scene, topHints); + } + } + + @Override + public void writeHead(List head) { + sheet.setHead(head.stream().map(Lists::newArrayList).collect(Collectors.toList())); + countHead(head.size()); + + // XXX 写入head后需要调用一次write()确保excel中有数据，否则数据为空时将出现内容异常 + writer.write(ImmutableList.of(), sheet); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write head={}", scene, head); + } + } + + @Override + public void writeMultiLineHeads(List> multiLineHeads) { + // XXX setHead()方法有副作用，内部逻辑会改变传入的list, 因此需要确保传入的list可操作 + sheet.setHead(multiLineHeads.stream().map(Lists::newArrayList).collect(Collectors.toList())); + int lineNum = multiLineHeads.stream().mapToInt(List::size).max().orElse(0); + IntStream.range(0, lineNum).forEach(i -> countHead(multiLineHeads.size())); + + // XXX 写入head后需要调用一次write()确保excel中有数据，否则数据为空时将出现内容异常 + writer.write(ImmutableList.of(), sheet); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write multi line heads={}", scene, multiLineHeads); + } + } + + @Override + public void writeRow(List row) { + writer.write(ImmutableList.of(row), sheet); + countRow(row.size()); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write row={}", scene, row); + } + } + + @Override + public void finish() { + writer.finish(); + } + } + + /** + * Excel添加头部的批注 + * XXX: 此处不能继承AbstractCellWriteHandler 会导致 mybatis typehandler 扫描时被加载 + * 导致老项目没有使用 easyexcel2.2 的项目启动失败。 + */ + private static class HeaderCommentWriteHandler { + private static final Map, String> rangeBoundTypes = ImmutableMap.of( + ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN), RANGE_TYPE_OPEN_OPEN, + ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED), RANGE_TYPE_OPEN_CLOSED, + ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN), RANGE_TYPE_CLOSED_OPEN, + ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED), RANGE_TYPE_CLOSED_CLOSED); + + private String templateCode; + private String version; + private Map cellMetaMap; + + private Set typeWithParams = ImmutableSet.of( + DataSheetClient.CellMeta.Type.RANGE, + DataSheetClient.CellMeta.Type.DATE, + DataSheetClient.CellMeta.Type.DATETIME + ); + + private HeaderCommentWriteHandler(DataSheetClient.Meta meta) { + this.templateCode = meta.getTemplateCode(); + this.version = meta.getVersion(); + + this.cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), DataSheetClient.CellMeta::getName); + } + + public void afterCellDispose(Cell cell, boolean isHead) { + // 只添加表头行的批注 + if (!isHead) { + return; + } + + List comments = Lists.newArrayList(); + if (cell.getColumnIndex() == 0) { + String content = Joiner.on("_").join(ImmutableList.of(templateCode, version)); + comments.add(buildComment(PATTERN_VERSION_OPEN, content)); + } + + DataSheetClient.CellMeta cellMeta = cellMetaMap.get(cell.getStringCellValue()); + if (cellMeta == null) { + return; + } + // 格式： key_type_mandatory + String content = Joiner.on("_").join(ImmutableList.of( + cellMeta.getKey(), + cellMeta.getType().getCode(), + Boolean.TRUE.equals(cellMeta.getMandatory()) ? "1" : "0")); + + if (typeWithParams.contains(cellMeta.getType())) { + content = content.concat("_").concat(getParamsByType(cellMeta)); + } + comments.add(buildComment(PATTERN_CELL_OPEN, content)); + + // 一个Cell只能写入一次Comment + writeComment(cell, Joiner.on("\n").join(comments)); + } + + private String getParamsByType(DataSheetClient.CellMeta cellMeta) { + switch (cellMeta.getType()) { + case RANGE: + String lowerType = cellMeta.getParams().getString(EXT_KEY_RANGE_LOWER_TYPE); + String upperType = cellMeta.getParams().getString(EXT_KEY_RANGE_UPPER_TYPE); + return rangeBoundTypes.get(ImmutableList.of(lowerType, upperType)); + case DATE: + case DATETIME: + return cellMeta.getParams().getString(EXT_KEY_DATETIME_FORMAT); + default: + return StringUtils.EMPTY; + } + } + + private void writeComment(Cell cell, String comment) { + int columnIndex = cell.getColumnIndex(); + int rowIndex = cell.getRowIndex(); + Drawing drawing = cell.getSheet().createDrawingPatriarch(); + Comment cellComment = drawing.createCellComment(new XSSFClientAnchor(0, 0, 0, 0, + // 控制批注的位置 + columnIndex, rowIndex, columnIndex + 2, rowIndex + 2)); + cellComment.setString(new XSSFRichTextString(comment)); + cell.setCellComment(cellComment); + } + + private String buildComment(String patternOpen, String content) { + return patternOpen.concat(content).concat(PATTERN_CLOSE); + } + } + + /** + * Excel添加校验数据 + * XXX: 此处不能继承AbstractCellWriteHandler 会导致 mybatis typehandler 扫描时被加载 + * 导致老项目没有使用 easyexcel2.2 的项目启动失败。 + */ + private static class DataValidationWriteHandler { + private Map cellMetaMap; + + private DataValidationWriteHandler(DataSheetClient.Meta meta) { + this.cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), DataSheetClient.CellMeta::getName); + } + + public void afterCellDispose(Cell cell, boolean isHead) { + // 只在写入表头数据时才写入校验数据 + if (!isHead) { + return; + } + + DataSheetClient.CellMeta cellMeta = cellMetaMap.get(cell.getStringCellValue()); + // 目前仅支持将可选值作为下拉选项的校验数据 + if (cellMeta == null || CollectionUtils.isEmpty(cellMeta.getOptions())) { + return; + } + String[] options = cellMeta.getOptions().stream().map(String::valueOf).toArray(String[]::new); + + Sheet currentSheet = cell.getSheet(); + Workbook workbook = currentSheet.getWorkbook(); + // 直接将校验数据作为列表项的方式有数据大小限制(不超过256个字符), 因此需要将校验数据放在一个新增的隐藏表单中 + // 以表头的名字来命名新增的表单 + String sheetName = cell.getStringCellValue(); + Sheet validationSheet = workbook.createSheet(sheetName); + workbook.setSheetHidden(workbook.getSheetIndex(validationSheet), true); + + // 将可选项放在新增的隐藏表单里 + for (int i = 0; i < options.length; i++) { + Row rowInValidationSheet = validationSheet.createRow(i); + Cell cellInValidationSheet = rowInValidationSheet.createCell(0); + cellInValidationSheet.setCellValue(options[i]); + } + + // 设置列表校验数据的公式 [表名!开始列开始行:结束列结束行] + String listFormula = sheetName + "!$A$1:$A$" + options.length; + DataValidationHelper dataValidationHelper = currentSheet.getDataValidationHelper(); + DataValidationConstraint constraint = dataValidationHelper.createFormulaListConstraint(listFormula); + + // 从下一行开始10000行数据都增加校验数据 + CellRangeAddressList addressList = new CellRangeAddressList(cell.getRowIndex() + 1, + cell.getRowIndex() + 10001, cell.getColumnIndex(), cell.getColumnIndex()); + DataValidation dataValidation = dataValidationHelper.createValidation(constraint, addressList); + dataValidation.setSuppressDropDownArrow(true); + dataValidation.setShowErrorBox(true); + + currentSheet.addValidationData(dataValidation); + } + } + + private class RowColumnCounter { + @Getter + private long rowCount = 0; + @Getter + private long columnCount = 0; + @Getter + private long dataRowCount = 0; + + protected void countTopHints(int topHintsRowCount) { + rowCount += topHintsRowCount; + columnCount = Math.max(columnCount, 1); + } + + protected void countHead(int headColumnCount) { + rowCount += 1; + columnCount = Math.max(columnCount, headColumnCount); + } + + protected void countRow(int columnCount) { + rowCount += 1; + this.columnCount = Math.max(this.columnCount, columnCount); + dataRowCount += 1; + } + } + + private interface Exporter { + void writeTopHints(List topHints); + + void writeHead(List head); + + void writeMultiLineHeads(List> multiLineHeads); + + void writeRow(List row); + + void finish(); + + long getRowCount(); + + long getColumnCount(); + + long getDataRowCount(); + } +} diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetImporter.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetImporter.java new file mode 100644 index 0000000..74db59f --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetImporter.java @@ -0,0 +1,595 @@ +package cn.axzo.foundation.excel.support.impl; + +import cn.axzo.foundation.excel.support.DataSheetClient; +import cn.axzo.foundation.exception.BusinessException; +import cn.axzo.foundation.util.Regex; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.enums.CellExtraTypeEnum; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.excel.metadata.CellExtra; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; +import com.google.common.collect.*; +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*; +import static cn.axzo.foundation.excel.support.DataSheetClient.Meta.*; +import static cn.axzo.foundation.excel.support.DataSheetClient.ResultCode.IMPORT_CELL_CONVERT_FAILED; + + +/** + * 使用ali的EasyExcel来读入excel/csv文件 + * + * @author yuanyi + */ +@Slf4j +public class DataSheetImporter extends DataSheetClient.ImporterBuilder { + + private static final String LINE_KEY_ERRORS = "errors"; + private static final String LINE_KEY_ROW_INDEX = "rowIndex"; + + /** + * 完成后处理事件. + */ + @Setter + private Consumer onCompleted; + + @Override + public DataSheetClient.Importer build() { + Preconditions.checkArgument(this.scene() != null, "scene不能为空"); + Preconditions.checkArgument(this.format() != null, "format不能为空"); + + // TODO: 当支持更多format的时候，需要生成对应的Importer实例 + return ExcelImporter.builder() + .scene(scene()) + .tableName(tableName()) + .meta(meta()) + .ignoreHeaderMeta(ignoreHeaderMeta()) + .debugEnabled(debugEnabled()) + .allowMaxLineCount(allowMaxLineCount()) + .headerConverter(headerConverter()) + .includeLineErrors(includeLineErrors()) + .ignoreUnknownColumn(ignoreUnknownColumn()) + .autoTrim(autoTrim()) + .onCompleted(onCompleted) + .operator(operator()) + .build(); + } + + @Builder + private static class ExcelImporter implements DataSheetClient.Importer { + + private static final Map> rangeBoundTypes = ImmutableMap.of( + RANGE_TYPE_OPEN_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN), + RANGE_TYPE_OPEN_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED), + RANGE_TYPE_CLOSED_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN), + RANGE_TYPE_CLOSED_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED)); + + private String scene; + private String tableName; + private DataSheetClient.Meta meta; + private Boolean ignoreHeaderMeta; + private boolean debugEnabled; + private Integer allowMaxLineCount; + private boolean includeLineErrors; + private boolean ignoreUnknownColumn; + private Boolean autoTrim; + private Function headerConverter; + private Consumer onCompleted; + private String operator; + + @Override + public DataSheetClient.ImportResp readAll(InputStream inputStream) { + Stopwatch stopwatch = Stopwatch.createStarted(); + + NoModelDataListener dataListener = new NoModelDataListener(headerConverter, autoTrim); + EasyExcel.read(inputStream, dataListener) + .autoTrim(autoTrim) + .extraRead(CellExtraTypeEnum.COMMENT).sheet(tableName).doRead(); + if (allowMaxLineCount != null && dataListener.lines.size() > allowMaxLineCount) { + throw DataSheetClient.ResultCode.IMPORT_LINES_REACHED_LIMIT + .toException("导入的数据超过了最大行数" + allowMaxLineCount); + } + + if (BooleanUtils.isNotTrue(this.ignoreHeaderMeta)) { + // 聚合来自入参和文件批注的meta信息；如果都存在，使用入参的meta覆盖文件批注的meta + this.meta = mergeMeta(this.meta, parseMetaFromData(dataListener)); + } + filterHeadMap(dataListener); + filterLines(dataListener); + + validateHeaders(dataListener); + validateMeta(dataListener); + + List headers = parseHeaders(dataListener); + List lines = parseLines(dataListener); + if (!includeLineErrors) { + // 没有设置includeLineErrors时，抛第一个发现的异常 + Optional errorLine = lines.stream() + .filter(line -> line.containsKey(LINE_KEY_ERRORS)) + .findFirst(); + if (errorLine.isPresent()) { + JSONObject error = errorLine.get().getJSONArray(LINE_KEY_ERRORS).getJSONObject(0); + Integer rowIndex = lines.indexOf(errorLine.get()); + Integer columnIndex = error.getInteger("columnIndex"); + + String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1, + error.getString("columnName"), error.getString("errorMsg")); + throw new DataSheetClient.DataSheetException(error.getString("subErrorCode"), errMsg, rowIndex, columnIndex); + } + } + + DataSheetClient.ImportResp resp = DataSheetClient.ImportResp.builder() + .scene(scene) + .templateCode(Optional.ofNullable(meta).map(DataSheetClient.Meta::getTemplateCode).orElse(null)) + .version(Optional.ofNullable(meta).map(DataSheetClient.Meta::getVersion).orElse(null)) + .headers(headers) + .lines(lines) + .headerRowCount(1) + .rowCount(lines.size()) + .columnCount(headers.size()) + .meta(meta) + .elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS)) + .operator(operator) + .build(); + + if (null != onCompleted) { + onCompleted.accept(resp); + } + return resp; + } + + private void filterHeadMap(NoModelDataListener dataListener) { + if (meta == null) { + return; + } + + Map headMap = dataListener.getHeadMap(); + + Set ignoreColumnNames = meta.getIgnoreColumnNames(); + // 收集需要过滤的列号 + Set ignoreColumnIndexes = headMap.entrySet().stream() + .filter(entry -> ignoreColumnNames.contains(entry.getValue())) + .map(Map.Entry::getKey).collect(Collectors.toSet()); + // 加上指定过滤的列号 + ignoreColumnIndexes.addAll(meta.getIgnoreColumnIndexes()); + + ignoreColumnIndexes.forEach(headMap::remove); + } + + private void filterLines(NoModelDataListener dataListener) { + if (meta == null) { + return; + } + Set toRemoveLines = ImmutableSet.copyOf(meta.getIgnoreRowIndexes()); + List> lines = dataListener.getLines(); + dataListener.setLines(IntStream.range(0, lines.size()) + // 这里要加1，是因为removeIndex是整个文档的行数来计算，包括了header + // 但是lines的数据，已经排除了header + .filter(index -> !toRemoveLines.contains(index + 1)) + .mapToObj(lines::get) + .collect(Collectors.toList())); + } + + private DataSheetClient.Meta parseMetaFromData(NoModelDataListener dataListener) { + Map headMap = dataListener.getHeadMap(); + List headComments = dataListener.getCellComments().stream() + // 获取第一行Head的批注信息 + .filter(cellExtra -> cellExtra.getRowIndex() == 0) + // 过滤带有Cell参数配置的批注信息 + .filter(cellExtra -> !Strings.isNullOrEmpty(StringUtils.substringBetween(cellExtra.getText(), + PATTERN_CELL_OPEN, PATTERN_CLOSE))) + // 排序，方便找到第一列，获取templateCode与version + .sorted(Comparator.comparing(CellExtra::getColumnIndex)) + .collect(Collectors.toList()); + // 没有批注信息，直接返回null；会以String来解析值 + if (headComments.isEmpty()) { + return null; + } + + // 从文件中解析meta信息 + JSONObject codeAndVersion = parseTemplateCodeAndVersion(headComments); + List cellMetas = headComments.stream() + .map(cellExtra -> parseCellMeta(cellExtra.getText(), headMap.get(cellExtra.getColumnIndex()))) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (cellMetas.isEmpty()) { + return null; + } + + return DataSheetClient.Meta.builder() + .templateCode(codeAndVersion.getString("templateCode")) + .version(codeAndVersion.getString("version")) + .cellMetas(cellMetas) + .ignoreColumnIndexes(parseIgnoreColumns(dataListener)) + .ignoreRowIndexes(parseIgnoreRows(dataListener)) + .build(); + } + + private DataSheetClient.Meta mergeMeta(DataSheetClient.Meta metaFromParam, DataSheetClient.Meta metaFromHeader) { + if (metaFromHeader == null || metaFromParam == null) { + return metaFromHeader == null ? metaFromParam : metaFromHeader; + } + + // 在两者都存在的时候，以文件的meta为准；仅仅将入参的CellMetas覆盖文件的CellMetas + if (metaFromHeader.getCellMetas() == null) { + metaFromHeader.setCellMetas(metaFromParam.getCellMetas()); + return metaFromHeader; + } + + // 使用入参中定义的列信息，覆盖文件的cellMetas + Map cellMetaMap = + Maps.uniqueIndex(metaFromParam.getCellMetas(), DataSheetClient.CellMeta::getKey); + List cellMetas = metaFromHeader.getCellMetas().stream() + .map(cellMeta -> cellMetaMap.getOrDefault(cellMeta.getKey(), cellMeta)) + .collect(Collectors.toList()); + metaFromHeader.setCellMetas(cellMetas); + return metaFromHeader; + } + + private List parseIgnoreRows(NoModelDataListener dataListener) { + return dataListener.getCellComments().stream() + .map(cellExtra -> { + String ignoreKey = StringUtils.substringBetween(cellExtra.getText(), + PATTERN_IGNORE_OPEN, PATTERN_CLOSE); + if (IGNORE_ROW_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) { + return cellExtra.getRowIndex(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private List parseIgnoreColumns(NoModelDataListener dataListener) { + return dataListener.getCellComments().stream() + .map(cellExtra -> { + String ignoreKey = StringUtils.substringBetween(cellExtra.getText(), + PATTERN_IGNORE_OPEN, PATTERN_CLOSE); + if (IGNORE_COLUMN_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) { + return cellExtra.getColumnIndex(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + public JSONObject parseTemplateCodeAndVersion(List headComments) { + // 从整个头部的批注，获取模版code和版本信息 + Optional parsedText = headComments.stream() + .map(comment -> StringUtils.substringBetween(comment.getText(), PATTERN_VERSION_OPEN, PATTERN_CLOSE)) + .filter(str -> !Strings.isNullOrEmpty(str)) + .findFirst(); + if (!parsedText.isPresent()) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_MISSING_VERSION.toException(); + } + // 格式为"CODE_VERSION" + List values = Splitter.on("_").splitToList(parsedText.get()); + if (values.size() != 2) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_VERSION_FORMAT_ERROR.toException(); + } + return new JSONObject() + .fluentPut("templateCode", values.get(0)) + .fluentPut("version", values.get(1)); + } + + private DataSheetClient.CellMeta parseCellMeta(String text, String name) { + String value = StringUtils.substringBetween(text, PATTERN_CELL_OPEN, PATTERN_CLOSE); + if (Strings.isNullOrEmpty(value)) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_MISSING_TYPE + .toException("没有找到列[{}]的类型信息", name); + } + List values = Splitter.on("_").splitToList(value); + // 格式： key_type_mandatory_params... + if (values.size() < 3) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_FORMAT_ERROR + .toException("列[{}]类型的批注格式不对[{}]", name, text); + } + + DataSheetClient.CellMeta.Type type = DataSheetClient.CellMeta.Type.from(values.get(1)); + JSONObject params = new JSONObject(); + if (type == DataSheetClient.CellMeta.Type.RANGE) { + if (values.size() != 4 || Strings.isNullOrEmpty(values.get(3))) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR + .toException("列[{}]范围类型的批注格式不对[{}]", name, text); + } + List boundType = rangeBoundTypes.get(values.get(3)); + if (boundType == null) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR + .toException(String.format("列[{}]范围类型的值不对", name)); + } + params.put(EXT_KEY_RANGE_LOWER_TYPE, boundType.get(0)); + params.put(EXT_KEY_RANGE_UPPER_TYPE, boundType.get(1)); + } + + JSONObject ext = new JSONObject(); + value = StringUtils.substringBetween(text, PATTERN_EXTRA_OPEN, PATTERN_CLOSE); + if (!Strings.isNullOrEmpty(value)) { + ext.putAll(Splitter.on("&").withKeyValueSeparator("=").split(value)); + } + + return DataSheetClient.CellMeta.builder() + .key(values.get(0)) + .name(name) + .type(type) + .mandatory("1".equals(values.get(2))) + .params(params) + .ext(ext) + .build(); + } + + private List parseHeaders(NoModelDataListener dataListener) { + return dataListener.getHeadMap().keySet().stream() + .sorted() + .map(key -> Strings.nullToEmpty(dataListener.getHeadMap().get(key))) + .collect(Collectors.toList()); + } + + private List parseLines(NoModelDataListener dataListener) { + // 如果没有找到meta信息，按照key=header(很可能是中文), 类型就为String + if (this.meta == null) { + Map headerMap = dataListener.getHeadMap(); + return dataListener.getLines().stream() + .map(line -> new JSONObject().fluentPutAll(line.entrySet().stream() + .map(entry -> Pair.of(headerMap.get(entry.getKey()), Strings.nullToEmpty(entry.getValue()))) + .collect(Collectors.toMap(Pair::getKey, Pair::getValue)))) + .collect(Collectors.toList()); + } + + // 根据meta来校验cell的类型 + Map headerMap = dataListener.getHeadMap(); + Map cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), + DataSheetClient.CellMeta::getName); + List> lines = dataListener.getLines(); + return IntStream.range(0, lines.size()) + .mapToObj(lineIndex -> { + Map line = lines.get(lineIndex); + // 收集每一行每一列的转换结果 + List convertRespList = headerMap.entrySet().stream() + .filter(e -> { + //XXX 当支持忽略列的时候, 只处理定义了cellMeta的数据 + if (ignoreUnknownColumn) { + return cellMetaMap.containsKey(e.getValue()); + } + + return true; + }) + .map(entry -> { + Integer columnIndex = entry.getKey(); + String header = entry.getValue(); + DataSheetClient.CellMeta cellMeta = cellMetaMap.get(header); + return convertType(cellMeta, line.get(columnIndex), lineIndex, columnIndex); + }) + .collect(Collectors.toList()); + + JSONObject row = new JSONObject() + .fluentPutAll(convertRespList.stream() + .filter(ColumnConvertResp::getSuccess) + // convertValue可能为null, 有非必需的字段 + .collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll)); + convertRespList.stream() + .filter(r -> BooleanUtils.isTrue(r.getSuccess())) + .filter(r -> r.getCellMeta().getCellValidator() != null) + .forEach(r -> { + try { + r.getCellMeta().getCellValidator().accept(r.getConvertedValue(), row); + } catch (BusinessException e) { + log.warn("failed to validate cell, resp={}", r, e); + String subErrorCode = null; + if (e instanceof DataSheetClient.DataSheetException) { + subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode(); + } + r.setSuccess(false); + r.setErrorCode(e.getErrorCode()); + r.setErrorMsg(e.getErrorMsg()); + r.setSubErrorCode(subErrorCode); + } + }); + + Map> convertRespMap = convertRespList.stream() + .collect(Collectors.groupingBy(ColumnConvertResp::getSuccess)); + + JSONObject convertedLine = new JSONObject() + .fluentPutAll(convertRespMap.getOrDefault(Boolean.TRUE, ImmutableList.of()).stream() + // convertValue可能为null, 有非必需的字段 + .collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll)); + if (convertRespMap.get(Boolean.FALSE) != null) { + // 转换失败的，将失败信息放到errors字段中 + convertedLine.put(LINE_KEY_ERRORS, convertRespMap.get(Boolean.FALSE).stream() + .map(ColumnConvertResp::getError) + .collect(Collectors.toList())); + } + convertedLine.put(LINE_KEY_ROW_INDEX, lineIndex); + return convertedLine; + }) + .collect(Collectors.toList()); + } + + private ColumnConvertResp convertType(DataSheetClient.CellMeta cellMeta, String rawValue, int rowIndex, int columnIndex) { + try { + return ColumnConvertResp.builder() + .success(true).cellMeta(cellMeta) + .rawValue(rawValue).convertedValue(cellMeta.convertType(rawValue)) + .build(); + } catch (BusinessException e) { + log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}", + cellMeta, rawValue, rowIndex, columnIndex, e); + String subErrorCode = null; + if (e instanceof DataSheetClient.DataSheetException) { + subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode(); + } + return ColumnConvertResp.builder() + .success(false).cellMeta(cellMeta).rawValue(rawValue) + .columnIndex(columnIndex) + .errorCode(e.getErrorCode()) + .errorMsg(e.getErrorMsg()) + .subErrorCode(subErrorCode) + .build(); + } catch (Exception e) { + log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}", + cellMeta, rawValue, rowIndex, columnIndex, e); + String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1, + cellMeta.getName(), Optional.ofNullable(e.getMessage()) + .orElse(IMPORT_CELL_CONVERT_FAILED.getMessage())); + throw new DataSheetClient.DataSheetException(IMPORT_CELL_CONVERT_FAILED.getSubBizCode(), errMsg, rowIndex, columnIndex); + } + } + + private void validateHeaders(NoModelDataListener dataListener) { + Map headMap = dataListener.getHeadMap(); + Set headerNames = ImmutableSet.copyOf(headMap.values()); + if (headerNames.size() != headMap.size()) { + List columnNames = headMap.values().stream() + .collect(Collectors.groupingBy(Function.identity())) + .entrySet().stream() + .filter(grouped -> grouped.getValue().size() > 1) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + throw DataSheetClient.ResultCode.IMPORT_COLUMN_DUPLICATED_NAME.toException(columnNames); + } + } + + private void validateMeta(NoModelDataListener dataListener) { + if (this.meta == null) { + return; + } + + if (meta.getCellMetas().size() != dataListener.getHeadMap().size()) { + if (dataListener.getHeadMap().size() > meta.getCellMetas().size()) { + if (!ignoreUnknownColumn) { + throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException(); + } + } + + long mandatoryColumnSize = meta.getCellMetas().stream() + .filter(cm -> BooleanUtils.isTrue(cm.getMandatory())) + .count(); + if (dataListener.getHeadMap().size() < mandatoryColumnSize) { + throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException(); + } + } + + Set headerNames = ImmutableSet.copyOf(dataListener.getHeadMap().values()); + Set cellNames = meta.getCellMetas().stream() + .map(DataSheetClient.CellMeta::getName) + .collect(Collectors.toSet()); + if (!ignoreUnknownColumn && !headerNames.equals(cellNames)) { + Set missingNames = Sets.difference(cellNames, headerNames); + Set redundantNames = Sets.difference(headerNames, cellNames); + List columnNames = Stream.of(missingNames, redundantNames) + .flatMap(Set::stream) + .collect(Collectors.toList()); + throw DataSheetClient.ResultCode.IMPORT_COLUMN_NAME_NOT_MATCHED.toException(columnNames); + } + meta.getCellMetas().forEach(DataSheetClient.CellMeta::validate); + } + + @Slf4j + @Data + private static class NoModelDataListener extends AnalysisEventListener> { + private Map headMap = Maps.newHashMap(); + private List> lines = Lists.newArrayList(); + private List cellComments = Lists.newArrayList(); + + private Function headerConverter; + private Boolean autoTrim; + + public NoModelDataListener(Function headerConverter, Boolean autoTrim) { + this.headerConverter = headerConverter; + this.autoTrim = autoTrim; + } + + @Override + public void invoke(Map data, AnalysisContext context) { + lines.add(strip(data)); + } + + @Override + public void invokeHeadMap(Map headMap, AnalysisContext context) { + Map convertedMap = headMap; + if (headerConverter != null) { + convertedMap = headMap.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> headerConverter.apply(e.getValue()))); + } + this.headMap.putAll(strip(convertedMap)); + } + + private Map strip(Map data) { + if (BooleanUtils.isFalse(autoTrim)) { + return data; + } + return data.entrySet().stream() + .map(entry -> Maps.immutableEntry(entry.getKey(), StringUtils.strip(entry.getValue(), Regex.WHITESPACE_CHARS))) + // value有可能为null，不能直接用Collectors.toMap + .collect(Maps::newHashMap, (m, v) -> m.put(v.getKey(), v.getValue()), HashMap::putAll); + } + + /** + * extra是在整个文件被解析后才会调用 + * + * @param extra + * @param context + */ + @Override + public void extra(CellExtra extra, AnalysisContext context) { + if (extra.getType() == CellExtraTypeEnum.COMMENT) { + cellComments.add(extra); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + + } + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + private static class ColumnConvertResp { + private Boolean success; + private DataSheetClient.CellMeta cellMeta; + private String rawValue; + private Object convertedValue; + + private Integer columnIndex; + private String errorCode; + private String errorMsg; + private String subErrorCode; + + public String getKey() { + return cellMeta.getKey(); + } + + public JSONObject getError() { + return new JSONObject() + .fluentPut("columnIndex", columnIndex) + .fluentPut("columnKey", cellMeta.getKey()) + .fluentPut("columnName", cellMeta.getName()) + .fluentPut("rawValue", rawValue) + .fluentPut("errorCode", errorCode) + .fluentPut("errorMsg", errorMsg) + .fluentPut("subErrorCode", subErrorCode); + } + } + } +} diff --git a/pom.xml b/pom.xml index 9df7ec1..11260fc 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ gateway-support-lib event-support-lib redis-support-lib + excel-support-lib