diff --git a/pom.xml b/pom.xml index de9f7fe..6d4a7b2 100644 --- a/pom.xml +++ b/pom.xml @@ -20,6 +20,8 @@ 2.0.0-SNAPSHOT 2.0.0-SNAPSHOT 2.0.0-SNAPSHOT + 5.2.2 + 3.3.3 @@ -113,6 +115,18 @@ org.redisson redisson-spring-boot-starter + + org.apache.poi + poi-ooxml + ${poi.version} + + + + + com.alibaba + easyexcel + ${easyexcel.version} + diff --git a/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java b/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java new file mode 100644 index 0000000..16a2c07 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java @@ -0,0 +1,608 @@ +package cn.axzo.pokonyan.client; + +import cn.axzo.pokonyan.exception.BizResultCode; +import cn.axzo.pokonyan.exception.BusinessException; +import cn.axzo.pokonyan.exception.VarParamFormatter; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSONPath; +import com.alibaba.fastjson.util.TypeUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.experimental.Accessors; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; + +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_DATETIME_FORMAT; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_RANGE_LOWER_TYPE; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_RANGE_UPPER_TYPE; + +/** + * 用于发送sms & email验证码, 或者获取图片验证码. 并提供校验验证码是否正确的能力 + */ +public interface DataSheetClient { + + /** + * 数据导入的builder + * + * @return + */ + ImporterBuilder importBuilder(); + + @Data + @NoArgsConstructor + @AllArgsConstructor + @Accessors(fluent = true) + abstract class ImporterBuilder { + /** + * 导入的场景名称, 必填 + */ + @NonNull + private String scene; + + /** + * 导入格式 + */ + @NonNull + private ImportFormat format; + + /** + * EXCEL格式的sheetName + */ + private String tableName; + + private Meta meta; + + private boolean debugEnabled; + + /** + * 解析结果ImportResp中的lines是否包含解析失败的行 + * true: 解析失败的时候,不会抛异常,会在每行的JSONObject中添加 + * "errors":[{"columnIndex":1,"columnKey":"","columnName":"","rawValue":"","errorCode":"","errorMsg":""}] + * false: 解析失败的时候,抛第一个异常 + */ + private boolean includeLineErrors; + /** + * 允许最大的导入行数 + */ + private Integer allowMaxLineCount; + + public abstract Importer build(); + } + + interface Importer { + /** + * 所有字段读取为String类型 + * + * @param inputStream + * @return + */ + ImportResp readAll(InputStream inputStream); + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class ImportResp { + /** + * 导入的场景名称 + */ + private String scene; + + /** + * 模版对应的code以及版本 + */ + private String templateCode; + private String version; + + /** + * headers + */ + private List headers; + /** + * 每一行的数据 + */ + private List lines; + + /** + * header的行数 + */ + private Integer headerRowCount; + /** + * 导入数据总行数, 不包含header + */ + private Integer rowCount; + /** + * 导入数据列数 + */ + private Integer columnCount; + + /** + * Meta数据 + */ + private Meta meta; + + /** + * 消耗的时间 + */ + private Long elapsedMillis; + } + + @AllArgsConstructor + @Getter + enum ImportFormat { + EXCEL("xlsx"), + // TODO: 支持其他类型 + // CSV("csv") + ; + + private String suffix; + } + + @Data + class ExportField { + private String column; + private String header; + private CellMeta.Type type; + private Boolean mandatory; + + private String lowerType; + private String upperType; + private String dateTimeFormat; + private List options; + + private BiFunction reader; + + @Builder + public ExportField(String column, String header, CellMeta.Type type, Boolean mandatory, + Function resultConverter, BiFunction reader, + List options) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(column), "column不能为空"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(header), "header不能为空"); + Preconditions.checkArgument(!(resultConverter != null && reader != null), "reader & resultConverter不能同时存在"); + + + this.column = column; + this.header = header; + this.type = Optional.ofNullable(type).orElse(CellMeta.Type.STRING); + this.mandatory = Optional.ofNullable(mandatory).orElse(Boolean.FALSE); + if (reader != null) { + this.reader = reader; + } else { + this.reader = Optional.ofNullable(resultConverter) + .map(e -> (BiFunction) (row, integer) -> e.apply(Strings.nullToEmpty(row.getString(column)))) + .orElseGet(() -> DataSheetClient.stringCellReader(column)); + } + this.options = options; + } + + public CellMeta toCellMeta() { + JSONObject params = new JSONObject(); + if (CellMeta.Type.RANGE == type) { + params.put(EXT_KEY_RANGE_LOWER_TYPE, lowerType); + params.put(EXT_KEY_RANGE_UPPER_TYPE, upperType); + } + if (CellMeta.Type.DATE == type || CellMeta.Type.DATETIME == type) { + params.put(EXT_KEY_DATETIME_FORMAT, dateTimeFormat); + } + + return CellMeta.builder() + .key(column).name(header).mandatory(mandatory).type(type) + .params(params).options(options).build(); + } + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class Meta { + public static final String PATTERN_VERSION_OPEN = "$V{"; + public static final String PATTERN_CELL_OPEN = "$C{"; + public static final String PATTERN_IGNORE_OPEN = "$I{"; + public static final String PATTERN_EXTRA_OPEN = "$E{"; + public static final String PATTERN_CLOSE = "}"; + + public static final String RANGE_TYPE_OPEN_OPEN = "1"; + public static final String RANGE_TYPE_OPEN_CLOSED = "2"; + public static final String RANGE_TYPE_CLOSED_OPEN = "3"; + public static final String RANGE_TYPE_CLOSED_CLOSED = "4"; + + public static final String IGNORE_ROW_KEY = "r"; + public static final String IGNORE_COLUMN_KEY = "c"; + public static final String IGNORE_ROW_AND_COLUMN_KEY = "b"; + + private String templateCode; + private String version; + private List cellMetas; + /** + * 忽略掉的行号 + */ + private List ignoreRowIndexes; + /** + * 忽略掉的列号 + */ + private List ignoreColumnIndexes; + + public List getIgnoreRowIndexes() { + return Optional.ofNullable(ignoreRowIndexes).orElseGet(ImmutableList::of); + } + + public List getIgnoreColumnIndexes() { + return Optional.ofNullable(ignoreColumnIndexes).orElseGet(ImmutableList::of); + } + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class CellMeta { + public static final String EXT_KEY_DATETIME_FORMAT = "dateTimeFormat"; + public static final String EXT_KEY_RANGE_LOWER_TYPE = "lowerType"; + public static final String EXT_KEY_RANGE_UPPER_TYPE = "upperType"; + + public static final String RANGE_BOUND_TYPE_OPEN = "open"; + public static final String RANGE_BOUND_TYPE_CLOSED = "closed"; + public static final Set RANGE_TYPES = ImmutableSet.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED); + + private static final List DEFAULT_DATE_FORMATS = ImmutableList.of( + "yyyy-MM-dd", + "yyyy-MM-dd HH:mm", + "yyyy-MM-dd HH:mm:ss", + "yyyy.MM.dd", + "yyyy.MM.dd HH:mm", + "yyyy.MM.dd HH:mm:ss", + "yyyy年MM月dd日", + "yyyy年MM月dd日 HH:mm", + "yyyy年MM月dd日 HH:mm:ss", + "yyyy年MM月dd日 HH时mm分", + "yyyy年MM月dd日 HH时mm分ss秒", + "yyyy/MM/dd", + "yyyy/MM/dd HH:mm", + "yyyy/MM/dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss'Z'", + "yyyy-MM-dd'T'HH:mm:ssZ", + "yyyy-MM-dd'T'HH:mm:ssz", + "yyyy-MM-dd'T'HH:mm:ss", + "MM/dd/yyyy HH:mm:ss a", + "yyyyMMddHHmmss", + "yyyyMMdd" + ); + + private static final Map BOOLEAN_MAP = ImmutableMap.builder() + .put("是", Boolean.TRUE) + .put("yes", Boolean.TRUE) + .put("y", Boolean.TRUE) + .put("1", Boolean.TRUE) + .put("否", Boolean.FALSE) + .put("no", Boolean.FALSE) + .put("n", Boolean.FALSE) + .put("0", Boolean.FALSE) + .build(); + + /** + * cell的key + */ + private String key; + /** + * cell的header,一般为中文 + */ + private String name; + /** + * 类型 + */ + private Type type; + /** + * 是否必需 + */ + private Boolean mandatory; + /** + * 可选值 + */ + private List options; + /** + * 存放一些和type相关的参数,例如DateTime的pattern格式,Range的开闭区间 + */ + private JSONObject params; + /** + * 存放一些额外的信息 + */ + private JSONObject ext; + + /** + * 导入的时候,将原始的String转换为特定的类型 + */ + private Function importConverter; + + public void validate() { + if (type == Type.RANGE + && (!RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_LOWER_TYPE)) + || !RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_UPPER_TYPE)))) { + throw ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR.toException(); + } + } + + public Object convertType(String value) { + if (importConverter != null) { + return importConverter.apply(value); + } + + if (Boolean.TRUE.equals(mandatory) && Strings.isNullOrEmpty(value)) { + throw ResultCode.IMPORT_CELL_META_MISSING_MANDATORY_VALUE.toException(); + } + // 非必填的字段如果为空,直接返回 + if (value == null) { + return type == Type.STRING ? StringUtils.EMPTY : null; + } + + switch (type) { + case INT: + return Integer.valueOf(value); + case LONG: + return Long.valueOf(value); + case FLOAT: + return Float.valueOf(value); + case DOUBLE: + return Double.valueOf(value); + case BIG_DECIMAL: + return new BigDecimal(value); + case DATE: + case DATETIME: + return parseDate(value, DEFAULT_DATE_FORMATS); + case RANGE: + // 格式 100-200 + List values = Splitter.on("-").omitEmptyStrings().trimResults().splitToList(value) + .stream() + .map(Integer::valueOf) + .collect(Collectors.toList()); + if (values.size() != 2) { + throw ResultCode.IMPORT_CELL_RANGE_FORMAT_ERROR.toException(); + } + if (values.get(0) >= values.get(1)) { + throw ResultCode.IMPORT_CELL_RANGE_VALUE_ERROR.toException(); + } + return new JSONObject() + .fluentPut("lower", values.get(0)) + .fluentPut("upper", values.get(1)) + .fluentPut(EXT_KEY_RANGE_LOWER_TYPE, params.getString(EXT_KEY_RANGE_LOWER_TYPE)) + .fluentPut(EXT_KEY_RANGE_UPPER_TYPE, params.getString(EXT_KEY_RANGE_UPPER_TYPE)); + case BOOLEAN: + return Optional.ofNullable(BOOLEAN_MAP.get(value.toLowerCase())) + .orElseThrow(ResultCode.IMPORT_CELL_BOOLEAN_VALUE_ERROR::toException); + default: + return Strings.nullToEmpty(value); + } + } + + private LocalDateTime parseDate(String dateStr, List formats) { + try { + Date date = DateUtils.parseDate(dateStr, formats.toArray(new String[0])); + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } catch (Exception e) { + throw ResultCode.IMPORT_CELL_DATETIME_CONVERT_FAILED.toException("不支持的日期格式{}", dateStr); + } + } + + @AllArgsConstructor + public enum Type { + INT("00"), + LONG("01"), + FLOAT("02"), + DOUBLE("03"), + STRING("04"), + BIG_DECIMAL("05"), + DATE("06"), + DATETIME("07"), + RANGE("08"), + BOOLEAN("09") + ; + + @Getter + private String code; + + private static Map map = Stream.of(Type.values()) + .collect(Collectors.toMap(Type::getCode, Function.identity())); + + public static Type from(String code) { + return map.get(code); + } + } + } + + @AllArgsConstructor + @Getter + enum ExportFormat { + EXCEL("application/vnd.ms-excel", ".xlsx"), + CSV("text/csv", ".csv"); + + private String contentType; + private String suffix; + } + + static BiFunction indexCellReader() { + return (row, index) -> String.valueOf(index + 1); + } + + static BiFunction stringCellReader(String columnName) { + return (row, index) -> Strings.nullToEmpty(row.getString(columnName)); + } + + static BiFunction dateTimeCellReader(String columnName, String pattern) { + return (row, index) -> { + Long epochMillis = row.getLong(columnName); + if (epochMillis == null) { + return StringUtils.EMPTY; + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); + Instant instant = Instant.ofEpochMilli(epochMillis); + LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); + return formatter.format(localDateTime); + }; + } + + static BiFunction bigDecimalCellReader(String columnName) { + return bigDecimalCellReader(columnName, 2); + } + + static BiFunction bigDecimalCellReader(String columnName, int scale) { + return (row, index) -> { + BigDecimal number = row.getBigDecimal(columnName); + if (number == null) { + number = BigDecimal.ZERO; + } + + return number.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString(); + }; + } + + static BiFunction jsonPathCellReader(String jsonPath) { + return (row, index) -> JSONPath.eval(row, jsonPath); + } + + static BiFunction jsonPathBigDecimalCellReader(String jsonPath) { + return jsonPathBigDecimalCellReader(jsonPath, 2); + } + + static BiFunction jsonPathBigDecimalCellReader(String jsonPath, int scale) { + return (row, index) -> { + Object val = JSONPath.eval(row, jsonPath); + if (val == null) { + return BigDecimal.ZERO; + } + return TypeUtils.castToBigDecimal(val).stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString(); + }; + } + + @Getter + class DataSheetException extends BusinessException { + private String subErrorCode; + /** 异常相关的行号,列号,0开始 */ + private Integer rowIndex; + private Integer columnIndex; + /** 异常相关的列名称,中文 */ + private List columnNames; + + public DataSheetException(ResultCode resultCode) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + } + + public DataSheetException(ResultCode resultCode, Integer rowIndex, Integer columnIndex) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + this.rowIndex = rowIndex; + this.columnIndex = columnIndex; + } + + public DataSheetException(ResultCode resultCode, List columnNames) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + this.columnNames = columnNames; + } + + public DataSheetException(ResultCode resultCode, String message) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), message); + this.subErrorCode = resultCode.getSubBizCode(); + } + + public DataSheetException(String subErrorCode, String errorMsg, Integer rowIndex, Integer columnIndex) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), errorMsg); + this.subErrorCode = subErrorCode; + this.rowIndex = rowIndex; + this.columnIndex = columnIndex; + } + + public DataSheetException(String errorCode, String subErrorCode, String errorMsg) { + super(errorCode, errorMsg); + this.subErrorCode = subErrorCode; + } + } + + @AllArgsConstructor + @Getter + enum ResultCode { + /** 解析Excel的批注时,相关的errorCode*/ + IMPORT_PARSE_MISSING_VERSION("C00", "批注中没有找到版本信息"), + IMPORT_PARSE_VERSION_FORMAT_ERROR("C01", "批注中的版本格式不对"), + IMPORT_PARSE_CELL_META_MISSING_TYPE("C02", "批注中的字段没有找到类型信息"), + IMPORT_PARSE_CELL_META_FORMAT_ERROR("C03", "批注中的字段格式不对"), + IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR("C04", "批注中的范围类型字段的格式不对"), + IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR("C05", "批注中的范围类型字段的区间类型不对"), + + /** 导入数据时,相关的errorCode*/ + IMPORT_LINES_REACHED_LIMIT("C20", "导入的数据超过了最大行数"), + IMPORT_COLUMN_RANGE_MISSING_TYPES("C21", "范围类型缺少区间类型的定义:开区间还是闭区间"), + IMPORT_COLUMN_DUPLICATED_NAME("C22", "列的名称不能重复"), + IMPORT_COLUMN_MISSING_CELL_META("C23", "字段缺少类型定义"), + IMPORT_COLUMN_NAME_NOT_MATCHED("C24", "字段的名称与类型定义的名称不一致"), + IMPORT_CELL_RANGE_FORMAT_ERROR("C25", "范围类型的格式不对"), + IMPORT_CELL_RANGE_VALUE_ERROR("C26", "范围类型的下限值必须小于上限值"), + IMPORT_CELL_DATETIME_CONVERT_FAILED("C27", "时间类型解析失败"), + IMPORT_CELL_CONVERT_FAILED("C28", "类型转换失败"), + IMPORT_CELL_META_MISSING_MANDATORY_VALUE("C29", "必填字段不能为空"), + IMPORT_CELL_BOOLEAN_VALUE_ERROR("C30", "布尔类型的值不支持"), + ; + + private String subBizCode; + private String message; + + public DataSheetException toException() { + return new DataSheetException(this); + } + + public DataSheetException toException(String message) { + return new DataSheetException(this, message); + } + + public DataSheetException toException(List columnNames) { + return new DataSheetException(this, columnNames); + } + + public DataSheetException toException(Integer rowIndex, Integer columnIndex) { + return new DataSheetException(this, rowIndex, columnIndex); + } + + public DataSheetException toException(String customMsg, Object... objects) { + if (objects == null) { + return toException(customMsg); + } + + String msg = VarParamFormatter.format(customMsg, objects); + //如果最后一个参数是Throwable. 则将SimpleName附加到msg中 + if (objects[objects.length - 1] instanceof Throwable) { + Throwable throwable = (Throwable) objects[objects.length - 1]; + msg = String.format("%s (%s)", msg, throwable.getClass().getSimpleName()); + } + + return toException(msg); + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetClientImpl.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetClientImpl.java new file mode 100644 index 0000000..f189e12 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetClientImpl.java @@ -0,0 +1,54 @@ +package cn.axzo.pokonyan.client.impl; + +import cn.axzo.pokonyan.client.DataSheetClient; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; + +@Slf4j +public class DataSheetClientImpl implements DataSheetClient { + + @Override + public ImporterBuilder importBuilder() { + return new DataSheetImporter(); + } + + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + private static class ReportReq { + String appName; + String scene; + ReportReq.Action action; + Map resp; + Long rowCount; + Long elapsedMillis; + String filePath; + String operatorId; + String operatorName; + String operatorTenantId; + + public enum Action { + IMPORT, + EXPORT; + } + } + + + public static Builder builder() { + return new Builder(); + } + + @Data + public static class Builder { + + public DataSheetClient build() { + return new DataSheetClientImpl(); + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java new file mode 100644 index 0000000..93d0f1d --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java @@ -0,0 +1,536 @@ +package cn.axzo.pokonyan.client.impl; + +import cn.axzo.pokonyan.client.DataSheetClient; +import cn.axzo.pokonyan.exception.BusinessException; +import cn.axzo.pokonyan.util.Regex; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.enums.CellExtraTypeEnum; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.excel.metadata.CellExtra; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.io.InputStream; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_RANGE_LOWER_TYPE; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_RANGE_UPPER_TYPE; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.RANGE_BOUND_TYPE_CLOSED; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.RANGE_BOUND_TYPE_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.IGNORE_COLUMN_KEY; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.IGNORE_ROW_AND_COLUMN_KEY; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.IGNORE_ROW_KEY; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_CELL_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_CLOSE; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_EXTRA_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_IGNORE_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_VERSION_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.RANGE_TYPE_CLOSED_CLOSED; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.RANGE_TYPE_CLOSED_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.RANGE_TYPE_OPEN_CLOSED; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.RANGE_TYPE_OPEN_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.ResultCode.IMPORT_CELL_CONVERT_FAILED; + +/** + * 使用ali的EasyExcel来读入excel/csv文件 + * + * @author yuanyi + */ +@Slf4j +public class DataSheetImporter extends DataSheetClient.ImporterBuilder { + + private static final String LINE_KEY_ERRORS = "errors"; + private static final String LINE_KEY_ROW_INDEX = "rowIndex"; + + /** + * 完成后处理事件. + */ + @Setter + private Consumer onCompleted; + + @Override + public DataSheetClient.Importer build() { + Preconditions.checkArgument(this.scene() != null, "scene不能为空"); + Preconditions.checkArgument(this.format() != null, "format不能为空"); + + // TODO: 当支持更多format的时候,需要生成对应的Importer实例 + return ExcelImporter.builder() + .scene(scene()) + .tableName(tableName()) + .meta(meta()) + .debugEnabled(debugEnabled()) + .allowMaxLineCount(allowMaxLineCount()) + .includeLineErrors(includeLineErrors()) + .onCompleted(onCompleted) + .build(); + } + + @Builder + private static class ExcelImporter implements DataSheetClient.Importer { + + private static final Map> rangeBoundTypes = ImmutableMap.of( + RANGE_TYPE_OPEN_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN), + RANGE_TYPE_OPEN_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED), + RANGE_TYPE_CLOSED_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN), + RANGE_TYPE_CLOSED_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED)); + + private String scene; + private String tableName; + private DataSheetClient.Meta meta; + private boolean debugEnabled; + private Integer allowMaxLineCount; + private boolean includeLineErrors; + private Consumer onCompleted; + + @Override + public DataSheetClient.ImportResp readAll(InputStream inputStream) { + Stopwatch stopwatch = Stopwatch.createStarted(); + + NoModelDataListener dataListener = new NoModelDataListener(); + EasyExcel.read(inputStream, dataListener).extraRead(CellExtraTypeEnum.COMMENT).sheet(tableName).doRead(); + if (allowMaxLineCount != null && dataListener.lines.size() > allowMaxLineCount) { + throw DataSheetClient.ResultCode.IMPORT_LINES_REACHED_LIMIT + .toException("导入的数据超过了最大行数" + allowMaxLineCount); + } + + // 聚合来自入参和文件批注的meta信息;如果都存在,使用入参的meta覆盖文件批注的meta + this.meta = mergeMeta(this.meta, parseMetaFromData(dataListener)); + filterHeadMap(dataListener); + filterLines(dataListener); + + validateHeaders(dataListener); + validateMeta(dataListener); + + List headers = parseHeaders(dataListener); + List lines = parseLines(dataListener); + if (!includeLineErrors) { + // 没有设置includeLineErrors时,抛第一个发现的异常 + Optional errorLine = lines.stream() + .filter(line -> line.containsKey(LINE_KEY_ERRORS)) + .findFirst(); + if (errorLine.isPresent()) { + JSONObject error = errorLine.get().getJSONArray(LINE_KEY_ERRORS).getJSONObject(0); + Integer rowIndex = lines.indexOf(errorLine.get()); + Integer columnIndex = error.getInteger("columnIndex"); + + String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1, + error.getString("columnName"), error.getString("errorMsg")); + throw new DataSheetClient.DataSheetException(error.getString("subErrorCode"), errMsg, rowIndex, columnIndex); + } + } + + DataSheetClient.ImportResp resp = DataSheetClient.ImportResp.builder() + .scene(scene) + .templateCode(Optional.ofNullable(meta).map(DataSheetClient.Meta::getTemplateCode).orElse(null)) + .version(Optional.ofNullable(meta).map(DataSheetClient.Meta::getVersion).orElse(null)) + .headers(headers) + .lines(lines) + .headerRowCount(1) + .rowCount(lines.size()) + .columnCount(headers.size()) + .meta(meta) + .elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS)) + .build(); + + if (null != onCompleted) { + onCompleted.accept(resp); + } + return resp; + } + + private void filterHeadMap(NoModelDataListener dataListener) { + if (meta == null) { + return; + } + + Map headMap = dataListener.getHeadMap(); + meta.getIgnoreColumnIndexes().forEach(headMap::remove); + } + + private void filterLines(NoModelDataListener dataListener) { + if (meta == null) { + return; + } + Set toRemoveLines = ImmutableSet.copyOf(meta.getIgnoreRowIndexes()); + List> lines = dataListener.getLines(); + dataListener.setLines(IntStream.range(0, lines.size()) + // 这里要加1,是因为removeIndex是整个文档的行数来计算,包括了header + // 但是lines的数据,已经排除了header + .filter(index -> !toRemoveLines.contains(index + 1)) + .mapToObj(lines::get) + .collect(Collectors.toList())); + } + + private DataSheetClient.Meta parseMetaFromData(NoModelDataListener dataListener) { + Map headMap = dataListener.getHeadMap(); + List headComments = dataListener.getCellComments().stream() + // 获取第一行Head的批注信息 + .filter(cellExtra -> cellExtra.getRowIndex() == 0) + // 过滤带有Cell参数配置的批注信息 + .filter(cellExtra -> !Strings.isNullOrEmpty(StringUtils.substringBetween(cellExtra.getText(), + PATTERN_CELL_OPEN, PATTERN_CLOSE))) + // 排序,方便找到第一列,获取templateCode与version + .sorted(Comparator.comparing(CellExtra::getColumnIndex)) + .collect(Collectors.toList()); + // 没有批注信息,直接返回null;会以String来解析值 + if (headComments.isEmpty()) { + return null; + } + + // 从文件中解析meta信息 + JSONObject codeAndVersion = parseTemplateCodeAndVersion(headComments); + List cellMetas = headComments.stream() + .map(cellExtra -> parseCellMeta(cellExtra.getText(), headMap.get(cellExtra.getColumnIndex()))) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (cellMetas.isEmpty()) { + return null; + } + + return DataSheetClient.Meta.builder() + .templateCode(codeAndVersion.getString("templateCode")) + .version(codeAndVersion.getString("version")) + .cellMetas(cellMetas) + .ignoreColumnIndexes(parseIgnoreColumns(dataListener)) + .ignoreRowIndexes(parseIgnoreRows(dataListener)) + .build(); + } + + private DataSheetClient.Meta mergeMeta(DataSheetClient.Meta metaFromParam, DataSheetClient.Meta metaFromHeader) { + if (metaFromHeader == null || metaFromParam == null) { + return metaFromHeader == null ? metaFromParam : metaFromHeader; + } + + // 在两者都存在的时候,以文件的meta为准;仅仅将入参的CellMetas覆盖文件的CellMetas + if (metaFromHeader.getCellMetas() == null) { + metaFromHeader.setCellMetas(metaFromParam.getCellMetas()); + return metaFromHeader; + } + + // 使用入参中定义的列信息,覆盖文件的cellMetas + Map cellMetaMap = + Maps.uniqueIndex(metaFromParam.getCellMetas(), DataSheetClient.CellMeta::getKey); + List cellMetas = metaFromHeader.getCellMetas().stream() + .map(cellMeta -> cellMetaMap.getOrDefault(cellMeta.getKey(), cellMeta)) + .collect(Collectors.toList()); + metaFromHeader.setCellMetas(cellMetas); + return metaFromHeader; + } + + private List parseIgnoreRows(NoModelDataListener dataListener) { + return dataListener.getCellComments().stream() + .map(cellExtra -> { + String ignoreKey = StringUtils.substringBetween(cellExtra.getText(), + PATTERN_IGNORE_OPEN, PATTERN_CLOSE); + if (IGNORE_ROW_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) { + return cellExtra.getRowIndex(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private List parseIgnoreColumns(NoModelDataListener dataListener) { + return dataListener.getCellComments().stream() + .map(cellExtra -> { + String ignoreKey = StringUtils.substringBetween(cellExtra.getText(), + PATTERN_IGNORE_OPEN, PATTERN_CLOSE); + if (IGNORE_COLUMN_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) { + return cellExtra.getColumnIndex(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + public JSONObject parseTemplateCodeAndVersion(List headComments) { + // 从整个头部的批注,获取模版code和版本信息 + Optional parsedText = headComments.stream() + .map(comment -> StringUtils.substringBetween(comment.getText(), PATTERN_VERSION_OPEN, PATTERN_CLOSE)) + .filter(str -> !Strings.isNullOrEmpty(str)) + .findFirst(); + if (!parsedText.isPresent()) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_MISSING_VERSION.toException(); + } + // 格式为"CODE_VERSION" + List values = Splitter.on("_").splitToList(parsedText.get()); + if (values.size() != 2) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_VERSION_FORMAT_ERROR.toException(); + } + return new JSONObject() + .fluentPut("templateCode", values.get(0)) + .fluentPut("version", values.get(1)); + } + + private DataSheetClient.CellMeta parseCellMeta(String text, String name) { + String value = StringUtils.substringBetween(text, PATTERN_CELL_OPEN, PATTERN_CLOSE); + if (Strings.isNullOrEmpty(value)) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_MISSING_TYPE + .toException("没有找到列[{}]的类型信息", name); + } + List values = Splitter.on("_").splitToList(value); + // 格式: key_type_mandatory_params... + if (values.size() < 3) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_FORMAT_ERROR + .toException("列[{}]类型的批注格式不对[{}]", name, text); + } + + DataSheetClient.CellMeta.Type type = DataSheetClient.CellMeta.Type.from(values.get(1)); + JSONObject params = new JSONObject(); + if (type == DataSheetClient.CellMeta.Type.RANGE) { + if (values.size() != 4 || Strings.isNullOrEmpty(values.get(3))) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR + .toException("列[{}]范围类型的批注格式不对[{}]", name, text); + } + List boundType = rangeBoundTypes.get(values.get(3)); + if (boundType == null) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR + .toException(String.format("列[{}]范围类型的值不对", name)); + } + params.put(EXT_KEY_RANGE_LOWER_TYPE, boundType.get(0)); + params.put(EXT_KEY_RANGE_UPPER_TYPE, boundType.get(1)); + } + + JSONObject ext = new JSONObject(); + value = StringUtils.substringBetween(text, PATTERN_EXTRA_OPEN, PATTERN_CLOSE); + if (!Strings.isNullOrEmpty(value)) { + ext.putAll(Splitter.on("&").withKeyValueSeparator("=").split(value)); + } + + return DataSheetClient.CellMeta.builder() + .key(values.get(0)) + .name(name) + .type(type) + .mandatory("1".equals(values.get(2))) + .params(params) + .ext(ext) + .build(); + } + + private List parseHeaders(NoModelDataListener dataListener) { + return dataListener.getHeadMap().keySet().stream() + .sorted() + .map(key -> Strings.nullToEmpty(dataListener.getHeadMap().get(key))) + .collect(Collectors.toList()); + } + + private List parseLines(NoModelDataListener dataListener) { + // 如果没有找到meta信息,按照key=header(很可能是中文), 类型就为String + if (this.meta == null) { + Map headerMap = dataListener.getHeadMap(); + return dataListener.getLines().stream() + .map(line -> new JSONObject().fluentPutAll(line.entrySet().stream() + .map(entry -> Pair.of(headerMap.get(entry.getKey()), Strings.nullToEmpty(entry.getValue()))) + .collect(Collectors.toMap(Pair::getKey, Pair::getValue)))) + .collect(Collectors.toList()); + } + + // 根据meta来校验cell的类型 + Map headerMap = dataListener.getHeadMap(); + Map cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), + DataSheetClient.CellMeta::getName); + List> lines = dataListener.getLines(); + return IntStream.range(0, lines.size()) + .mapToObj(lineIndex -> { + Map line = lines.get(lineIndex); + // 收集每一行每一列的转换结果 + Map> convertRespMap = headerMap.entrySet().stream() + .map(entry -> { + Integer columnIndex = entry.getKey(); + String header = entry.getValue(); + DataSheetClient.CellMeta cellMeta = cellMetaMap.get(header); + return convertType(cellMeta, line.get(columnIndex), lineIndex, columnIndex); + }) + .collect(Collectors.groupingBy(ColumnConvertResp::getSuccess)); + + JSONObject convertedLine = new JSONObject() + .fluentPutAll(convertRespMap.getOrDefault(Boolean.TRUE, ImmutableList.of()).stream() + // convertValue可能为null, 有非必需的字段 + .collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll)); + if (convertRespMap.get(Boolean.FALSE) != null) { + // 转换失败的,将失败信息放到errors字段中 + convertedLine.put(LINE_KEY_ERRORS, convertRespMap.get(Boolean.FALSE).stream() + .map(ColumnConvertResp::getError) + .collect(Collectors.toList())); + } + convertedLine.put(LINE_KEY_ROW_INDEX, lineIndex); + return convertedLine; + }) + .collect(Collectors.toList()); + } + + private ColumnConvertResp convertType(DataSheetClient.CellMeta cellMeta, String rawValue, int rowIndex, int columnIndex) { + try { + return ColumnConvertResp.builder() + .success(true).cellMeta(cellMeta) + .rawValue(rawValue).convertedValue(cellMeta.convertType(rawValue)) + .build(); + } catch (BusinessException e) { + log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}", + cellMeta, rawValue, rowIndex, columnIndex, e); + String subErrorCode = null; + if (e instanceof DataSheetClient.DataSheetException) { + subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode(); + } + return ColumnConvertResp.builder() + .success(false).cellMeta(cellMeta).rawValue(rawValue) + .columnIndex(columnIndex) + .errorCode(e.getErrorCode()) + .errorMsg(e.getErrorMsg()) + .subErrorCode(subErrorCode) + .build(); + } catch (Exception e) { + log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}", + cellMeta, rawValue, rowIndex, columnIndex, e); + String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1, + cellMeta.getName(), Optional.ofNullable(e.getMessage()) + .orElse(IMPORT_CELL_CONVERT_FAILED.getMessage())); + throw new DataSheetClient.DataSheetException(IMPORT_CELL_CONVERT_FAILED.getSubBizCode(), errMsg, rowIndex, columnIndex); + } + } + + private void validateHeaders(NoModelDataListener dataListener) { + Map headMap = dataListener.getHeadMap(); + Set headerNames = ImmutableSet.copyOf(headMap.values()); + if (headerNames.size() != headMap.size()) { + List columnNames = headMap.values().stream() + .collect(Collectors.groupingBy(Function.identity())) + .entrySet().stream() + .filter(grouped -> grouped.getValue().size() > 1) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + throw DataSheetClient.ResultCode.IMPORT_COLUMN_DUPLICATED_NAME.toException(columnNames); + } + } + + private void validateMeta(NoModelDataListener dataListener) { + if (this.meta == null) { + return; + } + + if (meta.getCellMetas().size() != dataListener.getHeadMap().size()) { + throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException(); + } + + Set headerNames = ImmutableSet.copyOf(dataListener.getHeadMap().values()); + Set cellNames = meta.getCellMetas().stream() + .map(DataSheetClient.CellMeta::getName) + .collect(Collectors.toSet()); + if (!headerNames.equals(cellNames)) { + Set missingNames = Sets.difference(cellNames, headerNames); + Set redundantNames = Sets.difference(headerNames, cellNames); + List columnNames = Stream.of(missingNames, redundantNames) + .flatMap(Set::stream) + .collect(Collectors.toList()); + throw DataSheetClient.ResultCode.IMPORT_COLUMN_NAME_NOT_MATCHED.toException(columnNames); + } + meta.getCellMetas().forEach(DataSheetClient.CellMeta::validate); + } + + @Slf4j + @Data + private static class NoModelDataListener extends AnalysisEventListener> { + private Map headMap = Maps.newHashMap(); + private List> lines = Lists.newArrayList(); + private List cellComments = Lists.newArrayList(); + + @Override + public void invoke(Map data, AnalysisContext context) { + lines.add(strip(data)); + } + + @Override + public void invokeHeadMap(Map headMap, AnalysisContext context) { + this.headMap.putAll(strip(headMap)); + } + + private Map strip(Map data) { + return data.entrySet().stream() + .map(entry -> Maps.immutableEntry(entry.getKey(), StringUtils.strip(entry.getValue(), Regex.WHITESPACE_CHARS))) + // value有可能为null,不能直接用Collectors.toMap + .collect(Maps::newHashMap, (m, v) -> m.put(v.getKey(), v.getValue()), HashMap::putAll); + } + + /** + * extra是在整个文件被解析后才会调用 + * + * @param extra + * @param context + */ + @Override + public void extra(CellExtra extra, AnalysisContext context) { + if (extra.getType() == CellExtraTypeEnum.COMMENT) { + cellComments.add(extra); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + + } + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + private static class ColumnConvertResp { + private Boolean success; + private DataSheetClient.CellMeta cellMeta; + private String rawValue; + private Object convertedValue; + + private Integer columnIndex; + private String errorCode; + private String errorMsg; + private String subErrorCode; + + public String getKey() { + return cellMeta.getKey(); + } + + public JSONObject getError() { + return new JSONObject() + .fluentPut("columnIndex", columnIndex) + .fluentPut("columnKey", cellMeta.getKey()) + .fluentPut("columnName", cellMeta.getName()) + .fluentPut("rawValue", rawValue) + .fluentPut("errorCode", errorCode) + .fluentPut("errorMsg", errorMsg) + .fluentPut("subErrorCode", subErrorCode); + } + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/exception/BizResultCode.java b/src/main/java/cn/axzo/pokonyan/exception/BizResultCode.java new file mode 100644 index 0000000..c313ba4 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/exception/BizResultCode.java @@ -0,0 +1,16 @@ +package cn.axzo.pokonyan.exception; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum BizResultCode implements ResultCode { + + SYSTEM_PARAM_NOT_VALID_EXCEPTION("001", "参与异常"),; + + + private String errorCode; + private String errorMessage; + +} diff --git a/src/main/java/cn/axzo/pokonyan/util/Regex.java b/src/main/java/cn/axzo/pokonyan/util/Regex.java new file mode 100644 index 0000000..3ccd516 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/util/Regex.java @@ -0,0 +1,40 @@ +package cn.axzo.pokonyan.util; + +import org.apache.commons.lang3.StringUtils; + +import java.util.regex.Pattern; + +/** + * 重用的数据格式验证正则表达式 + */ +public class Regex { + + public final static String MOBILE_REGEX = "^1\\d{10}$"; + public final static String MOBILE_REGEX_MESSAGE = "手机号格式不正确"; + public final static Pattern MOBILE_PATTERN = Pattern.compile(MOBILE_REGEX); + + public static final String ID_NO_REGEX = "^(\\d{15}$|^\\d{18}$|^\\d{17}(\\d|X|x))$"; + public final static String ID_NO_REGEX_MESSAGE = "身份证格式不正确"; + public final static Pattern ID_NO_PATTERN = Pattern.compile(ID_NO_REGEX); + + public static final String COMPANY_LICENSE_NO_REGEX = "^(\\w{15}|\\w{18})$"; + public final static String COMPANY_LICENSE_NO_REGEX_MESSAGE = "营业执照号格式不正确"; + public final static Pattern COMPANY_LICENSE_NO_PATTERN = Pattern.compile(COMPANY_LICENSE_NO_REGEX); + + public static final String TEL_REGEX = "^((0\\d{2,3})-?)(\\d{7,8})?$"; + public final static String TEL_REGEX_MESSAGE = "固定电话格式不正确"; + public final static Pattern TEL_PATTERN = Pattern.compile(TEL_REGEX); + + public static final String EMAIL_REGEX = "^([-|\\w])+(\\.[-|\\w]+)*@(\\w)+((\\.\\w+)+)$"; + public final static String EMAIL_REGEX_MESSAGE = "邮件格式不正确"; + public final static Pattern EMAIL_PATTERN = Pattern.compile(EMAIL_REGEX); + + public static final String HTTP_REGEX = "^(http|https)://([\\w.]+/?)\\S*$"; + + public static final String REAL_NAME_REGEX = "^[\\u4E00-\\u9FA5]{2,32}$|^$"; + public static final String REAL_NAME_REGEX_MESSAGE = "姓名格式不正确"; + public final static Pattern REAL_NAME_PATTERN = Pattern.compile(REAL_NAME_REGEX); + + /**常用的制表符*/ + public static final String WHITESPACE_CHARS = "\r\n\0\t\b" + StringUtils.SPACE; +}