wangli/axzo-lib-box
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'test'

Browse Source
This commit is contained in:
zengxiaobo 2024-07-17 18:05:29 +08:00
commit ded88b3a60
70 changed files with 5379 additions and 695 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -48,3 +48,5 @@ build/
/web-support-lib/.flattened-pom.xml
/event-support-lib/.flattened-pom.xml
/gateway-support-lib/.flattened-pom.xml
/excel-support-lib/.flattened-pom.xml
/redis-support-lib/.flattened-pom.xml

1
.reviewboardrc Normal file
View File

@ -0,0 +1 @@
REPOSITORY = 'axzo-lib-box'

18
common-lib/src/main/java/cn/axzo/foundation/page/PageReq.java Normal file
View File

@ -0,0 +1,18 @@
package cn.axzo.foundation.page;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class PageReq implements IPageReq{
Integer page;
Integer pageSize;
List<String> sort;
}

17
common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java
View File

@ -25,4 +25,21 @@ public class PageResp<T> {
public List<T> getData() {
return Optional.ofNullable(data).orElse(ImmutableList.of());
}
public boolean hasNext() {
return this.current < this.getPages();
}
public long getPages() {
if (this.getSize() == 0L) {
return 0L;
}
long pages = this.getTotal() / this.getSize();
if (this.getTotal() % this.getSize() != 0L) {
++pages;
}
return pages;
}
}

27
common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java
View File

@ -1,10 +1,12 @@
package cn.axzo.foundation.result;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.base.Strings;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.commons.lang3.StringUtils;
@Data
@Builder
@ -12,13 +14,13 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
public class ApiResult<T> {
public static final Integer SUCCESS_HTTP_CODE = 200;
private static final String SUCCESS_CODE = "200";
private static final Integer SUCCESS_CODE = 200;
public static final Integer FAILED_HTTP_CODE = 400;
private static final String FAILED_CODE = "400";
protected Integer httpCode;
protected String code;
protected Integer code;
protected String msg;
@ -59,7 +61,7 @@ public class ApiResult<T> {
public static ApiResult error(Integer httpCode, String errorCode, String errorMsg) {
return ApiResult.builder()
.httpCode(httpCode)
.code(errorCode)
.code(Integer.parseInt(errorCode))
.msg(errorMsg)
.build();
}
@ -68,4 +70,23 @@ public class ApiResult<T> {
public boolean isSuccess() {
return SUCCESS_CODE.equals(getCode());
}
/**
* 根据appId 获取标准的code
* 如果code > 100000 则认为可能已经带了appId
* 否则拼接当前的appId 到appCode中
*/
public static Integer getStandardCode(String appId, Integer code) {
if (code == null || Strings.isNullOrEmpty(appId) || SUCCESS_CODE.equals(code)) {
return code;
}
if (code >= 1000000) {
return code;
}
try {
return Integer.parseInt(StringUtils.right(StringUtils.getDigits(appId), 4) + StringUtils.leftPad(code + "", 3, "0"));
} catch (Exception ex) {
return code;
}
}
}

4
common-lib/src/main/java/cn/axzo/foundation/result/ResultCode.java
View File

@ -11,7 +11,9 @@ public enum ResultCode implements IResultCode {
INVALID_PARAMS("002", "请求参数格式错误", 400),
NETWORK_FAILURE("003", "内部网络错误", 500),
PROCESS_TIMEOUT("004", "内部处理超时", 500),
APP_CONFIG_ERROR("005", "服务配置错误", 500);
APP_CONFIG_ERROR("005", "服务配置错误", 500),
OPERATE_TOO_FREQUENTLY("006", "操作过于频繁", 500),
;
final private String code;

2
common-lib/src/main/java/cn/axzo/foundation/util/TraceUtils.java
View File

@ -6,7 +6,7 @@ import org.slf4j.MDC;
@UtilityClass
public class TraceUtils {
public static final String TRACE_ID = "axzo-trace-id";
public static final String TRACE_ID = "traceId";
/**
* 多设置一个key = TraceId, value为traceId的变量到MDC. 以兼容目前的logback-spring.xml的配置
*/

4
event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/DefaultRocketMQListener.java
View File

@ -3,6 +3,7 @@ package cn.axzo.foundation.event.support.consumer;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import lombok.Builder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExt;
@ -14,7 +15,8 @@ import java.util.Optional;
@Slf4j
public class DefaultRocketMQListener implements RocketMQListener<MessageExt> {
EventConsumer eventConsumer;
@Getter
protected EventConsumer eventConsumer;
@Builder
public DefaultRocketMQListener(EventConsumer eventConsumer) {

34
event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/EventHandlerRepository.java
View File

@ -6,7 +6,9 @@ import cn.axzo.foundation.util.FastjsonUtils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import com.google.common.base.*;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Maps;
@ -16,11 +18,11 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.util.AntPathMatcher;
import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.Optional;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.stream.Collectors;
/**
@ -29,8 +31,7 @@ import java.util.stream.Collectors;
*/
@Slf4j
public class EventHandlerRepository {
final protected ListMultimap<Event.EventCode, EventHandlerContext> handlers = ArrayListMultimap.create();
private final Consumer<EventHandledWrapper> DEFAULT_EXCEPTION_HANDLER = EventHandledWrapper::doPrintException;
protected final ListMultimap<Event.EventCode, EventHandlerContext> handlers = ArrayListMultimap.create();
private AntPathMatcher antPathMatcher;
/**
@ -41,18 +42,22 @@ public class EventHandlerRepository {
private final Boolean logEnabled;
private final Predicate<Event> logFilter;
private Consumer<EventHandledWrapper> globalExceptionHandler;
@Builder
public EventHandlerRepository(boolean supportPattern,
Boolean logEnabled,
Predicate<Event> logFilter,
Long logElapsedThreshold,
Long maxAllowElapsedMillis) {
Long maxAllowElapsedMillis,
Consumer<EventHandledWrapper> globalExceptionHandler) {
this.logEnabled = Optional.ofNullable(logEnabled).orElse(Boolean.TRUE);
this.logFilter = logFilter;
this.logElapsedThreshold = Optional.ofNullable(logElapsedThreshold).orElse(3L);
this.maxAllowElapsedMillis = Optional.ofNullable(maxAllowElapsedMillis).orElse(10_000L);
this.globalExceptionHandler = Optional.ofNullable(globalExceptionHandler).orElse(EventHandledWrapper::doPrintException);
if (supportPattern) {
antPathMatcher = new AntPathMatcher(Event.EventCode.SEPARATOR);
antPathMatcher.setCachePatterns(true);
@ -65,7 +70,7 @@ public class EventHandlerRepository {
//如果传入了key则检查部重复
if (StringUtils.isNoneBlank(handlerHolder.getHandlerKey())) {
Set<String> existsHandlerKeys = handlers.values().stream()
.map(e -> e.getHandlerKey())
.map(EventHandlerContext::getHandlerKey)
.filter(StringUtils::isNoneBlank)
.collect(Collectors.toSet());
Preconditions.checkArgument(!existsHandlerKeys.contains(handlerHolder.getHandlerKey()));
@ -100,7 +105,7 @@ public class EventHandlerRepository {
&& context.getHeaders().containsKey(DiffablePayload.DIFF_META_HEADER)) {
byte[] diffMetaHeader = context.getHeaders().get(DiffablePayload.DIFF_META_HEADER);
List<DiffablePayload.DiffMeta> diffMetas = JSON.parseObject(
new String(diffMetaHeader, Charsets.UTF_8), new TypeReference<List<DiffablePayload.DiffMeta>>() {
new String(diffMetaHeader, StandardCharsets.UTF_8), new TypeReference<List<DiffablePayload.DiffMeta>>() {
});
differentiator = DiffablePayload.DiffMeta.toDifferentiator(diffMetas);
}
@ -135,7 +140,7 @@ public class EventHandlerRepository {
}
if (elapsed > maxAllowElapsedMillis) {
String msg = String.format("[%s] take too long %d millis for %s to handle %s\nevent=%s",
String msg = String.format("[%s] take too long %d millis for %s to handle %s event=%s",
context.getTraceId(), elapsed, handler.getName(), payloadDiffLog, eventLogText);
log.warn(msg);
}
@ -163,7 +168,7 @@ public class EventHandlerRepository {
public boolean batch(List<Event> events, EventConsumer.Context context) {
Stopwatch stopwatch = Stopwatch.createUnstarted();
List<EventHandlerContext> eventHandlers = getEventHandlers(context.getEventCode(), context);
eventHandlers.stream().forEach(handler -> {
eventHandlers.forEach(handler -> {
try {
stopwatch.start();
String clazzName = handler.getClass().getCanonicalName();
@ -210,10 +215,11 @@ public class EventHandlerRepository {
.handlerKey(context.getHandlerKey())
.exception(exception)
.build();
if (globalExceptionHandler != null) {
globalExceptionHandler.accept(wrapper);
}
if (context.getExceptionHandler() != null) {
context.getExceptionHandler().accept(wrapper);
} else {
DEFAULT_EXCEPTION_HANDLER.accept(wrapper);
}
}
@ -243,9 +249,9 @@ public class EventHandlerRepository {
}
// consumer显式声明了日志开关，则直接使用
if (logEnabled != null) {
return logEnabled && (logFilter == null || logFilter.apply(event));
return logEnabled && (logFilter == null || logFilter.test(event));
}
return logFilter == null || logFilter.apply(event);
return logFilter == null || logFilter.test(event);
}
private boolean isSupportPattern() {

1
event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/RetryableEventConsumer.java
View File

@ -39,7 +39,6 @@ public class RetryableEventConsumer implements EventConsumer {
Preconditions.checkNotNull(eventConsumer);
this.eventConsumer = eventConsumer;
exceptionHandler = context -> {
context.doPrintException();
if (Strings.isNullOrEmpty(context.getHandlerKey())) {
return;
}

44
excel-support-lib/pom.xml Normal file
View File

@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>cn.axzo.foundation</groupId>
<artifactId>axzo-lib-box</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<groupId>cn.axzo.maokai</groupId>
<artifactId>excel-support-lib</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>cn.axzo.foundation</groupId>
<artifactId>web-support-lib</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>3.3.4</version>
</dependency>
<dependency>
<groupId>com.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>5.9</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
</project>

918
excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/DataSheetClient.java Normal file
View File

@ -0,0 +1,918 @@
package cn.axzo.foundation.excel.support;
import cn.axzo.foundation.exception.BusinessException;
import cn.axzo.foundation.page.IPageReq;
import cn.axzo.foundation.page.PageResp;
import cn.axzo.foundation.util.VarParamFormatter;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPath;
import com.alibaba.fastjson.parser.ParserConfig;
import com.alibaba.fastjson.util.TypeUtils;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import lombok.*;
import lombok.experimental.Accessors;
import lombok.experimental.SuperBuilder;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.poi.ss.SpreadsheetVersion;
import javax.servlet.http.HttpServletResponse;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*;
/**
* 用于发送sms & email验证码, 或者获取图片验证码. 并提供校验验证码是否正确的能力
*/
public interface DataSheetClient {
ExporterBuilder exportBuilder();
/**
* 数据导入的builder
*
* @return
*/
ImporterBuilder importBuilder();
@Data
@NoArgsConstructor
@AllArgsConstructor
@Accessors(fluent = true)
abstract class ImporterBuilder {
/**
* 导入的场景名称, 必填
*/
@NonNull
private String scene;
/**
* 导入格式
*/
@NonNull
private ImportFormat format;
/**
* EXCEL格式的sheetName
*/
private String tableName;
private Meta meta;
/**
* 是否忽略导入文件表头的原meta
* 默认按false处理，即{@link #meta}为空时尝试使用原文件表头meta
*/
private Boolean ignoreHeaderMeta;
private boolean debugEnabled;
/**
* 忽略未知的列(cellMeta中未定义的列)
*/
private boolean ignoreUnknownColumn;
/**
* 是否自动清除表格内容的前后空白字符
* 默认true，即会清除前后空白字符
*/
private Boolean autoTrim;
/**
* 解析结果ImportResp中的lines是否包含解析失败的行
* true: 解析失败的时候，不会抛异常，会在每行的JSONObject中添加
* "errors":[{"columnIndex":1,"columnKey":"","columnName":"","rawValue":"","errorCode":"","errorMsg":""}]
* false: 解析失败的时候，抛第一个异常
*/
private boolean includeLineErrors;
/**
* 允许最大的导入行数
*/
private Integer allowMaxLineCount;
/** 将excel中的表头转换为cellMeta中的表头 */
private Function<String, String> headerConverter;
/**
* 操作人, 可选，用于导入统计报告
*/
private String operator;
public abstract Importer build();
}
interface Importer {
/**
* 所有字段读取为String类型
*
* @param inputStream
* @return
*/
ImportResp<JSONObject> readAll(InputStream inputStream);
}
@Builder
@Data
@NoArgsConstructor
@AllArgsConstructor
class ImportResp<T> {
/**
* 导入的场景名称
*/
private String scene;
/**
* 模版对应的code以及版本
*/
private String templateCode;
private String version;
/**
* headers
*/
private List<String> headers;
/**
* 每一行的数据
*/
private List<T> lines;
/**
* header的行数
*/
private Integer headerRowCount;
/**
* 导入数据总行数, 不包含header
*/
private Integer rowCount;
/**
* 导入数据列数
*/
private Integer columnCount;
/**
* Meta数据
*/
private Meta meta;
/**
* 消耗的时间
*/
private Long elapsedMillis;
/**
* 操作人
*/
private String operator;
}
@AllArgsConstructor
@Getter
enum ImportFormat {
EXCEL("xlsx"),
// TODO: 支持其他类型
// CSV("csv")
;
private String suffix;
}
@Data
@NoArgsConstructor
@Accessors(fluent = true)
abstract class ExporterBuilder {
/**
* 导出的场景名称, 必填
*/
@NonNull
private String scene;
/**
* 导出格式
*/
@NonNull
private ExportFormat format;
/**
* 顶部提示内容，主要用于一些提示场景。如导出数据超范围等文案提示等，支持多行。
*/
private Function<PageResp<JSONObject>, List<String>> topHintsSupplier;
/**
* 提供分页获取数据的方法
*/
@NonNull
private Function<IPageReq, PageResp<JSONObject>> rowSupplier;
/**
* 导出的总数据量限制，不指定时，按照分页返回的结果全量导出
*/
private Long limit = Long.MAX_VALUE;
/**
* 获取数据的分页大小, 默认每页200条(MybatisPlus的PaginationInterceptor默认limit为500)
*/
private Long pageSize = 200L;
/**
* 表名，仅当导出格式为excel时有效
*/
private String sheetName;
@Deprecated
public ExporterBuilder tableName(String tableName) {
this.sheetName = tableName;
return this;
}
/**
* 列映射表, key=列名， value=获取字段值的方法, 方法参数1为当前row的JSONObject, 参数2为当前row的index，返回列字段值
*/
private ImmutableMap<String, BiFunction<JSONObject, Integer, Object>> columnMap;
/**
* 多行表头映射表, 当表头由多行数据组成时使用. key=表头, value=该表头对应的每行的内容
*/
private ImmutableMap<String, List<String>> multiLineHeadMap;
/**
* 行数据的转换器，外部可以通过它将原始的行数据根据需要做转换
*/
private Function<JSONObject, List<JSONObject>> rowConverter;
/**
* 页数据的转换器，外部可以通过它将也的数据根据需要做转换
* 它会在 rowconvert() 执行完成
*/
private Function<List<JSONObject>, List<JSONObject>> pageConverter;
private BiConsumer<Integer, String> onProgress;
/**
* 批注信息，可选
*/
private Meta meta;
/**
* 是否开启调试日志
*/
private Boolean debugEnabled = true;
/**
* 导出的fields. 优先使用fields来创建column&meta
*/
private List<ExportField> fields;
/**
* meta的version
*/
private String version;
/** 文件名 */
private String fileName;
/** 操作人, 可选，用于异步导出 */
private String operator;
/** 拓展字段，可用于存放请求参数等，该字段会回传给FileLoaderParam */
private JSONObject ext;
/**
* 仅对导出excel生效，是否自动清除表格内容的前后空白字符
* 默认true，即会清除前后空白字符
*/
private Boolean autoTrim;
abstract public Exporter build();
}
@Data
class ExportField {
private String column;
private String header;
private List<String> headerLines;
private Integer width;
private CellMeta.Type type;
private Boolean mandatory;
private Boolean wrapText;
private String lowerType;
private String upperType;
private String dateTimeFormat;
private List<Object> options;
private BiFunction<JSONObject, Integer, Object> reader;
@Builder
public ExportField(String column, String header, List<String> headerLines, Integer width, CellMeta.Type type,
Boolean mandatory, Boolean wrapText, Function<String, Object> resultConverter,
BiFunction<JSONObject, Integer, Object> reader, List<Object> options) {
Preconditions.checkArgument(!Strings.isNullOrEmpty(column), "column不能为空");
Preconditions.checkArgument(!Strings.isNullOrEmpty(header), "header不能为空");
Preconditions.checkArgument(!(resultConverter != null && reader != null), "reader & resultConverter不能同时存在");
this.column = column;
this.header = header;
this.headerLines = headerLines;
this.width = width;
this.type = Optional.ofNullable(type).orElse(CellMeta.Type.STRING);
this.mandatory = Optional.ofNullable(mandatory).orElse(Boolean.FALSE);
this.wrapText = wrapText;
if (reader != null) {
this.reader = reader;
} else {
this.reader = Optional.ofNullable(resultConverter)
.map(e -> (BiFunction<JSONObject, Integer, Object>) (row, integer) -> e.apply(Strings.nullToEmpty(row.getString(column))))
.orElseGet(() -> DataSheetClient.stringCellReader(column));
}
this.options = options;
}
public CellMeta toCellMeta() {
JSONObject params = new JSONObject();
if (CellMeta.Type.RANGE == type) {
params.put(EXT_KEY_RANGE_LOWER_TYPE, lowerType);
params.put(EXT_KEY_RANGE_UPPER_TYPE, upperType);
}
if (CellMeta.Type.DATE == type || CellMeta.Type.DATETIME == type) {
params.put(EXT_KEY_DATETIME_FORMAT, dateTimeFormat);
}
return CellMeta.builder()
.key(column).name(header).width(width).mandatory(mandatory).type(type).wrapText(wrapText)
.params(params).options(options).build();
}
}
interface Exporter {
/**
* 导出到OutputStream中
*
* @param outputStream
* @return
*/
ExportResp export(@NonNull OutputStream outputStream);
/**
* 导出到HttpServletResponse中
*
* @param response
* @return
*/
ExportResp export(@NonNull HttpServletResponse response);
/**
* 导出到HttpServletResponse中，可指定导出的文件名
*
* @param response
* @return
*/
ExportResp export(HttpServletResponse response, String filename);
}
interface AsyncExporter {
/**
* 异步导出数据
*
* @param dataSheetExporter 实现导出逻辑的 exporter
* @return batchId
*/
String export(@NonNull ExporterBuilder dataSheetExporter);
/**
* 多表单数据异步导出至同一文件中, 仅支持导出格式是EXCEL
*
* @param fileName 导出文件名
* @param dataSheetExporters 实现导出逻辑的 exporter 列表
* @return batchId
*/
String export(String fileName, @NonNull List<ExporterBuilder> dataSheetExporters);
Progress getProgress(@NonNull String batchId);
@NoArgsConstructor
@AllArgsConstructor
@Data
@Builder
class Progress {
String fileName;
int progress;
String errorMsg;
String downloadUrl;
public boolean isSuccess() {
return Strings.isNullOrEmpty(errorMsg);
}
public boolean isDone() {
return progress < 0 || progress >= 100;
}
}
}
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
class ExportResp {
/**
* 导出场景
*/
private String scene;
/**
* 导出数据条数
*/
private Long exportedCount;
/**
* 导出数据总行数，包含了顶部提示行及表头行
*/
private Long rowCount;
/**
* 导出数据列数
*/
private Long columnCount;
/**
* 导出数据行数
*/
private Long dataRowCount;
/**
* 耗时
*/
private Long elapsedMillis;
/**
* 操作人
*/
private String operator;
}
@Builder
@Data
@NoArgsConstructor
@AllArgsConstructor
class Meta {
public static final String PATTERN_VERSION_OPEN = "$V{";
public static final String PATTERN_CELL_OPEN = "$C{";
public static final String PATTERN_IGNORE_OPEN = "$I{";
public static final String PATTERN_EXTRA_OPEN = "$E{";
public static final String PATTERN_CLOSE = "}";
public static final String RANGE_TYPE_OPEN_OPEN = "1";
public static final String RANGE_TYPE_OPEN_CLOSED = "2";
public static final String RANGE_TYPE_CLOSED_OPEN = "3";
public static final String RANGE_TYPE_CLOSED_CLOSED = "4";
public static final String IGNORE_ROW_KEY = "r";
public static final String IGNORE_COLUMN_KEY = "c";
public static final String IGNORE_ROW_AND_COLUMN_KEY = "b";
private String templateCode;
private String version;
private List<CellMeta> cellMetas;
/**
* 忽略掉的行号
*/
private List<Integer> ignoreRowIndexes;
/**
* 忽略掉的列号
*/
private List<Integer> ignoreColumnIndexes;
/**
* 忽略掉的列名
* 当导入文件中有meta信息时，该ignore信息将会丢失
*/
private Set<String> ignoreColumnNames;
public List<Integer> getIgnoreRowIndexes() {
return Optional.ofNullable(ignoreRowIndexes).orElseGet(ImmutableList::of);
}
public List<Integer> getIgnoreColumnIndexes() {
return Optional.ofNullable(ignoreColumnIndexes).orElseGet(ImmutableList::of);
}
public Set<String> getIgnoreColumnNames() {
return Optional.ofNullable(ignoreColumnNames).orElseGet(ImmutableSet::of);
}
}
@SuperBuilder
@Data
@NoArgsConstructor
@AllArgsConstructor
class CellMeta {
public static final String EXT_KEY_DATETIME_FORMAT = "dateTimeFormat";
public static final String EXT_KEY_RANGE_LOWER_TYPE = "lowerType";
public static final String EXT_KEY_RANGE_UPPER_TYPE = "upperType";
public static final String RANGE_BOUND_TYPE_OPEN = "open";
public static final String RANGE_BOUND_TYPE_CLOSED = "closed";
public static final Set<String> RANGE_TYPES = ImmutableSet.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED);
private static final List<String> DEFAULT_DATE_FORMATS = ImmutableList.of(
"yyyy-MM-dd",
"yyyy-MM-dd HH:mm",
"yyyy-MM-dd HH:mm:ss",
"yyyy.MM.dd",
"yyyy.MM.dd HH:mm",
"yyyy.MM.dd HH:mm:ss",
"yyyy年MM月dd日",
"yyyy年MM月dd日 HH:mm",
"yyyy年MM月dd日 HH:mm:ss",
"yyyy年MM月dd日 HH时mm分",
"yyyy年MM月dd日 HH时mm分ss秒",
"yyyy/MM/dd",
"yyyy/MM/dd HH:mm",
"yyyy/MM/dd HH:mm:ss",
"yyyy-MM-dd'T'HH:mm:ss'Z'",
"yyyy-MM-dd'T'HH:mm:ssZ",
"yyyy-MM-dd'T'HH:mm:ssz",
"yyyy-MM-dd'T'HH:mm:ss",
"MM/dd/yyyy HH:mm:ss a",
"yyyyMMddHHmmss",
"yyyyMMdd"
);
private static final Map<String, Boolean> BOOLEAN_MAP = ImmutableMap.<String, Boolean>builder()
.put("是", Boolean.TRUE)
.put("yes", Boolean.TRUE)
.put("y", Boolean.TRUE)
.put("1", Boolean.TRUE)
.put("否", Boolean.FALSE)
.put("no", Boolean.FALSE)
.put("n", Boolean.FALSE)
.put("0", Boolean.FALSE)
.build();
/**
* cell的key
*/
private String key;
/**
* cell的header，一般为中文
*/
private String name;
/**
* cell的宽度
*/
private Integer width;
/**
* 类型
*/
private Type type;
/**
* 是否必需, 为false时, 允许该列在Excel中不存在
*/
private Boolean mandatory;
/**
* 可选值
*/
private List<Object> options;
/**
* 存放一些和type相关的参数，例如DateTime的pattern格式，Range的开闭区间
*/
private JSONObject params;
/**
* 存放一些额外的信息
*/
private JSONObject ext;
/**
* wrap the text automatically
*/
private Boolean wrapText;
/**
* 导入的时候，将原始的String转换为特定的类型
*/
private Function<String, Object> importConverter;
/**
* 字段值额外的校验方法，支持根据当前字段值及整行数据进行自定义校验
*/
private BiConsumer<Object, JSONObject> cellValidator;
public void validate() {
if (type == Type.RANGE
&& (!RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_LOWER_TYPE))
|| !RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_UPPER_TYPE)))) {
throw ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR.toException();
}
}
public Object convertType(String value) {
if (importConverter != null) {
return importConverter.apply(value);
}
if (Boolean.TRUE.equals(mandatory) && Strings.isNullOrEmpty(value)) {
throw ResultCode.IMPORT_CELL_META_MISSING_MANDATORY_VALUE.toException();
}
// 非必填的字段如果为空，直接返回
if (value == null) {
return type == Type.STRING ? StringUtils.EMPTY : null;
}
switch (type) {
case INT:
// 移除千分符
return Integer.valueOf(value.replace(",", ""));
case LONG:
// 移除千分符
return Long.valueOf(value.replace(",", ""));
case FLOAT:
// 移除千分符
return Float.valueOf(value.replace(",", ""));
case DOUBLE:
// 移除千分符
return Double.valueOf(value.replace(",", ""));
case BIG_DECIMAL:
// 移除千分符
return new BigDecimal(value.replace(",", ""));
case DATE:
case DATETIME:
return parseDate(value, DEFAULT_DATE_FORMATS);
case RANGE:
// 格式 100-200
List<Integer> values = Splitter.on("-").omitEmptyStrings().trimResults().splitToList(value)
.stream()
.map(Integer::valueOf)
.collect(Collectors.toList());
if (values.size() != 2) {
throw ResultCode.IMPORT_CELL_RANGE_FORMAT_ERROR.toException();
}
if (values.get(0) >= values.get(1)) {
throw ResultCode.IMPORT_CELL_RANGE_VALUE_ERROR.toException();
}
return new JSONObject()
.fluentPut("lower", values.get(0))
.fluentPut("upper", values.get(1))
.fluentPut(EXT_KEY_RANGE_LOWER_TYPE, params.getString(EXT_KEY_RANGE_LOWER_TYPE))
.fluentPut(EXT_KEY_RANGE_UPPER_TYPE, params.getString(EXT_KEY_RANGE_UPPER_TYPE));
case BOOLEAN:
return Optional.ofNullable(BOOLEAN_MAP.get(value.toLowerCase()))
.orElseThrow(ResultCode.IMPORT_CELL_BOOLEAN_VALUE_ERROR::toException);
default:
return Strings.nullToEmpty(value);
}
}
private LocalDateTime parseDate(String dateStr, List<String> formats) {
try {
Date date = DateUtils.parseDate(dateStr, formats.toArray(new String[0]));
return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
} catch (Exception e) {
throw ResultCode.IMPORT_CELL_DATETIME_CONVERT_FAILED.toException("不支持的日期格式{}", dateStr);
}
}
@AllArgsConstructor
public enum Type {
INT("00"),
LONG("01"),
FLOAT("02"),
DOUBLE("03"),
STRING("04"),
BIG_DECIMAL("05"),
DATE("06"),
DATETIME("07"),
RANGE("08"),
BOOLEAN("09");
@Getter
private String code;
private static Map<String, Type> map = Stream.of(Type.values())
.collect(Collectors.toMap(Type::getCode, Function.identity()));
public static Type from(String code) {
return map.get(code);
}
}
}
@AllArgsConstructor
@Getter
enum ExportFormat {
EXCEL("application/vnd.ms-excel", ".xlsx"),
CSV("text/csv", ".csv");
private String contentType;
private String suffix;
}
static BiFunction<JSONObject, Integer, Object> indexCellReader() {
return (row, index) -> String.valueOf(index + 1);
}
static BiFunction<JSONObject, Integer, Object> stringCellReader(String columnName) {
// excel库poi会检查文本格式的单元格内容，当超过32767时，将会抛出IllegalArgumentException异常
// 参考 {@link org.apache.poi.xssf.streaming.SXSSFCell.setCellValue}
return (row, index) -> StringUtils.left(Strings.nullToEmpty(row.getString(columnName)),
SpreadsheetVersion.EXCEL2007.getMaxTextLength());
}
static BiFunction<JSONObject, Integer, Object> dateTimeCellReader(String columnName, String pattern) {
return (row, index) -> {
Long epochMillis = row.getLong(columnName);
if (epochMillis == null) {
return StringUtils.EMPTY;
}
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
Instant instant = Instant.ofEpochMilli(epochMillis);
LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
return formatter.format(localDateTime);
};
}
static BiFunction<JSONObject, Integer, Object> bigDecimalCellReader(String columnName) {
return bigDecimalCellReader(columnName, 2);
}
static BiFunction<JSONObject, Integer, Object> bigDecimalCellReader(String columnName, int scale) {
return (row, index) -> {
BigDecimal number = row.getBigDecimal(columnName);
if (number == null) {
number = BigDecimal.ZERO;
}
return number.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString();
};
}
static BiFunction<JSONObject, Integer, Object> jsonPathCellReader(String jsonPath) {
return (row, index) -> {
Object obj = JSONPath.eval(row, jsonPath);
/**
* excel单元格对于数值最大只能保有15位，超过15位的数值(Long)会被置为0，比如在导出uid的场景，会导致数据错误
* https://docs.microsoft.com/en-us/office/troubleshoot/excel/last-digits-changed-to-zeros
* https://wenku.baidu.com/view/e1d2a497f624ccbff121dd36a32d7375a417c6ad.html
*/
if (obj instanceof Long && obj.toString().length() > 15) {
obj = obj.toString();
}
if (obj instanceof String) {
// excel库poi会检查文本格式的单元格内容，当超过32767时，将会抛出IllegalArgumentException异常
// 参考 {@link org.apache.poi.xssf.streaming.SXSSFCell.setCellValue}
return StringUtils.left(obj.toString(), SpreadsheetVersion.EXCEL2007.getMaxTextLength());
}
return obj;
};
}
static BiFunction<JSONObject, Integer, Object> jsonPathDateTimeCellReader(String jsonPath, String pattern) {
return (row, index) -> {
Object val = JSONPath.eval(row, jsonPath);
Long epochMillis = TypeUtils.cast(val, Long.class, ParserConfig.getGlobalInstance());
if (epochMillis == null) {
return StringUtils.EMPTY;
}
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
Instant instant = Instant.ofEpochMilli(epochMillis);
LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
return formatter.format(localDateTime);
};
}
static BiFunction<JSONObject, Integer, Object> jsonPathBigDecimalCellReader(String jsonPath) {
return jsonPathBigDecimalCellReader(jsonPath, 2);
}
static BiFunction<JSONObject, Integer, Object> jsonPathBigDecimalCellReader(String jsonPath, int scale) {
return (row, index) -> {
Object val = JSONPath.eval(row, jsonPath);
if (val == null) {
val = BigDecimal.ZERO;
}
BigDecimal bigDecimal = TypeUtils.cast(val, BigDecimal.class, ParserConfig.getGlobalInstance());
return bigDecimal.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString();
};
}
@Getter
class DataSheetException extends BusinessException {
private String subErrorCode;
/** 异常相关的行号，列号，0开始 */
private Integer rowIndex;
private Integer columnIndex;
/** 异常相关的列名称，中文 */
private List<String> columnNames;
public DataSheetException(ResultCode resultCode) {
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage());
this.subErrorCode = resultCode.getSubBizCode();
}
public DataSheetException(ResultCode resultCode, Integer rowIndex, Integer columnIndex) {
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage());
this.subErrorCode = resultCode.getSubBizCode();
this.rowIndex = rowIndex;
this.columnIndex = columnIndex;
}
public DataSheetException(ResultCode resultCode, List<String> columnNames) {
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage());
this.subErrorCode = resultCode.getSubBizCode();
this.columnNames = columnNames;
}
public DataSheetException(ResultCode resultCode, String message) {
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), message);
this.subErrorCode = resultCode.getSubBizCode();
}
public DataSheetException(String subErrorCode, String errorMsg, Integer rowIndex, Integer columnIndex) {
super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), errorMsg);
this.subErrorCode = subErrorCode;
this.rowIndex = rowIndex;
this.columnIndex = columnIndex;
}
public DataSheetException(String errorCode, String subErrorCode, String errorMsg) {
super(errorCode, errorMsg);
this.subErrorCode = subErrorCode;
}
}
@AllArgsConstructor
@Getter
enum ResultCode {
/** 解析Excel的批注时，相关的errorCode */
IMPORT_PARSE_MISSING_VERSION("C00", "批注中没有找到版本信息"),
IMPORT_PARSE_VERSION_FORMAT_ERROR("C01", "批注中的版本格式不对"),
IMPORT_PARSE_CELL_META_MISSING_TYPE("C02", "批注中的字段没有找到类型信息"),
IMPORT_PARSE_CELL_META_FORMAT_ERROR("C03", "批注中的字段格式不对"),
IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR("C04", "批注中的范围类型字段的格式不对"),
IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR("C05", "批注中的范围类型字段的区间类型不对"),
/** 导入数据时，相关的errorCode */
IMPORT_LINES_REACHED_LIMIT("C20", "导入的数据超过了最大行数"),
IMPORT_COLUMN_RANGE_MISSING_TYPES("C21", "范围类型缺少区间类型的定义：开区间还是闭区间"),
IMPORT_COLUMN_DUPLICATED_NAME("C22", "列的名称不能重复"),
IMPORT_COLUMN_MISSING_CELL_META("C23", "字段缺少类型定义"),
IMPORT_COLUMN_NAME_NOT_MATCHED("C24", "字段的名称与类型定义的名称不一致"),
IMPORT_CELL_RANGE_FORMAT_ERROR("C25", "范围类型的格式不对"),
IMPORT_CELL_RANGE_VALUE_ERROR("C26", "范围类型的下限值必须小于上限值"),
IMPORT_CELL_DATETIME_CONVERT_FAILED("C27", "时间类型解析失败"),
IMPORT_CELL_CONVERT_FAILED("C28", "类型转换失败"),
IMPORT_CELL_META_MISSING_MANDATORY_VALUE("C29", "必填字段不能为空"),
IMPORT_CELL_BOOLEAN_VALUE_ERROR("C30", "布尔类型的值不支持"),
;
private String subBizCode;
private String message;
public DataSheetException toException() {
return new DataSheetException(this);
}
public DataSheetException toException(String message) {
return new DataSheetException(this, message);
}
public DataSheetException toException(List<String> columnNames) {
return new DataSheetException(this, columnNames);
}
public DataSheetException toException(Integer rowIndex, Integer columnIndex) {
return new DataSheetException(this, rowIndex, columnIndex);
}
public DataSheetException toException(String customMsg, Object... objects) {
if (objects == null) {
return toException(customMsg);
}
String msg = VarParamFormatter.format(customMsg, objects);
//如果最后一个参数是Throwable. 则将SimpleName附加到msg中
if (objects[objects.length - 1] instanceof Throwable) {
Throwable throwable = (Throwable) objects[objects.length - 1];
msg = String.format("%s (%s)", msg, throwable.getClass().getSimpleName());
}
return toException(msg);
}
}
}

326
excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetAsyncExporter.java Normal file
View File

@ -0,0 +1,326 @@
package cn.axzo.foundation.excel.support.impl;
import cn.axzo.foundation.enums.AppEnvEnum;
import cn.axzo.foundation.excel.support.DataSheetClient;
import cn.axzo.foundation.util.UUIDBuilder;
import cn.axzo.foundation.web.support.AppRuntime;
import cn.axzo.foundation.web.support.utils.KeyBuilder;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.util.IoUtils;
import com.alibaba.excel.util.StringUtils;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.task.TaskExecutor;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.util.CollectionUtils;
import java.io.*;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* 异步数据导出.
* 将导出的进度防止redis 中，当导出完成后，将文件通过 fileUploader 上传到云存储。
*/
@Slf4j
public class DataSheetAsyncExporter implements DataSheetClient.AsyncExporter {
private static final int PROGRESS_EXPIRE_DAYS = 3;
private RedisTemplate<String, String> redisTemplate;
private AppRuntime appRuntime;
private Function<FileUploadParam, FileUploadResult> fileUploader;
private TaskExecutor executor;
public DataSheetAsyncExporter(@NonNull RedisTemplate<String, String> redisTemplate,
@NonNull AppRuntime appRuntime,
@NonNull Function<FileUploadParam, FileUploadResult> fileUploader,
@NonNull TaskExecutor executor) {
this.redisTemplate = redisTemplate;
this.appRuntime = appRuntime;
this.fileUploader = fileUploader;
this.executor = executor;
}
@Data
@lombok.Builder
@NoArgsConstructor
@AllArgsConstructor
public static class FileUploadParam {
private byte[] bytes;
private String fileName;
private String operator;
private JSONObject ext;
}
@Data
@lombok.Builder
@NoArgsConstructor
@AllArgsConstructor
public static class FileUploadResult {
private String downloadUrl;
}
@Override
public String export(@NonNull DataSheetClient.ExporterBuilder dataSheetExporter) {
final String batchId = UUIDBuilder.generateLongUuid(false);
final String scene = dataSheetExporter.scene();
ExportContext context = ExportContext.builder().batchId(batchId).fileName(dataSheetExporter.fileName())
.operator(dataSheetExporter.operator()).ext(dataSheetExporter.ext()).build();
executor.execute(() -> doExport(context, scene, dataSheetExporter));
return batchId;
}
@Override
public String export(String fileName, @NonNull List<DataSheetClient.ExporterBuilder> dataSheetExporters) {
checkMultiExporters(fileName, dataSheetExporters);
final String batchId = UUIDBuilder.generateLongUuid(false);
ExportContext context = ExportContext.builder()
.batchId(batchId)
.fileName(fileName)
.operator(dataSheetExporters.get(0).operator())
.writer(EasyExcel.write(new ByteArrayOutputStream())
.registerWriteHandler(new LongestMatchColumnWidthStyleStrategy())
.build())
.progressMap(dataSheetExporters.stream().collect(Collectors.toMap(e -> e.sheetName(), e -> 0)))
.build();
executor.execute(() -> dataSheetExporters.forEach(e -> doExport(context, e.scene(), e)));
return batchId;
}
void doExport(final ExportContext context, final String scene, DataSheetClient.ExporterBuilder dataSheetExporter) {
try {
final File tempFile = File.createTempFile(context.getBatchId(), ".tmp");
tempFile.deleteOnExit();
log.info("DataSheetAsyncExporter export() scene={} context={}, tempFile->{}", scene, context, tempFile.getAbsolutePath());
try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) {
DataSheetClient.Exporter exporter = dataSheetExporter.onProgress((progress, errorMsg) -> {
if (context.isMultiSheet()) {
onMultiSheetProgress(progress, errorMsg, context, tempFile, dataSheetExporter.sheetName());
} else {
onProgress(progress, errorMsg, context, tempFile);
}
}).build();
Preconditions.checkState(exporter != null);
exporter.export(out);
} catch (Exception e) {
log.error("DataSheetAsyncExporter export() FAIL scene={} context={} ", scene, context, e);
} finally {
if (tempFile != null && tempFile.exists()) {
tempFile.delete();
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private void checkMultiExporters(String fileName, List<DataSheetClient.ExporterBuilder> dataSheetExporters) {
Preconditions.checkArgument(!Strings.isNullOrEmpty(fileName), "导出文件名不能为空");
Preconditions.checkArgument(!CollectionUtils.isEmpty(dataSheetExporters), "导出参数不能为空");
dataSheetExporters.forEach(e -> Preconditions.checkArgument(e.format() == DataSheetClient.ExportFormat.EXCEL, "导出格式必须为EXCEL"));
dataSheetExporters.forEach(e -> Preconditions.checkArgument(!Strings.isNullOrEmpty(e.sheetName()), "表单名不能为空"));
long sheetCount = dataSheetExporters.stream().map(DataSheetClient.ExporterBuilder::sheetName).distinct().count();
Preconditions.checkArgument(sheetCount == dataSheetExporters.size(), "表单名不能重复");
}
private String buildRedisKey(String batchId) {
return KeyBuilder.build(appRuntime, "data_export", batchId);
}
private void onProgress(Integer progress, String errorMsg, ExportContext context, File tempFile) {
log.info("DataSheetAsyncExporter onProgress() progress={} errorMsg={} context={} tempFile={}", progress, errorMsg, context, tempFile.getAbsolutePath());
String downloadUrl = StringUtils.EMPTY;
if (Strings.isNullOrEmpty(errorMsg) && progress >= 100) {
try {
// 只有导出成功，并完成，才开始上传文件
byte[] bytes = IoUtils.toByteArray(new BufferedInputStream(new FileInputStream(tempFile)));
downloadUrl = fileUploader.apply(FileUploadParam.builder()
.bytes(bytes).fileName(context.getFileName())
.operator(context.getOperator()).ext(context.getExt()).build()).getDownloadUrl();
log.info("DataSheetAsyncExporter onProgress() DONE context={}, downloadUrl={}", context, downloadUrl);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
// 将导出的状态放入 redis 中。后续通过getProgress()来获取。
String jsonString = JSONObject.toJSONString(Progress.builder().fileName(context.getFileName()).progress(progress)
.downloadUrl(downloadUrl).errorMsg(errorMsg).build());
redisTemplate.opsForValue().set(buildRedisKey(context.getBatchId()), jsonString, PROGRESS_EXPIRE_DAYS, TimeUnit.DAYS);
}
private void onMultiSheetProgress(Integer progress, String errorMsg, ExportContext context, File tempFile, String sheetName) {
log.info("DataSheetAsyncExporter onMultiSheetProgress() context={}, progress={} errorMsg={} tempFile={} sheetName={}", context, progress, errorMsg, tempFile.getAbsolutePath(), sheetName);
context.getProgressMap().put(sheetName, progress);
Progress totalProgress = getMultiSheetProgress(context, errorMsg);
if (!Strings.isNullOrEmpty(errorMsg) && progress < 0) {
context.getWriter().write(ImmutableList.of(ImmutableList.of(errorMsg)),
EasyExcel.writerSheet(sheetName).build());
} else if (progress >= 100) {
EasyExcel.read(tempFile, new ReadAndWriteListener(context, sheetName)).sheet(sheetName).doRead();
}
if (totalProgress.isDone()) {
// 只有所有表单都导出完成，才开始上传文件
context.getWriter().finish();
ByteArrayOutputStream outputStream = (ByteArrayOutputStream) context.getWriter().writeContext()
.writeWorkbookHolder().getOutputStream();
String downloadUrl = fileUploader.apply(FileUploadParam.builder()
.bytes(outputStream.toByteArray()).fileName(context.getFileName())
.operator(context.getOperator()).build()).getDownloadUrl();
log.info("DataSheetAsyncExporter onMultiSheetProgress() DONE context={}, downloadUrl={}", context, downloadUrl);
totalProgress.setDownloadUrl(downloadUrl);
}
// 将导出的状态放入 redis 中。后续通过getProgress()来获取。
String jsonString = JSONObject.toJSONString(totalProgress);
redisTemplate.opsForValue().set(buildRedisKey(context.getBatchId()), jsonString, PROGRESS_EXPIRE_DAYS, TimeUnit.DAYS);
}
private Progress getMultiSheetProgress(ExportContext context, String errorMsg) {
Map<String, Integer> progressMap = context.getProgressMap();
int totalProgress = progressMap.values().stream().mapToInt(a -> a).sum() / progressMap.size();
boolean isDone = progressMap.values().stream().allMatch(a -> a < 0 || a >= 100);
if (isDone && totalProgress != 100) {
// 所有导出都已完成，但其中有失败，将progress设为-1
totalProgress = -1;
}
return Progress.builder()
.fileName(context.getFileName())
.progress(totalProgress)
.errorMsg(errorMsg)
.downloadUrl(StringUtils.EMPTY)
.build();
}
/**
* 使用在export()返回的 batchId 来查询导出进度。
*
* @param batchId
* @return
*/
@Override
public Progress getProgress(String batchId) {
String res = redisTemplate.opsForValue().get(buildRedisKey(batchId));
if (Strings.isNullOrEmpty(res)) {
return Progress.builder().fileName(StringUtils.EMPTY)
.progress(0).errorMsg(StringUtils.EMPTY).downloadUrl(StringUtils.EMPTY).build();
}
return JSONObject.parseObject(res, Progress.class);
}
@Data
public static class Builder {
private RedisTemplate<String, String> redisTemplate;
private AppRuntime appRuntime;
private Function<FileUploadParam, FileUploadResult> fileUploader;
private TaskExecutor executor;
public Builder redisTemplate(RedisTemplate redisTemplate) {
this.redisTemplate = redisTemplate;
return this;
}
public Builder appRuntime(AppRuntime appRuntime) {
this.appRuntime = appRuntime;
return this;
}
public Builder fileUploader(Function<FileUploadParam, FileUploadResult> fileUploader) {
this.fileUploader = fileUploader;
return this;
}
public Builder executor(TaskExecutor executor) {
this.executor = executor;
return this;
}
public DataSheetClient.AsyncExporter build() {
if (appRuntime.getEnv() == AppEnvEnum.unittest) {
return new DataSheetClient.AsyncExporter() {
@Override
public String export(DataSheetClient.@NonNull ExporterBuilder dataSheetExporter) {
return "8888";
}
@Override
public String export(String fileName, @NonNull List<DataSheetClient.ExporterBuilder> dataSheetExporters) {
return "8888";
}
@Override
public Progress getProgress(String batchId) {
return Progress.builder().progress(100).downloadUrl("http://test/test.jpg").build();
}
};
}
return new DataSheetAsyncExporter(redisTemplate, this.appRuntime, this.fileUploader, this.executor);
}
}
private static class ReadAndWriteListener extends AnalysisEventListener<Map<Integer, String>> {
private ExportContext context;
private WriteSheet sheet;
public ReadAndWriteListener(ExportContext context, String sheetName) {
this.context = context;
this.sheet = EasyExcel.writerSheet(sheetName).build();
}
@Override
public void invoke(Map<Integer, String> data, AnalysisContext context) {
List<String> row = data.entrySet().stream().sorted(Map.Entry.comparingByKey())
.map(Map.Entry::getValue).collect(Collectors.toList());
this.context.getWriter().write(ImmutableList.of(row), sheet);
}
@Override
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
List<String> row = headMap.entrySet().stream().sorted(Map.Entry.comparingByKey())
.map(Map.Entry::getValue).collect(Collectors.toList());
this.context.getWriter().write(ImmutableList.of(row), sheet);
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
}
}
@Data
@lombok.Builder
@NoArgsConstructor
@AllArgsConstructor
static class ExportContext {
private String batchId;
private String fileName;
private String operator;
private ExcelWriter writer;
private Map<String, Integer> progressMap;
private JSONObject ext;
private boolean isMultiSheet() {
return writer != null && !CollectionUtils.isEmpty(progressMap);
}
}
}

90
excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetClientImpl.java Normal file
View File

@ -0,0 +1,90 @@
package cn.axzo.foundation.excel.support.impl;
import cn.axzo.foundation.excel.support.DataSheetClient;
import com.alibaba.fastjson.JSONObject;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.cglib.beans.BeanMap;
import java.util.HashMap;
import java.util.Map;
@Slf4j
public class DataSheetClientImpl implements DataSheetClient {
private final static String UNKNOWN_OPERATOR = "unknown@fiture.com";
@Override
public ExporterBuilder exportBuilder() {
DataSheetExporter exporter = new DataSheetExporter();
exporter.setOnCompleted(this::reportExport);
return exporter;
}
@Override
public ImporterBuilder importBuilder() {
DataSheetImporter importer = new DataSheetImporter();
importer.setOnCompleted(this::reportImport);
return importer;
}
public void reportExport(DataSheetClient.ExportResp resp) {
String operator = resp.getOperator();
ReportReq req = ReportReq.builder()
.scene(resp.getScene())
.action(ReportReq.Action.EXPORT)
.resp(BeanMap.create(resp))
.rowCount(resp.getRowCount())
.elapsedMillis(resp.getElapsedMillis())
.operatorId(operator)
.operatorName(operator)
.build();
report(req);
}
public void reportImport(DataSheetClient.ImportResp resp) {
String operator = resp.getOperator();
JSONObject jsonObject = new JSONObject(new HashMap<>(BeanMap.create(resp)));
ReportReq req = ReportReq.builder()
.scene(resp.getScene())
.action(ReportReq.Action.IMPORT)
.resp(jsonObject.fluentRemove("lines").fluentRemove("meta"))
.rowCount(resp.getRowCount().longValue())
.elapsedMillis(resp.getElapsedMillis())
.operatorId(operator)
.operatorName(operator)
.build();
report(req);
}
private boolean report(ReportReq req) {
return true;
}
@Data
@lombok.Builder
@NoArgsConstructor
@AllArgsConstructor
private static class ReportReq {
String appName;
String scene;
Action action;
Map resp;
Long rowCount;
Long elapsedMillis;
String filePath;
String operatorId;
String operatorName;
String operatorTenantId;
public enum Action {
IMPORT,
EXPORT;
}
}
}

698
excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetExporter.java Normal file
View File

@ -0,0 +1,698 @@
package cn.axzo.foundation.excel.support.impl;
import cn.axzo.foundation.excel.support.DataSheetClient;
import cn.axzo.foundation.exception.BusinessException;
import cn.axzo.foundation.page.PageReq;
import cn.axzo.foundation.page.PageResp;
import cn.axzo.foundation.result.ResultCode;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.metadata.Head;
import com.alibaba.excel.metadata.data.WriteCellData;
import com.alibaba.excel.util.StringUtils;
import com.alibaba.excel.write.builder.ExcelWriterBuilder;
import com.alibaba.excel.write.handler.AbstractCellWriteHandler;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.excel.write.metadata.holder.WriteSheetHolder;
import com.alibaba.excel.write.metadata.holder.WriteTableHolder;
import com.alibaba.excel.write.style.AbstractCellStyleStrategy;
import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy;
import com.alibaba.excel.write.style.column.SimpleColumnWidthStyleStrategy;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.common.base.Strings;
import com.google.common.collect.*;
import com.opencsv.CSVWriter;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.CharEncoding;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.ss.util.CellRangeAddressList;
import org.apache.poi.xssf.usermodel.XSSFClientAnchor;
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*;
import static cn.axzo.foundation.excel.support.DataSheetClient.Meta.*;
@Slf4j
public class DataSheetExporter extends DataSheetClient.ExporterBuilder implements DataSheetClient.Exporter {
/** 完成后处理事件. */
@Setter
private Consumer<DataSheetClient.ExportResp> onCompleted;
@Override
public DataSheetClient.Exporter build() {
if (!CollectionUtils.isEmpty(this.fields())) {
if (this.columnMap() != null) {
throw new RuntimeException("column map is present, please remove column or fields");
}
if (this.meta() != null) {
throw new RuntimeException("meta is present, please remove meta or fields");
}
ImmutableMap<String, BiFunction<JSONObject, Integer, Object>> columns = this.fields().stream()
.collect(ImmutableMap.toImmutableMap(DataSheetClient.ExportField::getHeader, DataSheetClient.ExportField::getReader));
this.columnMap(columns);
ImmutableMap<String, List<String>> multiLineHeadMap = this.fields().stream()
.filter(f -> !CollectionUtils.isEmpty(f.getHeaderLines()))
.collect(ImmutableMap.toImmutableMap(DataSheetClient.ExportField::getHeader, DataSheetClient.ExportField::getHeaderLines));
this.multiLineHeadMap(multiLineHeadMap);
DataSheetClient.Meta meta = DataSheetClient.Meta.builder()
.templateCode(this.scene())
.version(Optional.ofNullable(this.version()).orElse("00"))
.cellMetas(this.fields().stream().map(DataSheetClient.ExportField::toCellMeta).collect(Collectors.toList()))
.build();
this.meta(meta);
}
Preconditions.checkArgument(this.scene() != null, "scene不能为空");
Preconditions.checkArgument(this.format() != null, "format不能为空");
Preconditions.checkArgument(this.rowSupplier() != null, "rowSupplier不能为空");
Preconditions.checkArgument(this.columnMap() != null, "columnMap不能为空");
return this;
}
@Override
public DataSheetClient.ExportResp export(OutputStream outputStream) {
Preconditions.checkArgument(outputStream != null, "outputStream不能为空");
long exportedCount = 0;
Exporter exporter = null;
Stopwatch stopwatch = Stopwatch.createStarted();
String errorMsg = "";
try {
exporter = getDataExporter(outputStream);
long pageNumber = 1;
while (limit() > exportedCount) {
// 每页最多允许数量
long pageMaxSize = Math.min(limit() - exportedCount, pageSize());
if (pageMaxSize <= 0) {
break;
}
PageReq pageParam = PageReq.builder()
.page((int) pageNumber)
// 这里使用pageSize，而不是pageMaxSize，否则会导致最后一次查询的数据和前面的查询数据重复
.pageSize(pageSize().intValue())
.build();
if (debugEnabled()) {
log.info("-------ExportClient[{}]------, request data, pageParam={}", scene(), pageParam);
}
PageResp<JSONObject> result = rowSupplier().apply(pageParam);
// nextPageMaxSize少于查询的pageSize，说明达到了最大导出数，只提取允许的数量
if (result.getData().size() > pageMaxSize && pageMaxSize < pageSize()) {
result = PageResp.<JSONObject>builder().current(result.getCurrent())
.size(result.getSize()).total(result.getTotal())
.data(result.getData().subList(0, (int) pageMaxSize))
.build();
if (debugEnabled()) {
log.info("-------ExportClient[{}]------, result data, and split result size to pageMaxSize={}, size={}",
scene(), pageMaxSize, result.getData().size());
}
} else {
if (debugEnabled()) {
log.info("-------ExportClient[{}]------, result data, size={}",
scene(), result.getData().size());
}
}
if (pageNumber == 1) {
if (!CollectionUtils.isEmpty(multiLineHeadMap())) {
List<List<String>> multiLineHeads = columnMap().keySet().stream()
.map(headName -> multiLineHeadMap().getOrDefault(headName, ImmutableList.of(headName)))
.collect(Collectors.toList());
exporter.writeMultiLineHeads(multiLineHeads);
} else {
exporter.writeHead(Lists.newArrayList(columnMap().keySet()));
}
// 第一页数据需要写顶部提示内容及表头
if (topHintsSupplier() != null) {
exporter.writeTopHints(topHintsSupplier().apply(result));
}
}
writeData(exporter, result);
exportedCount += result.getData().size();
if (onProgress() != null) {
//XXX： 不知道最大的页数，这里模拟下进度，保证不超过 100%
int progress = Math.min((int) pageNumber, 90);
onProgress().accept(progress, "");
if (debugEnabled()) {
log.info("-------ExportClient[{}]------, progress={}", scene(), progress);
}
}
if (result.getData().size() < result.getSize()) {
break;
}
pageNumber += 1;
}
} catch (Exception e) {
errorMsg = e.getMessage();
if (!StringUtils.isEmpty(errorMsg) && onProgress() != null) {
onProgress().accept(-1, errorMsg);
log.error("-------ExportClient ERROR [{}]------, progress={}", scene(), -1);
}
throw e;
} finally {
if (exporter != null) {
exporter.finish();
if (StringUtils.isEmpty(errorMsg) && onProgress() != null) {
onProgress().accept(100, "");
if (debugEnabled()) {
log.info("-------ExportClient[{}]------, progress={}", scene(), 100);
}
}
}
}
stopwatch.stop();
DataSheetClient.ExportResp exportResp = DataSheetClient.ExportResp.builder()
.scene(scene())
.exportedCount(exportedCount)
.rowCount(exporter.getRowCount())
.columnCount(exporter.getColumnCount())
.dataRowCount(exporter.getDataRowCount())
.elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS))
.operator(operator())
.build();
if (onCompleted != null) {
onCompleted.accept(exportResp);
}
log.info("-------ExportClient[{}]------, finish export, exportResp={}", scene(), exportResp);
return exportResp;
}
@Override
public DataSheetClient.ExportResp export(HttpServletResponse response) {
return export(response, null);
}
@Override
public DataSheetClient.ExportResp export(HttpServletResponse response, String filename) {
Preconditions.checkArgument(response != null, "response不能为空");
if (filename == null) {
filename = getFilename(format().getSuffix());
}
try {
filename = URLEncoder.encode(filename, CharEncoding.UTF_8);
} catch (UnsupportedEncodingException e) {
throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出失败，请稍后重试");
}
response.setCharacterEncoding(CharEncoding.UTF_8);
response.setContentType(format().getContentType());
try {
response.setHeader("Content-Disposition", "attachment;filename=" + filename);
log.info("-------ExportClient[{}]------, write to servlet response, contentType={}, filename={}",
scene(), format().getContentType(), filename);
return export(response.getOutputStream());
} catch (Exception ex) {
log.error("导出数据异常，文件名：" + filename, ex);
throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出失败，请稍后重试");
}
}
private Exporter getDataExporter(OutputStream outputStream) {
return format() == DataSheetClient.ExportFormat.CSV ?
new CsvExporter(outputStream, scene(), debugEnabled()) :
new ExcelExporter(sheetName(), outputStream, scene(), meta(), debugEnabled(), autoTrim());
}
private String getFilename(String suffix) {
if (fileName() != null) {
return fileName();
}
String formatDatetime = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").format(LocalDateTime.now());
return (Strings.isNullOrEmpty(sheetName()) ? "" : sheetName() + "-" + formatDatetime) + suffix;
}
private void writeData(Exporter exporter, PageResp<JSONObject> page) {
if (CollectionUtils.isEmpty(page.getData())) {
return;
}
List<JSONObject> rowPage = page.getData();
if (rowConverter() != null) {
rowPage = page.getData().stream()
.map(row -> {
List<JSONObject> converted = rowConverter().apply(row);
if (debugEnabled()) {
log.info("-------ExportClient[{}]------, row converted, row={}, converted={}",
scene(), row, converted);
}
return converted;
})
.flatMap(Collection::stream)
.collect(Collectors.toList());
}
if (pageConverter() != null) {
final List<JSONObject> originRowPage = rowPage;
rowPage = pageConverter().apply(rowPage);
if (debugEnabled()) {
log.info("-------ExportClient[{}]------, page converted, old={}, converted={}",
scene(), originRowPage, rowPage);
}
}
int rowIndex = (int) exporter.getDataRowCount();
List<JSONObject> finalRowPage = rowPage;
List<List<Object>> rows = IntStream.range(0, finalRowPage.size())
.mapToObj(index -> columnMap().values().stream()
.map(v -> v.apply(finalRowPage.get(index), rowIndex + index))
.collect(Collectors.toList()))
.collect(Collectors.toList());
rows.forEach(exporter::writeRow);
}
private class CsvExporter extends RowColumnCounter implements Exporter {
private CSVWriter writer;
private String scene;
private boolean debugEnabled;
private CsvExporter(OutputStream outputStream, String scene, boolean debugEnabled) {
try {
// encoding
outputStream.write(0xef);
outputStream.write(0xbb);
outputStream.write(0xbf);
} catch (IOException e) {
throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出数据失败");
}
writer = new CSVWriter(new OutputStreamWriter(outputStream));
this.scene = scene;
this.debugEnabled = debugEnabled;
}
@Override
public void writeTopHints(List<String> topHints) {
if (CollectionUtils.isEmpty(topHints)) {
return;
}
topHints.forEach(t -> writer.writeNext(new String[]{t}));
countTopHints(topHints.size());
if (debugEnabled) {
log.info("-------ExportClient[{}]------, write top={}", scene, topHints);
}
}
@Override
public void writeHead(List<String> head) {
writer.writeNext(head.toArray(new String[0]));
countHead(head.size());
if (debugEnabled) {
log.info("-------ExportClient[{}]------, write head={}", scene, head);
}
}
@Override
public void writeMultiLineHeads(List<List<String>> multiLineHeads) {
int lineNum = multiLineHeads.stream().mapToInt(List::size).max().orElse(0);
IntStream.range(0, lineNum).forEach(i -> {
List<String> heads = multiLineHeads.stream().map(head -> {
// 当表头的行数小于最大行数时，最上面的行中表头内容写入空字符串
int line = i - lineNum + head.size();
return line < 0 ? StringUtils.EMPTY : head.get(line);
}).collect(Collectors.toList());
writeHead(heads);
});
if (debugEnabled) {
log.info("-------ExportClient[{}]------, write multi line heads={}", scene, multiLineHeads);
}
}
@Override
public void writeRow(List<Object> row) {
String[] arrayContents = row.stream()
.map(c -> c == null ? StringUtils.EMPTY : c.toString())
.toArray(String[]::new);
writer.writeNext(arrayContents);
countRow(row.size());
if (debugEnabled) {
log.info("-------ExportClient[{}]------, write row={}", scene, row);
}
}
@Override
public void finish() {
try {
writer.close();
} catch (IOException e) {
throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出数据失败");
}
}
}
private class ExcelExporter extends RowColumnCounter implements Exporter {
private final Integer DEFAULT_WIDTH = 6;
private ExcelWriter writer;
private WriteSheet sheet;
private String scene;
private boolean debugEnabled;
private HeaderCommentWriteHandler commentWriteHandler;
private DataValidationWriteHandler validationWriteHandler;
private int topHintsRowCount = 0;
private ExcelExporter(String sheetName, OutputStream outputStream, String scene, DataSheetClient.Meta meta,
boolean debugEnabled, Boolean autoTrim) {
ExcelWriterBuilder excelWriterBuilder = EasyExcel.write(outputStream);
if (Objects.nonNull(meta) && !CollectionUtils.isEmpty(meta.getCellMetas())) {
Integer width = meta.getCellMetas().get(0).getWidth();
excelWriterBuilder.registerWriteHandler(new SimpleColumnWidthStyleStrategy(Objects.nonNull(width) ?
width : DEFAULT_WIDTH));
} else {
excelWriterBuilder.registerWriteHandler(new LongestMatchColumnWidthStyleStrategy());
}
if (meta != null) {
excelWriterBuilder.registerWriteHandler(new AbstractCellStyleStrategy() {
private CellStyle wrapStyle;
@Override
protected void setHeadCellStyle(Cell cell, Head head, Integer relativeRowIndex) {
}
@Override
protected void setContentCellStyle(Cell cell, Head head, Integer relativeRowIndex) {
if (cell.getColumnIndex() >= meta.getCellMetas().size()) {
return;
}
DataSheetClient.CellMeta cellMeta = meta.getCellMetas().get(cell.getColumnIndex());
if (cellMeta != null && BooleanUtils.isTrue(cellMeta.getWrapText())) {
cell.setCellStyle(wrapStyle);
}
}
});
commentWriteHandler = new HeaderCommentWriteHandler(meta);
validationWriteHandler = new DataValidationWriteHandler(meta);
excelWriterBuilder.registerWriteHandler(new AbstractCellWriteHandler() {
@Override
public void afterCellDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, List<WriteCellData<?>> cellDataList, Cell cell, Head head, Integer relativeRowIndex, Boolean isHead) {
boolean isHeadRow = cell.getRowIndex() == topHintsRowCount;
commentWriteHandler.afterCellDispose(cell, isHeadRow);
validationWriteHandler.afterCellDispose(cell, isHeadRow);
}
});
}
writer = excelWriterBuilder.autoTrim(autoTrim).build();
sheet = EasyExcel.writerSheet(sheetName).build();
this.scene = scene;
this.debugEnabled = debugEnabled;
}
@Override
public void writeTopHints(List<String> topHints) {
if (CollectionUtils.isEmpty(topHints)) {
return;
}
countTopHints(topHints.size());
topHintsRowCount += topHints.size();
writer.write(topHints.stream().map(ImmutableList::of).collect(Collectors.toList()), sheet);
if (debugEnabled) {
log.info("-------ExportClient[{}]------, write top={}", scene, topHints);
}
}
@Override
public void writeHead(List<String> head) {
sheet.setHead(head.stream().map(Lists::newArrayList).collect(Collectors.toList()));
countHead(head.size());
// XXX 写入head后需要调用一次write()确保excel中有数据，否则数据为空时将出现内容异常
writer.write(ImmutableList.of(), sheet);
if (debugEnabled) {
log.info("-------ExportClient[{}]------, write head={}", scene, head);
}
}
@Override
public void writeMultiLineHeads(List<List<String>> multiLineHeads) {
// XXX setHead()方法有副作用，内部逻辑会改变传入的list, 因此需要确保传入的list可操作
sheet.setHead(multiLineHeads.stream().map(Lists::newArrayList).collect(Collectors.toList()));
int lineNum = multiLineHeads.stream().mapToInt(List::size).max().orElse(0);
IntStream.range(0, lineNum).forEach(i -> countHead(multiLineHeads.size()));
// XXX 写入head后需要调用一次write()确保excel中有数据，否则数据为空时将出现内容异常
writer.write(ImmutableList.of(), sheet);
if (debugEnabled) {
log.info("-------ExportClient[{}]------, write multi line heads={}", scene, multiLineHeads);
}
}
@Override
public void writeRow(List<Object> row) {
writer.write(ImmutableList.of(row), sheet);
countRow(row.size());
if (debugEnabled) {
log.info("-------ExportClient[{}]------, write row={}", scene, row);
}
}
@Override
public void finish() {
writer.finish();
}
}
/**
* Excel添加头部的批注
* XXX: 此处不能继承AbstractCellWriteHandler 会导致 mybatis typehandler 扫描时被加载
* 导致老项目没有使用 easyexcel2.2 的项目启动失败。
*/
private static class HeaderCommentWriteHandler {
private static final Map<List<String>, String> rangeBoundTypes = ImmutableMap.of(
ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN), RANGE_TYPE_OPEN_OPEN,
ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED), RANGE_TYPE_OPEN_CLOSED,
ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN), RANGE_TYPE_CLOSED_OPEN,
ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED), RANGE_TYPE_CLOSED_CLOSED);
private String templateCode;
private String version;
private Map<String, DataSheetClient.CellMeta> cellMetaMap;
private Set<DataSheetClient.CellMeta.Type> typeWithParams = ImmutableSet.of(
DataSheetClient.CellMeta.Type.RANGE,
DataSheetClient.CellMeta.Type.DATE,
DataSheetClient.CellMeta.Type.DATETIME
);
private HeaderCommentWriteHandler(DataSheetClient.Meta meta) {
this.templateCode = meta.getTemplateCode();
this.version = meta.getVersion();
this.cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), DataSheetClient.CellMeta::getName);
}
public void afterCellDispose(Cell cell, boolean isHead) {
// 只添加表头行的批注
if (!isHead) {
return;
}
List<String> comments = Lists.newArrayList();
if (cell.getColumnIndex() == 0) {
String content = Joiner.on("_").join(ImmutableList.of(templateCode, version));
comments.add(buildComment(PATTERN_VERSION_OPEN, content));
}
DataSheetClient.CellMeta cellMeta = cellMetaMap.get(cell.getStringCellValue());
if (cellMeta == null) {
return;
}
// 格式： key_type_mandatory
String content = Joiner.on("_").join(ImmutableList.of(
cellMeta.getKey(),
cellMeta.getType().getCode(),
Boolean.TRUE.equals(cellMeta.getMandatory()) ? "1" : "0"));
if (typeWithParams.contains(cellMeta.getType())) {
content = content.concat("_").concat(getParamsByType(cellMeta));
}
comments.add(buildComment(PATTERN_CELL_OPEN, content));
// 一个Cell只能写入一次Comment
writeComment(cell, Joiner.on("\n").join(comments));
}
private String getParamsByType(DataSheetClient.CellMeta cellMeta) {
switch (cellMeta.getType()) {
case RANGE:
String lowerType = cellMeta.getParams().getString(EXT_KEY_RANGE_LOWER_TYPE);
String upperType = cellMeta.getParams().getString(EXT_KEY_RANGE_UPPER_TYPE);
return rangeBoundTypes.get(ImmutableList.of(lowerType, upperType));
case DATE:
case DATETIME:
return cellMeta.getParams().getString(EXT_KEY_DATETIME_FORMAT);
default:
return StringUtils.EMPTY;
}
}
private void writeComment(Cell cell, String comment) {
int columnIndex = cell.getColumnIndex();
int rowIndex = cell.getRowIndex();
Drawing<?> drawing = cell.getSheet().createDrawingPatriarch();
Comment cellComment = drawing.createCellComment(new XSSFClientAnchor(0, 0, 0, 0,
// 控制批注的位置
columnIndex, rowIndex, columnIndex + 2, rowIndex + 2));
cellComment.setString(new XSSFRichTextString(comment));
cell.setCellComment(cellComment);
}
private String buildComment(String patternOpen, String content) {
return patternOpen.concat(content).concat(PATTERN_CLOSE);
}
}
/**
* Excel添加校验数据
* XXX: 此处不能继承AbstractCellWriteHandler 会导致 mybatis typehandler 扫描时被加载
* 导致老项目没有使用 easyexcel2.2 的项目启动失败。
*/
private static class DataValidationWriteHandler {
private Map<String, DataSheetClient.CellMeta> cellMetaMap;
private DataValidationWriteHandler(DataSheetClient.Meta meta) {
this.cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), DataSheetClient.CellMeta::getName);
}
public void afterCellDispose(Cell cell, boolean isHead) {
// 只在写入表头数据时才写入校验数据
if (!isHead) {
return;
}
DataSheetClient.CellMeta cellMeta = cellMetaMap.get(cell.getStringCellValue());
// 目前仅支持将可选值作为下拉选项的校验数据
if (cellMeta == null || CollectionUtils.isEmpty(cellMeta.getOptions())) {
return;
}
String[] options = cellMeta.getOptions().stream().map(String::valueOf).toArray(String[]::new);
Sheet currentSheet = cell.getSheet();
Workbook workbook = currentSheet.getWorkbook();
// 直接将校验数据作为列表项的方式有数据大小限制(不超过256个字符), 因此需要将校验数据放在一个新增的隐藏表单中
// 以表头的名字来命名新增的表单
String sheetName = cell.getStringCellValue();
Sheet validationSheet = workbook.createSheet(sheetName);
workbook.setSheetHidden(workbook.getSheetIndex(validationSheet), true);
// 将可选项放在新增的隐藏表单里
for (int i = 0; i < options.length; i++) {
Row rowInValidationSheet = validationSheet.createRow(i);
Cell cellInValidationSheet = rowInValidationSheet.createCell(0);
cellInValidationSheet.setCellValue(options[i]);
}
// 设置列表校验数据的公式 [表名!开始列开始行:结束列结束行]
String listFormula = sheetName + "!$A$1:$A$" + options.length;
DataValidationHelper dataValidationHelper = currentSheet.getDataValidationHelper();
DataValidationConstraint constraint = dataValidationHelper.createFormulaListConstraint(listFormula);
// 从下一行开始10000行数据都增加校验数据
CellRangeAddressList addressList = new CellRangeAddressList(cell.getRowIndex() + 1,
cell.getRowIndex() + 10001, cell.getColumnIndex(), cell.getColumnIndex());
DataValidation dataValidation = dataValidationHelper.createValidation(constraint, addressList);
dataValidation.setSuppressDropDownArrow(true);
dataValidation.setShowErrorBox(true);
currentSheet.addValidationData(dataValidation);
}
}
private class RowColumnCounter {
@Getter
private long rowCount = 0;
@Getter
private long columnCount = 0;
@Getter
private long dataRowCount = 0;
protected void countTopHints(int topHintsRowCount) {
rowCount += topHintsRowCount;
columnCount = Math.max(columnCount, 1);
}
protected void countHead(int headColumnCount) {
rowCount += 1;
columnCount = Math.max(columnCount, headColumnCount);
}
protected void countRow(int columnCount) {
rowCount += 1;
this.columnCount = Math.max(this.columnCount, columnCount);
dataRowCount += 1;
}
}
private interface Exporter {
void writeTopHints(List<String> topHints);
void writeHead(List<String> head);
void writeMultiLineHeads(List<List<String>> multiLineHeads);
void writeRow(List<Object> row);
void finish();
long getRowCount();
long getColumnCount();
long getDataRowCount();
}
}

595
excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetImporter.java Normal file
View File

@ -0,0 +1,595 @@
package cn.axzo.foundation.excel.support.impl;
import cn.axzo.foundation.excel.support.DataSheetClient;
import cn.axzo.foundation.exception.BusinessException;
import cn.axzo.foundation.util.Regex;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.enums.CellExtraTypeEnum;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.metadata.CellExtra;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Stopwatch;
import com.google.common.base.Strings;
import com.google.common.collect.*;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import java.io.InputStream;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*;
import static cn.axzo.foundation.excel.support.DataSheetClient.Meta.*;
import static cn.axzo.foundation.excel.support.DataSheetClient.ResultCode.IMPORT_CELL_CONVERT_FAILED;
/**
* 使用ali的EasyExcel来读入excel/csv文件
*
* @author yuanyi
*/
@Slf4j
public class DataSheetImporter extends DataSheetClient.ImporterBuilder {
private static final String LINE_KEY_ERRORS = "errors";
private static final String LINE_KEY_ROW_INDEX = "rowIndex";
/**
* 完成后处理事件.
*/
@Setter
private Consumer<DataSheetClient.ImportResp> onCompleted;
@Override
public DataSheetClient.Importer build() {
Preconditions.checkArgument(this.scene() != null, "scene不能为空");
Preconditions.checkArgument(this.format() != null, "format不能为空");
// TODO: 当支持更多format的时候，需要生成对应的Importer实例
return ExcelImporter.builder()
.scene(scene())
.tableName(tableName())
.meta(meta())
.ignoreHeaderMeta(ignoreHeaderMeta())
.debugEnabled(debugEnabled())
.allowMaxLineCount(allowMaxLineCount())
.headerConverter(headerConverter())
.includeLineErrors(includeLineErrors())
.ignoreUnknownColumn(ignoreUnknownColumn())
.autoTrim(autoTrim())
.onCompleted(onCompleted)
.operator(operator())
.build();
}
@Builder
private static class ExcelImporter implements DataSheetClient.Importer {
private static final Map<String, List<String>> rangeBoundTypes = ImmutableMap.of(
RANGE_TYPE_OPEN_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN),
RANGE_TYPE_OPEN_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED),
RANGE_TYPE_CLOSED_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN),
RANGE_TYPE_CLOSED_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED));
private String scene;
private String tableName;
private DataSheetClient.Meta meta;
private Boolean ignoreHeaderMeta;
private boolean debugEnabled;
private Integer allowMaxLineCount;
private boolean includeLineErrors;
private boolean ignoreUnknownColumn;
private Boolean autoTrim;
private Function<String, String> headerConverter;
private Consumer<DataSheetClient.ImportResp> onCompleted;
private String operator;
@Override
public DataSheetClient.ImportResp<JSONObject> readAll(InputStream inputStream) {
Stopwatch stopwatch = Stopwatch.createStarted();
NoModelDataListener dataListener = new NoModelDataListener(headerConverter, autoTrim);
EasyExcel.read(inputStream, dataListener)
.autoTrim(autoTrim)
.extraRead(CellExtraTypeEnum.COMMENT).sheet(tableName).doRead();
if (allowMaxLineCount != null && dataListener.lines.size() > allowMaxLineCount) {
throw DataSheetClient.ResultCode.IMPORT_LINES_REACHED_LIMIT
.toException("导入的数据超过了最大行数" + allowMaxLineCount);
}
if (BooleanUtils.isNotTrue(this.ignoreHeaderMeta)) {
// 聚合来自入参和文件批注的meta信息；如果都存在，使用入参的meta覆盖文件批注的meta
this.meta = mergeMeta(this.meta, parseMetaFromData(dataListener));
}
filterHeadMap(dataListener);
filterLines(dataListener);
validateHeaders(dataListener);
validateMeta(dataListener);
List<String> headers = parseHeaders(dataListener);
List<JSONObject> lines = parseLines(dataListener);
if (!includeLineErrors) {
// 没有设置includeLineErrors时，抛第一个发现的异常
Optional<JSONObject> errorLine = lines.stream()
.filter(line -> line.containsKey(LINE_KEY_ERRORS))
.findFirst();
if (errorLine.isPresent()) {
JSONObject error = errorLine.get().getJSONArray(LINE_KEY_ERRORS).getJSONObject(0);
Integer rowIndex = lines.indexOf(errorLine.get());
Integer columnIndex = error.getInteger("columnIndex");
String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1,
error.getString("columnName"), error.getString("errorMsg"));
throw new DataSheetClient.DataSheetException(error.getString("subErrorCode"), errMsg, rowIndex, columnIndex);
}
}
DataSheetClient.ImportResp resp = DataSheetClient.ImportResp.<JSONObject>builder()
.scene(scene)
.templateCode(Optional.ofNullable(meta).map(DataSheetClient.Meta::getTemplateCode).orElse(null))
.version(Optional.ofNullable(meta).map(DataSheetClient.Meta::getVersion).orElse(null))
.headers(headers)
.lines(lines)
.headerRowCount(1)
.rowCount(lines.size())
.columnCount(headers.size())
.meta(meta)
.elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS))
.operator(operator)
.build();
if (null != onCompleted) {
onCompleted.accept(resp);
}
return resp;
}
private void filterHeadMap(NoModelDataListener dataListener) {
if (meta == null) {
return;
}
Map<Integer, String> headMap = dataListener.getHeadMap();
Set<String> ignoreColumnNames = meta.getIgnoreColumnNames();
// 收集需要过滤的列号
Set<Integer> ignoreColumnIndexes = headMap.entrySet().stream()
.filter(entry -> ignoreColumnNames.contains(entry.getValue()))
.map(Map.Entry::getKey).collect(Collectors.toSet());
// 加上指定过滤的列号
ignoreColumnIndexes.addAll(meta.getIgnoreColumnIndexes());
ignoreColumnIndexes.forEach(headMap::remove);
}
private void filterLines(NoModelDataListener dataListener) {
if (meta == null) {
return;
}
Set<Integer> toRemoveLines = ImmutableSet.copyOf(meta.getIgnoreRowIndexes());
List<Map<Integer, String>> lines = dataListener.getLines();
dataListener.setLines(IntStream.range(0, lines.size())
// 这里要加1，是因为removeIndex是整个文档的行数来计算，包括了header
// 但是lines的数据，已经排除了header
.filter(index -> !toRemoveLines.contains(index + 1))
.mapToObj(lines::get)
.collect(Collectors.toList()));
}
private DataSheetClient.Meta parseMetaFromData(NoModelDataListener dataListener) {
Map<Integer, String> headMap = dataListener.getHeadMap();
List<CellExtra> headComments = dataListener.getCellComments().stream()
// 获取第一行Head的批注信息
.filter(cellExtra -> cellExtra.getRowIndex() == 0)
// 过滤带有Cell参数配置的批注信息
.filter(cellExtra -> !Strings.isNullOrEmpty(StringUtils.substringBetween(cellExtra.getText(),
PATTERN_CELL_OPEN, PATTERN_CLOSE)))
// 排序，方便找到第一列，获取templateCode与version
.sorted(Comparator.comparing(CellExtra::getColumnIndex))
.collect(Collectors.toList());
// 没有批注信息，直接返回null；会以String来解析值
if (headComments.isEmpty()) {
return null;
}
// 从文件中解析meta信息
JSONObject codeAndVersion = parseTemplateCodeAndVersion(headComments);
List<DataSheetClient.CellMeta> cellMetas = headComments.stream()
.map(cellExtra -> parseCellMeta(cellExtra.getText(), headMap.get(cellExtra.getColumnIndex())))
.filter(Objects::nonNull)
.collect(Collectors.toList());
if (cellMetas.isEmpty()) {
return null;
}
return DataSheetClient.Meta.builder()
.templateCode(codeAndVersion.getString("templateCode"))
.version(codeAndVersion.getString("version"))
.cellMetas(cellMetas)
.ignoreColumnIndexes(parseIgnoreColumns(dataListener))
.ignoreRowIndexes(parseIgnoreRows(dataListener))
.build();
}
private DataSheetClient.Meta mergeMeta(DataSheetClient.Meta metaFromParam, DataSheetClient.Meta metaFromHeader) {
if (metaFromHeader == null || metaFromParam == null) {
return metaFromHeader == null ? metaFromParam : metaFromHeader;
}
// 在两者都存在的时候，以文件的meta为准；仅仅将入参的CellMetas覆盖文件的CellMetas
if (metaFromHeader.getCellMetas() == null) {
metaFromHeader.setCellMetas(metaFromParam.getCellMetas());
return metaFromHeader;
}
// 使用入参中定义的列信息，覆盖文件的cellMetas
Map<String, DataSheetClient.CellMeta> cellMetaMap =
Maps.uniqueIndex(metaFromParam.getCellMetas(), DataSheetClient.CellMeta::getKey);
List<DataSheetClient.CellMeta> cellMetas = metaFromHeader.getCellMetas().stream()
.map(cellMeta -> cellMetaMap.getOrDefault(cellMeta.getKey(), cellMeta))
.collect(Collectors.toList());
metaFromHeader.setCellMetas(cellMetas);
return metaFromHeader;
}
private List<Integer> parseIgnoreRows(NoModelDataListener dataListener) {
return dataListener.getCellComments().stream()
.map(cellExtra -> {
String ignoreKey = StringUtils.substringBetween(cellExtra.getText(),
PATTERN_IGNORE_OPEN, PATTERN_CLOSE);
if (IGNORE_ROW_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) {
return cellExtra.getRowIndex();
}
return null;
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
private List<Integer> parseIgnoreColumns(NoModelDataListener dataListener) {
return dataListener.getCellComments().stream()
.map(cellExtra -> {
String ignoreKey = StringUtils.substringBetween(cellExtra.getText(),
PATTERN_IGNORE_OPEN, PATTERN_CLOSE);
if (IGNORE_COLUMN_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) {
return cellExtra.getColumnIndex();
}
return null;
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
public JSONObject parseTemplateCodeAndVersion(List<CellExtra> headComments) {
// 从整个头部的批注，获取模版code和版本信息
Optional<String> parsedText = headComments.stream()
.map(comment -> StringUtils.substringBetween(comment.getText(), PATTERN_VERSION_OPEN, PATTERN_CLOSE))
.filter(str -> !Strings.isNullOrEmpty(str))
.findFirst();
if (!parsedText.isPresent()) {
throw DataSheetClient.ResultCode.IMPORT_PARSE_MISSING_VERSION.toException();
}
// 格式为"CODE_VERSION"
List<String> values = Splitter.on("_").splitToList(parsedText.get());
if (values.size() != 2) {
throw DataSheetClient.ResultCode.IMPORT_PARSE_VERSION_FORMAT_ERROR.toException();
}
return new JSONObject()
.fluentPut("templateCode", values.get(0))
.fluentPut("version", values.get(1));
}
private DataSheetClient.CellMeta parseCellMeta(String text, String name) {
String value = StringUtils.substringBetween(text, PATTERN_CELL_OPEN, PATTERN_CLOSE);
if (Strings.isNullOrEmpty(value)) {
throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_MISSING_TYPE
.toException("没有找到列[{}]的类型信息", name);
}
List<String> values = Splitter.on("_").splitToList(value);
// 格式： key_type_mandatory_params...
if (values.size() < 3) {
throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_FORMAT_ERROR
.toException("列[{}]类型的批注格式不对[{}]", name, text);
}
DataSheetClient.CellMeta.Type type = DataSheetClient.CellMeta.Type.from(values.get(1));
JSONObject params = new JSONObject();
if (type == DataSheetClient.CellMeta.Type.RANGE) {
if (values.size() != 4 || Strings.isNullOrEmpty(values.get(3))) {
throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR
.toException("列[{}]范围类型的批注格式不对[{}]", name, text);
}
List<String> boundType = rangeBoundTypes.get(values.get(3));
if (boundType == null) {
throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR
.toException(String.format("列[{}]范围类型的值不对", name));
}
params.put(EXT_KEY_RANGE_LOWER_TYPE, boundType.get(0));
params.put(EXT_KEY_RANGE_UPPER_TYPE, boundType.get(1));
}
JSONObject ext = new JSONObject();
value = StringUtils.substringBetween(text, PATTERN_EXTRA_OPEN, PATTERN_CLOSE);
if (!Strings.isNullOrEmpty(value)) {
ext.putAll(Splitter.on("&").withKeyValueSeparator("=").split(value));
}
return DataSheetClient.CellMeta.builder()
.key(values.get(0))
.name(name)
.type(type)
.mandatory("1".equals(values.get(2)))
.params(params)
.ext(ext)
.build();
}
private List<String> parseHeaders(NoModelDataListener dataListener) {
return dataListener.getHeadMap().keySet().stream()
.sorted()
.map(key -> Strings.nullToEmpty(dataListener.getHeadMap().get(key)))
.collect(Collectors.toList());
}
private List<JSONObject> parseLines(NoModelDataListener dataListener) {
// 如果没有找到meta信息，按照key=header(很可能是中文), 类型就为String
if (this.meta == null) {
Map<Integer, String> headerMap = dataListener.getHeadMap();
return dataListener.getLines().stream()
.map(line -> new JSONObject().fluentPutAll(line.entrySet().stream()
.map(entry -> Pair.of(headerMap.get(entry.getKey()), Strings.nullToEmpty(entry.getValue())))
.collect(Collectors.toMap(Pair::getKey, Pair::getValue))))
.collect(Collectors.toList());
}
// 根据meta来校验cell的类型
Map<Integer, String> headerMap = dataListener.getHeadMap();
Map<String, DataSheetClient.CellMeta> cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(),
DataSheetClient.CellMeta::getName);
List<Map<Integer, String>> lines = dataListener.getLines();
return IntStream.range(0, lines.size())
.mapToObj(lineIndex -> {
Map<Integer, String> line = lines.get(lineIndex);
// 收集每一行每一列的转换结果
List<ColumnConvertResp> convertRespList = headerMap.entrySet().stream()
.filter(e -> {
//XXX 当支持忽略列的时候, 只处理定义了cellMeta的数据
if (ignoreUnknownColumn) {
return cellMetaMap.containsKey(e.getValue());
}
return true;
})
.map(entry -> {
Integer columnIndex = entry.getKey();
String header = entry.getValue();
DataSheetClient.CellMeta cellMeta = cellMetaMap.get(header);
return convertType(cellMeta, line.get(columnIndex), lineIndex, columnIndex);
})
.collect(Collectors.toList());
JSONObject row = new JSONObject()
.fluentPutAll(convertRespList.stream()
.filter(ColumnConvertResp::getSuccess)
// convertValue可能为null, 有非必需的字段
.collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll));
convertRespList.stream()
.filter(r -> BooleanUtils.isTrue(r.getSuccess()))
.filter(r -> r.getCellMeta().getCellValidator() != null)
.forEach(r -> {
try {
r.getCellMeta().getCellValidator().accept(r.getConvertedValue(), row);
} catch (BusinessException e) {
log.warn("failed to validate cell, resp={}", r, e);
String subErrorCode = null;
if (e instanceof DataSheetClient.DataSheetException) {
subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode();
}
r.setSuccess(false);
r.setErrorCode(e.getErrorCode());
r.setErrorMsg(e.getErrorMsg());
r.setSubErrorCode(subErrorCode);
}
});
Map<Boolean, List<ColumnConvertResp>> convertRespMap = convertRespList.stream()
.collect(Collectors.groupingBy(ColumnConvertResp::getSuccess));
JSONObject convertedLine = new JSONObject()
.fluentPutAll(convertRespMap.getOrDefault(Boolean.TRUE, ImmutableList.of()).stream()
// convertValue可能为null, 有非必需的字段
.collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll));
if (convertRespMap.get(Boolean.FALSE) != null) {
// 转换失败的，将失败信息放到errors字段中
convertedLine.put(LINE_KEY_ERRORS, convertRespMap.get(Boolean.FALSE).stream()
.map(ColumnConvertResp::getError)
.collect(Collectors.toList()));
}
convertedLine.put(LINE_KEY_ROW_INDEX, lineIndex);
return convertedLine;
})
.collect(Collectors.toList());
}
private ColumnConvertResp convertType(DataSheetClient.CellMeta cellMeta, String rawValue, int rowIndex, int columnIndex) {
try {
return ColumnConvertResp.builder()
.success(true).cellMeta(cellMeta)
.rawValue(rawValue).convertedValue(cellMeta.convertType(rawValue))
.build();
} catch (BusinessException e) {
log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}",
cellMeta, rawValue, rowIndex, columnIndex, e);
String subErrorCode = null;
if (e instanceof DataSheetClient.DataSheetException) {
subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode();
}
return ColumnConvertResp.builder()
.success(false).cellMeta(cellMeta).rawValue(rawValue)
.columnIndex(columnIndex)
.errorCode(e.getErrorCode())
.errorMsg(e.getErrorMsg())
.subErrorCode(subErrorCode)
.build();
} catch (Exception e) {
log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}",
cellMeta, rawValue, rowIndex, columnIndex, e);
String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1,
cellMeta.getName(), Optional.ofNullable(e.getMessage())
.orElse(IMPORT_CELL_CONVERT_FAILED.getMessage()));
throw new DataSheetClient.DataSheetException(IMPORT_CELL_CONVERT_FAILED.getSubBizCode(), errMsg, rowIndex, columnIndex);
}
}
private void validateHeaders(NoModelDataListener dataListener) {
Map<Integer, String> headMap = dataListener.getHeadMap();
Set<String> headerNames = ImmutableSet.copyOf(headMap.values());
if (headerNames.size() != headMap.size()) {
List<String> columnNames = headMap.values().stream()
.collect(Collectors.groupingBy(Function.identity()))
.entrySet().stream()
.filter(grouped -> grouped.getValue().size() > 1)
.map(Map.Entry::getKey)
.collect(Collectors.toList());
throw DataSheetClient.ResultCode.IMPORT_COLUMN_DUPLICATED_NAME.toException(columnNames);
}
}
private void validateMeta(NoModelDataListener dataListener) {
if (this.meta == null) {
return;
}
if (meta.getCellMetas().size() != dataListener.getHeadMap().size()) {
if (dataListener.getHeadMap().size() > meta.getCellMetas().size()) {
if (!ignoreUnknownColumn) {
throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException();
}
}
long mandatoryColumnSize = meta.getCellMetas().stream()
.filter(cm -> BooleanUtils.isTrue(cm.getMandatory()))
.count();
if (dataListener.getHeadMap().size() < mandatoryColumnSize) {
throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException();
}
}
Set<String> headerNames = ImmutableSet.copyOf(dataListener.getHeadMap().values());
Set<String> cellNames = meta.getCellMetas().stream()
.map(DataSheetClient.CellMeta::getName)
.collect(Collectors.toSet());
if (!ignoreUnknownColumn && !headerNames.equals(cellNames)) {
Set<String> missingNames = Sets.difference(cellNames, headerNames);
Set<String> redundantNames = Sets.difference(headerNames, cellNames);
List<String> columnNames = Stream.of(missingNames, redundantNames)
.flatMap(Set::stream)
.collect(Collectors.toList());
throw DataSheetClient.ResultCode.IMPORT_COLUMN_NAME_NOT_MATCHED.toException(columnNames);
}
meta.getCellMetas().forEach(DataSheetClient.CellMeta::validate);
}
@Slf4j
@Data
private static class NoModelDataListener extends AnalysisEventListener<Map<Integer, String>> {
private Map<Integer, String> headMap = Maps.newHashMap();
private List<Map<Integer, String>> lines = Lists.newArrayList();
private List<CellExtra> cellComments = Lists.newArrayList();
private Function<String, String> headerConverter;
private Boolean autoTrim;
public NoModelDataListener(Function<String, String> headerConverter, Boolean autoTrim) {
this.headerConverter = headerConverter;
this.autoTrim = autoTrim;
}
@Override
public void invoke(Map<Integer, String> data, AnalysisContext context) {
lines.add(strip(data));
}
@Override
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
Map<Integer, String> convertedMap = headMap;
if (headerConverter != null) {
convertedMap = headMap.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> headerConverter.apply(e.getValue())));
}
this.headMap.putAll(strip(convertedMap));
}
private Map<Integer, String> strip(Map<Integer, String> data) {
if (BooleanUtils.isFalse(autoTrim)) {
return data;
}
return data.entrySet().stream()
.map(entry -> Maps.immutableEntry(entry.getKey(), StringUtils.strip(entry.getValue(), Regex.WHITESPACE_CHARS)))
// value有可能为null，不能直接用Collectors.toMap
.collect(Maps::newHashMap, (m, v) -> m.put(v.getKey(), v.getValue()), HashMap::putAll);
}
/**
* extra是在整个文件被解析后才会调用
*
* @param extra
* @param context
*/
@Override
public void extra(CellExtra extra, AnalysisContext context) {
if (extra.getType() == CellExtraTypeEnum.COMMENT) {
cellComments.add(extra);
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
}
}
@Builder
@Data
@NoArgsConstructor
@AllArgsConstructor
private static class ColumnConvertResp {
private Boolean success;
private DataSheetClient.CellMeta cellMeta;
private String rawValue;
private Object convertedValue;
private Integer columnIndex;
private String errorCode;
private String errorMsg;
private String subErrorCode;
public String getKey() {
return cellMeta.getKey();
}
public JSONObject getError() {
return new JSONObject()
.fluentPut("columnIndex", columnIndex)
.fluentPut("columnKey", cellMeta.getKey())
.fluentPut("columnName", cellMeta.getName())
.fluentPut("rawValue", rawValue)
.fluentPut("errorCode", errorCode)
.fluentPut("errorMsg", errorMsg)
.fluentPut("subErrorCode", subErrorCode);
}
}
}
}

5
gateway-support-lib/pom.xml
View File

@ -45,6 +45,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba.csp</groupId>
<artifactId>sentinel-core</artifactId>
<version>1.8.0</version>
</dependency>
</dependencies>
</project>

13
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/BizGatewayImpl.java
View File

@ -214,15 +214,6 @@ public class BizGatewayImpl implements BizGateway {
((StopWatchHook) hook).initialize();
}
private boolean isGrayStage(RequestContext requestContext) {
// TODO: correct gray stage
if (!CollectionUtils.isEmpty(requestContext.getHeaders())
&& StringUtils.equals(requestContext.getHeaders().get("gray_stage"), "true")) {
return true;
}
return false;
}
@Override
public Proxy findProxy(RequestContext context) {
initializeStopWatchHook();
@ -377,9 +368,7 @@ public class BizGatewayImpl implements BizGateway {
if (getRouteRules() == null) {
return ImmutableMap.of();
}
return getRouteRules().stream().map(e -> {
return buildProxy(e);
}).collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
return getRouteRules().stream().map(this::buildProxy).collect(Collectors.toMap(Pair::getKey, Pair::getValue));
}

18
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/entity/GlobalContext.java
View File

@ -2,6 +2,7 @@ package cn.axzo.foundation.gateway.support.entity;
import cn.axzo.foundation.enums.AppEnvEnum;
import cn.axzo.foundation.gateway.support.plugin.ProxyHookChain;
import cn.axzo.foundation.gateway.support.proxy.GateResponseSupplier;
import cn.axzo.foundation.gateway.support.utils.RpcClientProvider;
import cn.axzo.foundation.web.support.AppRuntime;
import com.google.common.base.Strings;
@ -34,20 +35,23 @@ public class GlobalContext {
@Getter
private AppEnvEnum gateEnv;
@Getter
final private RpcClientProvider rpcClientProvider;
private final RpcClientProvider rpcClientProvider;
@Getter
final private BiConsumer<String, Object[]> alertConsumer;
private final BiConsumer<String, Object[]> alertConsumer;
@Getter
final private Function<AppEnvEnum, List<Service>> serviceSupplier;
private final Function<AppEnvEnum, List<Service>> serviceSupplier;
/* 服务降级时默认返回 */
@Getter
private final Function<String, GateResponseSupplier> fallbackSupplier;
@Getter
/** 全局的代理Hook列表 */
final private ProxyHookChain proxyHookChain;
private final ProxyHookChain proxyHookChain;
@Getter
final private Long blockingMillis;
private final Long blockingMillis;
@Getter
final private Map<String, GateSettingResp.Proxy> localProxies;
private final Map<String, GateSettingResp.Proxy> localProxies;
@Getter
/** 网关代理配置最后更新时间 */
/* 网关代理配置最后更新时间 */
private Long version;
@Getter
/** 需要debug的URI列表 */

8
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiNotFoundException.java
View File

@ -1,22 +1,24 @@
package cn.axzo.foundation.gateway.support.exception;
import cn.axzo.foundation.exception.BusinessException;
/**
* API不存在导致代理失败的异常.
* <p>
* 实施时GateServer的ExceptionResolver可以拦截该异常做特殊处理, 如统一为HTTP标准状态码返回: ModelAndView.setStatus(HttpStatus.NOT_FOUND).
* </p>
*/
public class ApiNotFoundException extends RuntimeException {
public class ApiNotFoundException extends BusinessException {
private static final long serialVersionUID = 8821155936941903240L;
@SuppressWarnings("unused")
public ApiNotFoundException(String message) {
super(message);
super(GateResultCode.API_NOT_FOUND.getErrorCode(), message);
}
@SuppressWarnings("unused")
public ApiNotFoundException(String message, Throwable cause) {
super(message, cause);
super(GateResultCode.API_NOT_FOUND.getErrorCode(), message, cause);
}
}

8
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiUnauthorizedException.java
View File

@ -1,21 +1,23 @@
package cn.axzo.foundation.gateway.support.exception;
import cn.axzo.foundation.exception.BusinessException;
/**
* API未经授权访问的异常.
* <p>
* 实施时GateServer的ExceptionResolver可以拦截该异常做特殊处理, 如统一为HTTP标准状态码返回: ModelAndView.setStatus(HttpStatus.UNAUTHORIZED).
* </p>
*/
public class ApiUnauthorizedException extends RuntimeException {
public class ApiUnauthorizedException extends BusinessException {
private static final long serialVersionUID = -7679511760689237798L;
public ApiUnauthorizedException(String message) {
super(message);
super(GateResultCode.API_UNAUTHORIZED.getErrorCode(), message);
}
@SuppressWarnings("unused")
public ApiUnauthorizedException(String message, Throwable cause) {
super(message, cause);
super(GateResultCode.API_UNAUTHORIZED.getErrorCode(), message, cause);
}
}

31
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/GateResultCode.java Normal file
View File

@ -0,0 +1,31 @@
package cn.axzo.foundation.gateway.support.exception;
import cn.axzo.foundation.result.IResultCode;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor(access = AccessLevel.PRIVATE)
public enum GateResultCode implements IResultCode {
API_NOT_FOUND("050", "请求路径不存在", 500),
API_UNAUTHORIZED("051", "请求未授权", 403),
INPUT_FIELD_ABSENT("052", "参数错误", 400),
OUTPUT_FIELD_ABSENT("053", "返回结果错误", 500);
final private String code;
final private String message;
final private Integer httpCode;
@Override
public String getErrorCode() {
return code;
}
@Override
public String getErrorMessage() {
return message;
}
}

8
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/InputFieldAbsentException.java
View File

@ -1,17 +1,19 @@
package cn.axzo.foundation.gateway.support.exception;
import cn.axzo.foundation.exception.BusinessException;
/**
* ParameterFilter处理输入参数时输入参数中指定的字段不存在
*/
public class InputFieldAbsentException extends RuntimeException {
public class InputFieldAbsentException extends BusinessException {
private static final long serialVersionUID = -5646417429309513569L;
public InputFieldAbsentException(String message) {
super(message);
super(GateResultCode.INPUT_FIELD_ABSENT.getErrorCode(), message);
}
public InputFieldAbsentException(String message, Throwable cause) {
super(message, cause);
super(GateResultCode.INPUT_FIELD_ABSENT.getErrorCode(), message, cause);
}
}

8
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/OutputFieldAbsentException.java
View File

@ -1,17 +1,19 @@
package cn.axzo.foundation.gateway.support.exception;
import cn.axzo.foundation.exception.BusinessException;
/**
* ParameterFilter处理返回对象时返回对象中指定的字段不存在
*/
public class OutputFieldAbsentException extends RuntimeException {
public class OutputFieldAbsentException extends BusinessException {
private static final long serialVersionUID = 8830968633694688099L;
public OutputFieldAbsentException(String message) {
super(message);
super(GateResultCode.OUTPUT_FIELD_ABSENT.getErrorCode(), message);
}
public OutputFieldAbsentException(String message, Throwable cause) {
super(message, cause);
super(GateResultCode.OUTPUT_FIELD_ABSENT.getErrorCode(), message, cause);
}
}

50
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/fallback/FallbackConfig.java Normal file
View File

@ -0,0 +1,50 @@
package cn.axzo.foundation.gateway.support.fallback;
import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule;
import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableSet;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FallbackConfig {
String bean;
JSONObject config;
/** 策略, 支持FLOW_GRADE_QPS慢查询比例, DEGRADE_GRADE_EXCEPTION_RATIO异常比例, DEGRADE_GRADE_EXCEPTION_COUNT异常数 */
Integer grade = 2;
/** 默认10s内请求大于5个, 同时错误超过一半则降级30s */
Double count = 0.5D;
Integer timeWindow = 30;
Integer minRequestAmount = 5;
/** 慢比例有效 */
Double slowRatioThreshold = 1.0;
Integer statIntervalMs = 10000;
public static FallbackConfig fromConfig(JSONObject config) {
if (config == null || !config.containsKey("fallback")) {
return null;
}
return config.getJSONObject("fallback").toJavaObject(FallbackConfig.class);
}
public void registerRule(String resourceName) {
if (DegradeRuleManager.hasConfig(resourceName)) {
return;
}
DegradeRule degradeRule = new DegradeRule(resourceName);
degradeRule.setGrade(grade);
degradeRule.setCount(count);
degradeRule.setTimeWindow(timeWindow);
degradeRule.setMinRequestAmount(minRequestAmount);
degradeRule.setSlowRatioThreshold(slowRatioThreshold);
degradeRule.setStatIntervalMs(statIntervalMs);
DegradeRuleManager.setRulesForResource(resourceName, ImmutableSet.of(degradeRule));
}
}

16
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/ApiLogHook.java
View File

@ -12,16 +12,14 @@ import cn.axzo.foundation.web.support.rpc.RpcClient;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.AntPathMatcher;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@ -33,7 +31,8 @@ public class ApiLogHook implements ProxyHook {
private static final ThreadLocal<RequestParamsHolder> REQUEST_PARAMS_THREAD_LOCAL = new ThreadLocal<>();
private static final int API_RESPONSE_CONTENT_PRINT_LENGTH = 2048;
private final AntPathMatcher antPathMatcher = new AntPathMatcher();
private static final Set<String> DEFAULT_IGNORE_HEADER_NAMES = ImmutableSet.of("User-Agent", "Authorization", "_EMPLOYEE_PRINCIPAL", "Bfs-Authorization");
private static final Set<String> DEFAULT_IGNORE_HEADER_NAMES = ImmutableSet.of("User-Agent", "Authorization",
"Axzo-Authorization", "Bfs-Authorization", "referer", "sec-fetch-site","sec-ch-ua-platform","sec-fetch-dest");
private final int maxLogLength;
private final Predicate<HookContext> logEnableHook;
@ -52,7 +51,12 @@ public class ApiLogHook implements ProxyHook {
this.logEnableHook = Optional.ofNullable(logEnableHook).orElse(hookContext -> true);
this.maxLogLength = Optional.ofNullable(maxLogLength).orElse(API_RESPONSE_CONTENT_PRINT_LENGTH);
this.ignorePathPatterns = Optional.ofNullable(ignorePathPatterns).orElse(ImmutableList.of());
this.ignoreHeaderNames = Optional.ofNullable(ignoreHeaderNames).orElse(DEFAULT_IGNORE_HEADER_NAMES);
//忽略大小写
this.ignoreHeaderNames = Optional.ofNullable(ignoreHeaderNames).orElseGet(() -> {
TreeSet<String> sets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
sets.addAll(DEFAULT_IGNORE_HEADER_NAMES);
return sets;
});
this.printOriginalParams = BooleanUtils.isTrue(printOriginalParams);
// 默认 BusinessException 异常不打印ERROR级别日志.
this.warningExceptions = Optional.ofNullable(warningExceptions).orElse(ImmutableSet.of(BusinessException.class));

549
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java
View File

@ -3,37 +3,36 @@ package cn.axzo.foundation.gateway.support.plugin.impl;
import cn.axzo.foundation.gateway.support.entity.GateResponse;
import cn.axzo.foundation.gateway.support.entity.ProxyContext;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.fallback.FallbackConfig;
import cn.axzo.foundation.gateway.support.plugin.ProxyHook;
import cn.axzo.foundation.page.IPageReq;
import cn.axzo.foundation.page.PageResp;
import cn.axzo.foundation.result.ResultCode;
import cn.axzo.foundation.util.DataAssembleHelper;
import cn.axzo.foundation.gateway.support.plugin.impl.filters.*;
import cn.axzo.foundation.util.FastjsonUtils;
import cn.axzo.foundation.web.support.rpc.RequestParams;
import cn.axzo.foundation.web.support.rpc.RpcClient;
import com.alibaba.csp.sentinel.Entry;
import com.alibaba.csp.sentinel.EntryType;
import com.alibaba.csp.sentinel.SphU;
import com.alibaba.csp.sentinel.Tracer;
import com.alibaba.csp.sentinel.slots.block.BlockException;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPath;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.ImmutableMap;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
/**
* 对请求外部扩展插件。 使用如下：
@ -62,9 +61,6 @@ import java.util.stream.IntStream;
* </pre>
*/
@Slf4j
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RequestFilterHook implements ProxyHook {
public static final String FILTER_BEAN = "bean";
public static final String BEAN_SEPARATOR = "|";
@ -74,9 +70,35 @@ public class RequestFilterHook implements ProxyHook {
* key: bean 名称
* value: transformer 对象。
*/
@NonNull
private Function<String, RequestFilter> filterBeanResolver;
private static final Map<String, RequestFilter> DEFAULT_FILTERS = ImmutableMap.<String, RequestFilter>builder()
.put("convertListAsObjectFilter", new ConvertListAsObjectFilter())
.put("convertListAsPageFilter", new ConvertListAsPageFilter())
.put("convertPageAsListFilter", new ConvertPageAsListFilter())
.put("cropContentFilter", new CropContentFilter())
.put("requestParamCheckFilter", new RequestParamCheckFilter())
.put("moveInputFieldFilter", new MoveInputFieldFilter())
.put("keyNoQueryFilter", new KeyNoQueryFilter())
.put("emptyFilter", new RequestFilter() {
@Override
public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) {
return RequestFilter.super.filterIn(reqContext, params, config);
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
return RequestFilter.super.filterOut(reqContext, response, config);
}
})
.build();
@Builder
public RequestFilterHook(@NonNull Function<String, RequestFilter> filterBeanResolver) {
//优先从外部传入, 没有时则从default获取
this.filterBeanResolver = s -> Optional.ofNullable(filterBeanResolver.apply(s)).orElse(DEFAULT_FILTERS.get(s));
}
@Override
public RequestParams preRequest(RequestContext reqContext, ProxyContext proxyContext, RpcClient rpcClient, String postURL, RequestParams postParams) {
if (proxyContext.getParameterFilter() == null || proxyContext.getParameterFilter().getInFilterExtends().isEmpty()) {
@ -98,9 +120,31 @@ public class RequestFilterHook implements ProxyHook {
for (FilterBean bean : beans) {
RequestFilter requestFilter = filterBeanResolver.apply(bean.getName());
Preconditions.checkState(requestFilter != null, bean.getName() + " 没在系统注册");
requestBody = requestFilter.filterIn(reqContext, requestBody, bean.getConfig());
}
FallbackConfig fallbackConfig = FallbackConfig.fromConfig(bean.getConfig());
Entry entry = null;
try {
if (fallbackConfig != null) {
String resourceName = bean.getResourceName(reqContext.getRequestURI());
fallbackConfig.registerRule(resourceName);
entry = SphU.entry(resourceName, EntryType.IN);
}
requestBody = requestFilter.filterIn(reqContext, requestBody, bean.getConfig());
} catch (BlockException ex) {
String filterBean = StringUtils.firstNonBlank(fallbackConfig.getBean(), "emptyFilter");
log.warn("request fall back: {}, filter: {}, fallbackBean : {}",
reqContext.getRequestURI(), bean.getName(), filterBean, ex);
//降级处理
requestBody = filterBeanResolver.apply(filterBean).filterIn(reqContext, requestBody, fallbackConfig.getConfig());
} catch (Exception ex) {
Tracer.traceEntry(ex, entry);
throw ex;
} finally {
if (entry != null) {
entry.exit();
}
}
}
// 只支持 json 格式
return ((RequestParams.BodyParams) postParams).toBuilder().content(requestBody).build();
}
@ -128,7 +172,30 @@ public class RequestFilterHook implements ProxyHook {
for (FilterBean bean : beans) {
RequestFilter requestFilter = filterBeanResolver.apply(bean.getName());
Preconditions.checkState(requestFilter != null, bean.getName() + " 没在系统注册");
responseBody = requestFilter.filterOut(reqContext, responseBody, bean.getConfig());
FallbackConfig fallbackConfig = FallbackConfig.fromConfig(bean.getConfig());
Entry entry = null;
try {
if (fallbackConfig != null) {
String resourceName = bean.getResourceName(reqContext.getRequestURI());
fallbackConfig.registerRule(resourceName);
entry = SphU.entry(resourceName, EntryType.OUT);
}
responseBody = requestFilter.filterOut(reqContext, responseBody, bean.getConfig());
} catch (BlockException ex) {
String filterBean = StringUtils.firstNonBlank(fallbackConfig.getBean(), "emptyFilter");
log.warn("response fall back: {}, filter: {}, fallbackBean : {}",
reqContext.getRequestURI(), bean.getName(), filterBean, ex);
//降级处理
responseBody = filterBeanResolver.apply(filterBean).filterOut(reqContext, responseBody, fallbackConfig.getConfig());
} catch (Exception ex) {
Tracer.traceEntry(ex, entry);
throw ex;
} finally {
if (entry != null) {
entry.exit();
}
}
}
GateResponse res = response.toBuilder().build();
@ -183,446 +250,6 @@ public class RequestFilterHook implements ProxyHook {
}
}
/**
* 后台页面中通过关键字段查询时, 需要移除其他字段的查询场景. 添加通用的请求过滤器来处理此类需求.
* 使用时, 需要单独注入.
*/
public final static class KeyNoQueryFilter implements RequestFilter {
@Override
public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) {
if (params == null) {
return params;
}
Set<String> keyNoFields = Splitter.on(",").omitEmptyStrings().trimResults()
.splitToStream(Strings.nullToEmpty(config.getString("keyNoFields")))
.collect(Collectors.toSet());
Set<String> removeFields = Splitter.on(",").omitEmptyStrings().trimResults()
.splitToStream(Strings.nullToEmpty(config.getString("removeFields")))
.collect(Collectors.toSet());
if (keyNoFields.isEmpty() || removeFields.isEmpty()) {
return params;
}
JSONObject paramsJson = (JSONObject) params;
if (paramsJson.keySet().stream().noneMatch(keyNoFields::contains)) {
return paramsJson;
}
removeFields.forEach(paramsJson::remove);
return paramsJson;
}
}
public abstract static class ListOrPageRecordsFilter implements RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
if (!(content instanceof JSONObject) && !(content instanceof JSONArray)) {
return response;
}
if (content instanceof JSONArray) {
List<JSONObject> records = ((JSONArray) content).toJavaList(JSONObject.class);
if (CollectionUtils.isEmpty(records)) {
return response;
}
JSONArray filtered = new JSONArray().fluentAddAll(filterRecords(reqContext, records, config));
return jsonResp.fluentPut("data", filtered);
}
JSONObject page = (JSONObject) content;
JSONArray records = page.getJSONArray("data");
if (CollectionUtils.isEmpty(records)) {
return response;
}
List<JSONObject> filtered = filterRecords(reqContext, records.toJavaList(JSONObject.class), config);
page.put("data", new JSONArray().fluentAddAll(filtered));
return response;
}
public abstract List<JSONObject> filterRecords(RequestContext reqContext, List<JSONObject> records, JSONObject config);
}
public abstract static class ConvertContentFilter implements RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
if (content == null) {
return response;
}
Preconditions.checkArgument(content instanceof JSON, "ConvertContentFilter不支持原始response.content非json的情况");
return jsonResp.fluentPut("data", filterContent(reqContext, (JSON) content, config));
}
public abstract JSON filterContent(RequestContext reqContext, JSON content, JSONObject config);
}
public static class ConvertPageAsListFilter implements RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
if (!(content instanceof JSONObject)) {
return response;
}
JSONArray records = ((JSONObject) content).getJSONArray("data");
return jsonResp.fluentPut("data", Optional.ofNullable(records).orElseGet(JSONArray::new));
}
}
public static class ConvertListAsObjectFilter implements RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
JSONArray arrayContent = null;
if (content instanceof JSONArray) {
arrayContent = (JSONArray) content;
} else if (content instanceof JSONObject) {
JSONArray records = ((JSONObject) content).getJSONArray("data");
arrayContent = records == null ? new JSONArray() : records;
} else {
return response;
}
JSONObject obj = IntStream.range(0, arrayContent.size())
.mapToObj(arrayContent::getJSONObject)
.findFirst()
.orElseGet(JSONObject::new);
return jsonResp.fluentPut("data", filterObject(reqContext, obj, config));
}
public JSONObject filterObject(RequestContext reqContext, JSONObject obj) {
return obj;
}
public JSONObject filterObject(RequestContext reqContext, JSONObject obj, JSONObject config) {
return filterObject(reqContext, obj);
}
}
public static class ConvertListAsPageFilter implements RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
if (!(content instanceof JSONArray)) {
return response;
}
List<JSONObject> records = ((JSONArray) content).toJavaList(JSONObject.class);
PageResp<JSONObject> page = PageResp.<JSONObject>builder().total(records.size()).build();
IPageReq pageParam = JSONObject.parseObject(reqContext.getRequestBody(), IPageReq.class);
Optional.ofNullable(pageParam).map(IPageReq::getPage).ifPresent(page::setCurrent);
Optional.ofNullable(pageParam).map(IPageReq::getPageSize).ifPresent(page::setSize);
page.setData(records.stream()
.skip((page.getCurrent() - 1) * page.getSize())
.limit(page.getSize())
.collect(Collectors.toList()));
return jsonResp.fluentPut("data", page);
}
}
public static class RequestParamCheckFilter implements RequestFilter {
@Override
public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) {
boolean isConfigRulePresent = config != null && !config.isEmpty() && config.containsKey("rules");
Preconditions.checkArgument(isConfigRulePresent, "RequestParamCheckFilter必须指定检查规则");
Preconditions.checkArgument(JSONObject.class.isAssignableFrom(params.getClass()),
"RequestParamCheckFilter只能检查JSONObject类型参数");
//spring list默认读取为 map(key=0...N). 将values转换为Rule
List<Rule> rules = config.getJSONObject("rules").entrySet().stream()
.map(e -> ((JSONObject) e.getValue()).toJavaObject(Rule.class))
.collect(Collectors.toList());
rules.stream().forEach(p ->
Preconditions.checkArgument(!StringUtils.isAllBlank(p.getField(), p.getJsonPath()),
"RequestParamCheckFilter规则错误field, jsonPath不能都为空"));
Operator operator = Optional.ofNullable(config.getString("operator"))
.map(e -> Operator.valueOf(e)).orElse(Operator.AND);
List<Optional<String>> errors = rules.stream()
.map(rule -> rule.check((JSONObject) params))
.collect(Collectors.toList());
//如果operator=AND. & 任意一个检查错误 则告警
if (operator == Operator.AND && errors.stream().anyMatch(Optional::isPresent)) {
throw ResultCode.INVALID_PARAMS.toException(errors.stream().filter(Optional::isPresent).findFirst().get().get());
}
//如果operator=OR. & 所有检查都是错误 则告警
if (operator == Operator.OR && errors.stream().allMatch(Optional::isPresent)) {
throw ResultCode.INVALID_PARAMS.toException(errors.stream().filter(Optional::isPresent).findFirst().get().get());
}
return params;
}
@Data
private static class Rule {
String jsonPath;
String field;
boolean required;
String regex;
private transient Pattern pattern;
/** 检查值并返回错误信息, 正确则返回empty */
protected Optional<String> check(JSONObject param) {
Object value = getValue(param);
if (required && value == null) {
return Optional.of(field + "不能为空");
}
Optional<Pattern> pattern = tryGetPattern();
if (pattern.isPresent() && value != null && !pattern.get().matcher(value.toString()).find()) {
return Optional.of(field + "参数校验失败:" + regex);
}
return Optional.empty();
}
/** 优先根据field获取value. 否则根据jsonPath */
private Object getValue(JSONObject param) {
if (!Strings.isNullOrEmpty(field)) {
return param.get(field);
}
return JSONPath.eval(param, jsonPath);
}
private Optional<Pattern> tryGetPattern() {
if (Strings.isNullOrEmpty(regex)) {
return Optional.empty();
}
if (pattern == null) {
pattern = Pattern.compile(regex);
}
return Optional.of(pattern);
}
}
public enum Operator {
AND,
OR;
}
}
/**
* 裁剪返回的content内容
* 仅支持:
* 1.content是JSONObject
* 2.content是JSONArray，且其中是JSONObject
* 3.content是分页返回对象，即{"data":[{}]}
*/
public static class CropContentFilter extends ConvertContentFilter implements RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
throw new UnsupportedOperationException("使用CropContentFilter必须指定裁剪配置");
}
@Override
public JSON filterContent(RequestContext reqContext, JSON content, JSONObject config) {
// see class: CropConfig
if (CollectionUtils.isEmpty(config)) {
throw new UnsupportedOperationException("使用CropContentFilter必须指定裁剪配置");
}
return cropContent(content, config);
}
private JSON cropContent(JSON content, JSONObject config) {
// content是json数组
if (content instanceof JSONArray) {
return cropJSONArrayContent((JSONArray) content, config);
}
// content是json对象
if (content instanceof JSONObject) {
JSONObject contentJSONObject = (JSONObject) content;
// 可能是分页content，此时支持裁剪分页列表中的json对象
if (contentJSONObject.containsKey("data")) {
contentJSONObject.put("data",
cropJSONArrayContent(contentJSONObject.getJSONArray("data"), config));
}
return doCrop(contentJSONObject, config);
}
return content;
}
private JSONArray cropJSONArrayContent(JSONArray content, JSONObject config) {
// 只考虑JSONArray中是JSONObject对象的情况
List<JSONObject> contentList = content.stream()
.filter(obj -> obj instanceof JSONObject)
.map(obj -> JSONObject.parseObject(JSON.toJSONString(obj)))
.collect(Collectors.toList());
if (contentList.isEmpty()) {
return content;
}
return JSON.parseArray(JSON.toJSONString(contentList.stream()
.map(c -> doCrop(c, config))
.collect(Collectors.toList())));
}
/**
* 裁剪data中的字段
*
* @param data
* @param config {@link CropConfig}
* @return
*/
private static JSONObject doCrop(JSONObject data, JSONObject config) {
if (CollectionUtils.isEmpty(config) || CollectionUtils.isEmpty(data)) {
return data;
}
CropConfig cropConfig = config.toJavaObject(CropConfig.class);
// 优先用includeKeys
if (!CollectionUtils.isEmpty(cropConfig.getIncludeKeys())) {
return DataAssembleHelper.filterBean(data, cropConfig.getIncludeKeys(), true);
}
if (!CollectionUtils.isEmpty(cropConfig.getExcludeKeys())) {
return DataAssembleHelper.filterBean(data, cropConfig.getExcludeKeys(), false);
}
return data;
}
/**
* 裁剪content时的配置
*
* @see CropContentFilter
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
private static class CropConfig {
/**
* 希望只包含这些key，只支持第一层key
* 如果有值，将忽略excludeKeys
*/
private String includeKeys;
/**
* 希望排除的key，只支持第一层key
*/
private String excludeKeys;
public Set<String> getIncludeKeys() {
if (Strings.isNullOrEmpty(includeKeys)) {
return Collections.emptySet();
}
return ImmutableSet.copyOf(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(includeKeys));
}
public Set<String> getExcludeKeys() {
if (Strings.isNullOrEmpty(excludeKeys)) {
return Collections.emptySet();
}
return ImmutableSet.copyOf(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(excludeKeys));
}
}
}
/**
* 移动请求输入参数中字段到指定字段下.
*/
public final static class MoveInputFieldFilter implements RequestFilter {
@Override
public JSON filterIn(RequestContext reqContext, JSON params, JSONObject configJSON) {
if (params == null) {
return params;
}
Config config = configJSON.toJavaObject(Config.class);
if (!config.isValid()) {
throw new IllegalArgumentException("不正确的配置参数");
}
Set<String> sourceFields = config.getSourceFields();
JSONObject sourceValue = DataAssembleHelper.filterBean(params, sourceFields);
JSONObject res = DataAssembleHelper.filterBean(params, sourceFields, false);
Preconditions.checkArgument(!res.containsKey(config.getTargetField()), "目标字段已经存在, 不能被覆盖");
res.put(config.getTargetField(), config.getTargetFieldType() == Config.FieldType.PROPERTY
// 如果是把字段move到listField作为它的第一个元素
? sourceValue : Lists.newArrayList(sourceValue));
return res;
}
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
static class Config {
private String targetField;
private String sourceFields;
private FieldType targetFieldType;
public Set<String> getSourceFields() {
return Splitter.on(",").omitEmptyStrings().trimResults()
.splitToStream(Strings.nullToEmpty(sourceFields))
.collect(Collectors.toSet());
}
boolean isValid() {
return !getSourceFields().isEmpty() && !StringUtils.isBlank(targetField);
}
public FieldType getTargetFieldType() {
if (targetFieldType == null) {
return FieldType.PROPERTY;
}
return targetFieldType;
}
enum FieldType {
PROPERTY,
/**
* 将Field看做一个列表, 将 moveields 放在toField下作为第一个元素.
*/
LIST;
}
}
}
@Data
@Builder
@NoArgsConstructor
@ -630,5 +257,9 @@ public class RequestFilterHook implements ProxyHook {
private static class FilterBean {
private String name;
private JSONObject config;
public String getResourceName(String requestUrl) {
return "RequestFilterHook:" + name + "@" + requestUrl;
}
}
}

30
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertContentFilter.java Normal file
View File

@ -0,0 +1,30 @@
package cn.axzo.foundation.gateway.support.plugin.impl.filters;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Preconditions;
public abstract class ConvertContentFilter implements RequestFilterHook.RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
if (content == null) {
return response;
}
Preconditions.checkArgument(content instanceof JSON, "ConvertContentFilter不支持原始response.content非json的情况");
return jsonResp.fluentPut("data", filterContent(reqContext, (JSON) content, config));
}
public abstract JSON filterContent(RequestContext reqContext, JSON content, JSONObject config);
}

48
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsObjectFilter.java Normal file
View File

@ -0,0 +1,48 @@
package cn.axzo.foundation.gateway.support.plugin.impl.filters;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import java.util.stream.IntStream;
public class ConvertListAsObjectFilter implements RequestFilterHook.RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
JSONArray arrayContent = null;
if (content instanceof JSONArray) {
arrayContent = (JSONArray) content;
} else if (content instanceof JSONObject) {
JSONArray records = ((JSONObject) content).getJSONArray("data");
arrayContent = records == null ? new JSONArray() : records;
} else {
return response;
}
JSONObject obj = IntStream.range(0, arrayContent.size())
.mapToObj(arrayContent::getJSONObject)
.findFirst()
.orElseGet(JSONObject::new);
return jsonResp.fluentPut("data", filterObject(reqContext, obj, config));
}
public JSONObject filterObject(RequestContext reqContext, JSONObject obj) {
return obj;
}
public JSONObject filterObject(RequestContext reqContext, JSONObject obj, JSONObject config) {
return filterObject(reqContext, obj);
}
}

46
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsPageFilter.java Normal file
View File

@ -0,0 +1,46 @@
package cn.axzo.foundation.gateway.support.plugin.impl.filters;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook;
import cn.axzo.foundation.page.IPageReq;
import cn.axzo.foundation.page.PageResp;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
public class ConvertListAsPageFilter implements RequestFilterHook.RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
if (!(content instanceof JSONArray)) {
return response;
}
List<JSONObject> records = ((JSONArray) content).toJavaList(JSONObject.class);
PageResp<JSONObject> page = PageResp.<JSONObject>builder().total(records.size()).build();
IPageReq pageParam = JSONObject.parseObject(reqContext.getRequestBody(), IPageReq.class);
Optional.ofNullable(pageParam).map(IPageReq::getPage).ifPresent(page::setCurrent);
Optional.ofNullable(pageParam).map(IPageReq::getPageSize).ifPresent(page::setSize);
page.setData(records.stream()
.skip((page.getCurrent() - 1) * page.getSize())
.limit(page.getSize())
.collect(Collectors.toList()));
return jsonResp.fluentPut("data", page);
}
}

31
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertPageAsListFilter.java Normal file
View File

@ -0,0 +1,31 @@
package cn.axzo.foundation.gateway.support.plugin.impl.filters;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import java.util.Optional;
public class ConvertPageAsListFilter implements RequestFilterHook.RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
if (!(content instanceof JSONObject)) {
return response;
}
JSONArray records = ((JSONObject) content).getJSONArray("data");
return jsonResp.fluentPut("data", Optional.ofNullable(records).orElseGet(JSONArray::new));
}
}

133
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/CropContentFilter.java Normal file
View File

@ -0,0 +1,133 @@
package cn.axzo.foundation.gateway.support.plugin.impl.filters;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook;
import cn.axzo.foundation.util.DataAssembleHelper;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.util.CollectionUtils;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* 裁剪返回的content内容
* 仅支持:
* 1.content是JSONObject
* 2.content是JSONArray，且其中是JSONObject
* 3.content是分页返回对象，即{"data":[{}]}
*/
public class CropContentFilter extends ConvertContentFilter implements RequestFilterHook.RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
throw new UnsupportedOperationException("使用CropContentFilter必须指定裁剪配置");
}
@Override
public JSON filterContent(RequestContext reqContext, JSON content, JSONObject config) {
// see class: CropConfig
if (CollectionUtils.isEmpty(config)) {
throw new UnsupportedOperationException("使用CropContentFilter必须指定裁剪配置");
}
return cropContent(content, config);
}
private JSON cropContent(JSON content, JSONObject config) {
// content是json数组
if (content instanceof JSONArray) {
return cropJSONArrayContent((JSONArray) content, config);
}
// content是json对象
if (content instanceof JSONObject) {
JSONObject contentJSONObject = (JSONObject) content;
// 可能是分页content，此时支持裁剪分页列表中的json对象
if (contentJSONObject.containsKey("data")) {
contentJSONObject.put("data",
cropJSONArrayContent(contentJSONObject.getJSONArray("data"), config));
}
return doCrop(contentJSONObject, config);
}
return content;
}
private JSONArray cropJSONArrayContent(JSONArray content, JSONObject config) {
// 只考虑JSONArray中是JSONObject对象的情况
List<JSONObject> contentList = content.stream()
.filter(obj -> obj instanceof JSONObject)
.map(obj -> JSONObject.parseObject(JSON.toJSONString(obj)))
.collect(Collectors.toList());
if (contentList.isEmpty()) {
return content;
}
return JSON.parseArray(JSON.toJSONString(contentList.stream()
.map(c -> doCrop(c, config))
.collect(Collectors.toList())));
}
/**
* 裁剪data中的字段
*
* @param data
* @param config {@link CropConfig}
* @return
*/
private static JSONObject doCrop(JSONObject data, JSONObject config) {
if (CollectionUtils.isEmpty(config) || CollectionUtils.isEmpty(data)) {
return data;
}
CropConfig cropConfig = config.toJavaObject(CropConfig.class);
// 优先用includeKeys
if (!CollectionUtils.isEmpty(cropConfig.getIncludeKeys())) {
return DataAssembleHelper.filterBean(data, cropConfig.getIncludeKeys(), true);
}
if (!CollectionUtils.isEmpty(cropConfig.getExcludeKeys())) {
return DataAssembleHelper.filterBean(data, cropConfig.getExcludeKeys(), false);
}
return data;
}
/**
* 裁剪content时的配置
*
* @see CropContentFilter
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
private static class CropConfig {
/**
* 希望只包含这些key，只支持第一层key
* 如果有值，将忽略excludeKeys
*/
private String includeKeys;
/**
* 希望排除的key，只支持第一层key
*/
private String excludeKeys;
public Set<String> getIncludeKeys() {
if (Strings.isNullOrEmpty(includeKeys)) {
return Collections.emptySet();
}
return ImmutableSet.copyOf(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(includeKeys));
}
public Set<String> getExcludeKeys() {
if (Strings.isNullOrEmpty(excludeKeys)) {
return Collections.emptySet();
}
return ImmutableSet.copyOf(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(excludeKeys));
}
}
}

42
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/KeyNoQueryFilter.java Normal file
View File

@ -0,0 +1,42 @@
package cn.axzo.foundation.gateway.support.plugin.impl.filters;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import java.util.Set;
import java.util.stream.Collectors;
/**
* 后台页面中通过关键字段查询时, 需要移除其他字段的查询场景. 添加通用的请求过滤器来处理此类需求.
* 使用时, 需要单独注入.
*/
public class KeyNoQueryFilter implements RequestFilterHook.RequestFilter {
@Override
public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) {
if (params == null) {
return params;
}
Set<String> keyNoFields = Splitter.on(",").omitEmptyStrings().trimResults()
.splitToStream(Strings.nullToEmpty(config.getString("keyNoFields")))
.collect(Collectors.toSet());
Set<String> removeFields = Splitter.on(",").omitEmptyStrings().trimResults()
.splitToStream(Strings.nullToEmpty(config.getString("removeFields")))
.collect(Collectors.toSet());
if (keyNoFields.isEmpty() || removeFields.isEmpty()) {
return params;
}
JSONObject paramsJson = (JSONObject) params;
if (paramsJson.keySet().stream().noneMatch(keyNoFields::contains)) {
return paramsJson;
}
removeFields.forEach(paramsJson::remove);
return paramsJson;
}
}

49
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ListOrPageRecordsFilter.java Normal file
View File

@ -0,0 +1,49 @@
package cn.axzo.foundation.gateway.support.plugin.impl.filters;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.springframework.util.CollectionUtils;
import java.util.List;
public abstract class ListOrPageRecordsFilter implements RequestFilterHook.RequestFilter {
@Override
public JSON filterOut(RequestContext reqContext, JSON response) {
return filterOut(reqContext, response, new JSONObject());
}
@Override
public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) {
if (!(response instanceof JSONObject)) {
return response;
}
JSONObject jsonResp = (JSONObject) response;
Object content = jsonResp.get("data");
if (!(content instanceof JSONObject) && !(content instanceof JSONArray)) {
return response;
}
if (content instanceof JSONArray) {
List<JSONObject> records = ((JSONArray) content).toJavaList(JSONObject.class);
if (CollectionUtils.isEmpty(records)) {
return response;
}
JSONArray filtered = new JSONArray().fluentAddAll(filterRecords(reqContext, records, config));
return jsonResp.fluentPut("data", filtered);
}
JSONObject page = (JSONObject) content;
JSONArray records = page.getJSONArray("data");
if (CollectionUtils.isEmpty(records)) {
return response;
}
List<JSONObject> filtered = filterRecords(reqContext, records.toJavaList(JSONObject.class), config);
page.put("data", new JSONArray().fluentAddAll(filtered));
return response;
}
public abstract List<JSONObject> filterRecords(RequestContext reqContext, List<JSONObject> records, JSONObject config);
}

82
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/MoveInputFieldFilter.java Normal file
View File

@ -0,0 +1,82 @@
package cn.axzo.foundation.gateway.support.plugin.impl.filters;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook;
import cn.axzo.foundation.util.DataAssembleHelper;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import java.util.Set;
import java.util.stream.Collectors;
/**
* 移动请求输入参数中字段到指定字段下.
*/
public class MoveInputFieldFilter implements RequestFilterHook.RequestFilter {
@Override
public JSON filterIn(RequestContext reqContext, JSON params, JSONObject configJSON) {
if (params == null) {
return params;
}
Config config = configJSON.toJavaObject(Config.class);
if (!config.isValid()) {
throw new IllegalArgumentException("不正确的配置参数");
}
Set<String> sourceFields = config.getSourceFields();
JSONObject sourceValue = DataAssembleHelper.filterBean(params, sourceFields);
JSONObject res = DataAssembleHelper.filterBean(params, sourceFields, false);
Preconditions.checkArgument(!res.containsKey(config.getTargetField()), "目标字段已经存在, 不能被覆盖");
res.put(config.getTargetField(), config.getTargetFieldType() == Config.FieldType.PROPERTY
// 如果是把字段move到listField作为它的第一个元素
? sourceValue : Lists.newArrayList(sourceValue));
return res;
}
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
static class Config {
private String targetField;
private String sourceFields;
private Config.FieldType targetFieldType;
public Set<String> getSourceFields() {
return Splitter.on(",").omitEmptyStrings().trimResults()
.splitToStream(Strings.nullToEmpty(sourceFields))
.collect(Collectors.toSet());
}
boolean isValid() {
return !getSourceFields().isEmpty() && !StringUtils.isBlank(targetField);
}
public Config.FieldType getTargetFieldType() {
if (targetFieldType == null) {
return Config.FieldType.PROPERTY;
}
return targetFieldType;
}
enum FieldType {
PROPERTY,
/**
* 将Field看做一个列表, 将 moveields 放在toField下作为第一个元素.
*/
LIST;
}
}
}

95
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/RequestParamCheckFilter.java Normal file
View File

@ -0,0 +1,95 @@
package cn.axzo.foundation.gateway.support.plugin.impl.filters;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook;
import cn.axzo.foundation.result.ResultCode;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPath;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class RequestParamCheckFilter implements RequestFilterHook.RequestFilter {
@Override
public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) {
boolean isConfigRulePresent = config != null && !config.isEmpty() && config.containsKey("rules");
Preconditions.checkArgument(isConfigRulePresent, "RequestParamCheckFilter必须指定检查规则");
Preconditions.checkArgument(JSONObject.class.isAssignableFrom(params.getClass()),
"RequestParamCheckFilter只能检查JSONObject类型参数");
//spring list默认读取为 map(key=0...N). 将values转换为Rule
List<Rule> rules = config.getJSONObject("rules").entrySet().stream()
.map(e -> ((JSONObject) e.getValue()).toJavaObject(Rule.class))
.collect(Collectors.toList());
rules.stream().forEach(p ->
Preconditions.checkArgument(!StringUtils.isAllBlank(p.getField(), p.getJsonPath()),
"RequestParamCheckFilter规则错误field, jsonPath不能都为空"));
Operator operator = Optional.ofNullable(config.getString("operator"))
.map(e -> Operator.valueOf(e)).orElse(Operator.AND);
List<Optional<String>> errors = rules.stream()
.map(rule -> rule.check((JSONObject) params))
.collect(Collectors.toList());
//如果operator=AND. & 任意一个检查错误 则告警
if (operator == Operator.AND && errors.stream().anyMatch(Optional::isPresent)) {
throw ResultCode.INVALID_PARAMS.toException(errors.stream().filter(Optional::isPresent).findFirst().get().get());
}
//如果operator=OR. & 所有检查都是错误 则告警
if (operator == Operator.OR && errors.stream().allMatch(Optional::isPresent)) {
throw ResultCode.INVALID_PARAMS.toException(errors.stream().filter(Optional::isPresent).findFirst().get().get());
}
return params;
}
@Data
private static class Rule {
String jsonPath;
String field;
boolean required;
String regex;
private transient Pattern pattern;
/** 检查值并返回错误信息, 正确则返回empty */
protected Optional<String> check(JSONObject param) {
Object value = getValue(param);
if (required && value == null) {
return Optional.of(field + "不能为空");
}
Optional<Pattern> pattern = tryGetPattern();
if (pattern.isPresent() && value != null && !pattern.get().matcher(value.toString()).find()) {
return Optional.of(field + "参数校验失败:" + regex);
}
return Optional.empty();
}
/** 优先根据field获取value. 否则根据jsonPath */
private Object getValue(JSONObject param) {
if (!Strings.isNullOrEmpty(field)) {
return param.get(field);
}
return JSONPath.eval(param, jsonPath);
}
private Optional<Pattern> tryGetPattern() {
if (Strings.isNullOrEmpty(regex)) {
return Optional.empty();
}
if (pattern == null) {
pattern = Pattern.compile(regex);
}
return Optional.of(pattern);
}
}
public enum Operator {
AND,
OR;
}
}

31
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/GateResponseSupplier.java Normal file
View File

@ -0,0 +1,31 @@
package cn.axzo.foundation.gateway.support.proxy;
import cn.axzo.foundation.gateway.support.entity.GateResponse;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.result.ApiResult;
import cn.axzo.foundation.web.support.rpc.RequestParams;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.springframework.http.HttpStatus;
import java.nio.charset.StandardCharsets;
/**
* 返回GateResponse
* 目前在服务降级时会调用
*/
public interface GateResponseSupplier {
GateResponse getResponse(RequestContext requestContext, String resolvedUrl, RequestParams requestParams);
GateResponseSupplier DEFAULT = (requestContext, resolvedUrl, requestParams) -> {
byte[] bytes = JSON.toJSONString(ApiResult.success()).getBytes(StandardCharsets.UTF_8);
return GateResponse.builder()
.status(HttpStatus.OK)
.content(bytes)
.headers(ImmutableMap.of("content-type", ImmutableList.of("application/json;charset=UTF-8")
, "content-length", ImmutableList.of(bytes.length + "")))
.build();
};
}

44
gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/impl/SimpleProxy.java
View File

@ -3,8 +3,16 @@ package cn.axzo.foundation.gateway.support.proxy.impl;
import cn.axzo.foundation.gateway.support.entity.GateResponse;
import cn.axzo.foundation.gateway.support.entity.ProxyContext;
import cn.axzo.foundation.gateway.support.entity.RequestContext;
import cn.axzo.foundation.gateway.support.exception.ApiNotFoundException;
import cn.axzo.foundation.gateway.support.fallback.FallbackConfig;
import cn.axzo.foundation.gateway.support.plugin.ProxyHookChain;
import cn.axzo.foundation.gateway.support.proxy.GateResponseSupplier;
import cn.axzo.foundation.web.support.rpc.RequestParams;
import cn.axzo.foundation.web.support.rpc.RpcClient;
import com.alibaba.csp.sentinel.Entry;
import com.alibaba.csp.sentinel.SphU;
import com.alibaba.csp.sentinel.Tracer;
import com.alibaba.csp.sentinel.slots.block.BlockException;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONException;
@ -14,6 +22,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.net.UrlEscapers;
import lombok.extern.slf4j.Slf4j;
import okhttp3.MediaType;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@ -23,12 +32,10 @@ import javax.servlet.http.HttpServletRequest;
import java.util.*;
import java.util.stream.Collectors;
import cn.axzo.foundation.gateway.support.exception.ApiNotFoundException;
import cn.axzo.foundation.web.support.rpc.RpcClient;
/**
* 简单请求Proxy, 根据配置中的服务编码对应的域名, 在具体实现代理中进行URL及参数转换.
*/
@Slf4j
public abstract class SimpleProxy extends AbstractProxy {
abstract String resolveServiceCode(RequestContext context);
@ -74,8 +81,33 @@ public abstract class SimpleProxy extends AbstractProxy {
RpcClient rpcClient = getServiceResolver().getRpcClient(requestContext, hookedCode);
RequestParams resolvedParams = getHookChain().preRequest(requestContext, getContext(), rpcClient, resolvedUrl, requestParams);
GateResponse response = requestContext.getRequestMethod().request(
rpcClient, resolvedUrl, resolvedParams);
FallbackConfig fallbackConfig = FallbackConfig.fromConfig(getContext().getProxyParam());
GateResponse response;
Entry entry = null;
try {
if (fallbackConfig != null) {
String resourceName = "SimpleProxy:" + requestContext.getServiceCode() + "@" + requestContext.getRequestURI();
fallbackConfig.registerRule(resourceName);
entry = SphU.entry(resourceName);
}
response = requestContext.getRequestMethod().request(
rpcClient, resolvedUrl, resolvedParams);
} catch (BlockException ex) {
log.warn("proxy fall back: {}, fallbackBean : {}", requestContext.getRequestURI(), fallbackConfig.getBean(), ex);
//降级处理
response = Optional.ofNullable(getContext().getGlobalContext().getFallbackSupplier())
.flatMap(e -> Optional.ofNullable(e.apply(fallbackConfig.getBean())))
.orElse(GateResponseSupplier.DEFAULT).getResponse(requestContext, resolvedUrl, requestParams);
} catch (Exception ex) {
Tracer.traceEntry(ex, entry);
throw ex;
} finally {
if (entry != null) {
entry.exit();
}
}
GateResponse res = getHookChain().postResponse(requestContext, getContext(), response);
filterResponse(requestContext, res);
@ -102,7 +134,7 @@ public abstract class SimpleProxy extends AbstractProxy {
Optional<String> subtypeOptional = Optional.ofNullable(context.getOriginalRequest())
.map(HttpServletRequest::getContentType)
.map(MediaType::parse)
.map(e->e.subtype().toLowerCase());
.map(e -> e.subtype().toLowerCase());
if (subtypeOptional.isPresent() && JSON_YML_SUBTYPES.contains(subtypeOptional.get())) {
String requestBody = StringUtils.firstNonBlank(context.getRequestBody(), "{}");
Object body;

57
gateway-support-lib/src/test/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHookTest.java Normal file
View File

@ -0,0 +1,57 @@
package cn.axzo.foundation.gateway.support.plugin.impl;
import com.alibaba.csp.sentinel.Entry;
import com.alibaba.csp.sentinel.EntryType;
import com.alibaba.csp.sentinel.SphU;
import com.alibaba.csp.sentinel.Tracer;
import com.alibaba.csp.sentinel.slots.block.BlockException;
import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule;
import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
class RequestFilterHookTest {
public static void main(String[] args) {
String resourceName = "aaaa";
DegradeRule degradeRule = new DegradeRule(resourceName);
degradeRule.setGrade(2);
degradeRule.setCount(0.5);
degradeRule.setTimeWindow(30);
degradeRule.setMinRequestAmount(5);
degradeRule.setStatIntervalMs(10);
DegradeRuleManager.setRulesForResource(resourceName, ImmutableSet.of(degradeRule));
List<Integer> blocked = IntStream.range(0, 20).mapToObj(e -> {
Entry entry = null;
try {
entry = SphU.entry(resourceName, EntryType.IN);
if (e % 2 == 0) {
throw new RuntimeException();
}
return e;
} catch (BlockException ex) {
return -1;
} catch (Exception ex) {
Tracer.traceEntry(ex, entry);
return -2;
} finally {
if (entry != null) {
entry.exit();
}
}
})
.collect(Collectors.toList());
System.out.printf(blocked.toString());
}
}

2
pom.xml
View File

@ -32,6 +32,8 @@
<module>web-support-lib</module>
<module>gateway-support-lib</module>
<module>event-support-lib</module>
<module>redis-support-lib</module>
<module>excel-support-lib</module>
</modules>
<dependencies>

31
redis-support-lib/pom.xml Normal file
View File

@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>cn.axzo.foundation</groupId>
<artifactId>axzo-lib-box</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>redis-support-lib</artifactId>
<dependencies>
<dependency>
<groupId>cn.axzo.foundation</groupId>
<artifactId>web-support-lib</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
</dependency>
</dependencies>
</project>

75
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/EventBroadcast.java Normal file
View File

@ -0,0 +1,75 @@
package cn.axzo.foundation.redis.support;
import com.alibaba.fastjson.JSONObject;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.function.BiConsumer;
/**
* 提供简单的广播Client. 通过build获得的BroadcastQueue发送广播. 所有节点都会收到广播并回调consumer
* <ul>
* <li>通过build获得BroadcastQueue, 调用queue的broadcast方法. 发送广播</li>
* <li>收到广播会会主动回调注册时的BiConsumer</li>
* <li>目前支持通过redis的pub/sub实现, 因此需要依赖redisTemplate</li>
* <li>lettuce来处理回调, 不建议在回调用做比较重的业务</li>
* </ul>
*/
public interface EventBroadcast {
/**
* 通过queueName, 回调consumer来构建广播队列
*
* @param queueName
* @param consumer
* @return
*/
BroadcastQueue build(String queueName, BiConsumer<String, BroadcastEvent> consumer);
/**
* 广播队列. 向队列中广播消息. 所有的节点(包含自身)都会收到广播消息
*/
interface BroadcastQueue {
/**
* 获得队列名称, 与注册时的queueName相同
*
* @return
*/
String getName();
/**
* 广播, data为需要广播的内容
*
* @param data
* @return
*/
boolean broadcast(JSONObject data);
}
/**
* 广播事件
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
class BroadcastEvent {
String name;
JSONObject data;
/**
* 发起广播的节点信息
*/
JSONObject senderRuntime;
public String toJSONString() {
return new JSONObject()
.fluentPut("name", name)
.fluentPut("data", data)
.fluentPut("senderRuntime", senderRuntime)
.toJSONString();
}
}
}

49
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/LocalCacheCoordinate.java Normal file
View File

@ -0,0 +1,49 @@
package cn.axzo.foundation.redis.support;
import com.alibaba.fastjson.JSONObject;
import lombok.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
/**
* 简单的本地缓存协调客户端. 主要的目的是通过事件在<s>不同节点</s>间处理缓存dirty的场景.
* 提供了2中机制来处理缓存dirty的场景
* registerCacheDirtyListener 精确处理key匹配
* registerCacheDirtyInterceptor 统一的事件匹配. 可以通过检查 CacheDirtiedEvent.data中的数据是否包含特定的json-path来处理缓存.
*/
public interface LocalCacheCoordinate {
void notifyCacheDirty(CacheDirtiedEvent event);
/**
* @param key {@link CacheDirtiedEvent#key} 中定义数据
* @param consumer
*/
void registerCacheDirtyListener(String key, Consumer<CacheDirtiedEvent> consumer);
void registerCacheDirtyInterceptor(CacheDirtiedEventInterceptor interceptor);
@NoArgsConstructor
@AllArgsConstructor
@Data
@Builder
final class CacheDirtiedEvent {
@NonNull
String key;
@NonNull
JSONObject data;
}
@NoArgsConstructor
@AllArgsConstructor
@Data
@Builder
final class CacheDirtiedEventInterceptor {
@NonNull
Predicate<CacheDirtiedEvent> filter;
@NonNull
Consumer<CacheDirtiedEvent> consumer;
}
}

116
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiter.java Normal file
View File

@ -0,0 +1,116 @@
package cn.axzo.foundation.redis.support;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import lombok.*;
import java.util.*;
import java.util.stream.Collectors;
public interface RateLimiter {
/**
* 尝试获得锁, 获取失败则返回Optional.empty()
* 如果获取锁成功. 则返回Optional<Permit>. 同时计数器增加
* Permit支持取消
*
* @param value 业务标识
* @param step 指定步长
* @return
*/
Optional<Permit> tryAcquire(Object value, long step);
default Optional<Permit> tryAcquire(Object value) {
return tryAcquire(value, 1);
}
/**
* 重置value对应的锁, 便于特殊场景下重新获取锁
*/
void reset(Object value);
/**
* 获取窗口类型
*
* @return
*/
WindowType getWindowType();
class Permit {
private List<Runnable> cancelRunners;
@Builder
public Permit(List<Runnable> cancelRunners) {
Objects.requireNonNull(cancelRunners);
this.cancelRunners = cancelRunners;
}
public void cancel() {
if (!cancelRunners.isEmpty()) {
cancelRunners.stream().forEach(e -> e.run());
}
}
}
@Getter
@AllArgsConstructor
enum WindowType {
/**
* 固定窗口, 窗口范围: start = visitTim, end = start + WindowDuration
*/
FIXED("f"),
/**
* 固定窗口, 窗口范围: start = currentMillis/WindowDuration, end = currentMillis/WindowDuration
*/
FIXED_BUCKET("fb"),
/**
* 滑动窗口, 窗口范围: start = currentMillis - WindowDuration, end = currentMillis
*/
SLIDING("s");
//减少redisKey长度
private final String shortName;
}
/**
* 限流规则
* <pre>
* seconds: 窗口时长
* permits: 允许发放的令牌数量
* </pre>
*/
@Data
@Builder
@AllArgsConstructor
@ToString
class LimitRule {
long seconds;
int permits;
public boolean isValid() {
return seconds > 0 && permits > 0;
}
/**
* 根据约定规则创建Rules
* eg: express 10/1,20/1... 代表10秒1次 & 20秒1次...
* note: seconds不可重复
*/
public static List<LimitRule> fromExpression(String expression) {
if (Strings.isNullOrEmpty(expression)) {
return Collections.emptyList();
}
Map<String, String> rulesMap = Splitter.on(",")
.omitEmptyStrings()
.trimResults()
.withKeyValueSeparator("/")
.split(expression);
return rulesMap.entrySet().stream()
.map(e -> LimitRule.builder()
.seconds(Long.parseLong(e.getKey()))
.permits(Integer.parseInt(e.getValue()))
.build())
.collect(Collectors.toList());
}
}
}

32
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiterFactory.java Normal file
View File

@ -0,0 +1,32 @@
package cn.axzo.foundation.redis.support;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
public interface RateLimiterFactory {
/**
* 构建一个基于RateLimiter
*/
RateLimiter build(RateLimiterReq rateLimiterReq);
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
class RateLimiterReq {
RateLimiter.WindowType windowType;
List<RateLimiter.LimitRule> rules;
String limiterKey;
RateType rateType;
}
enum RateType {
LOCAL,
REDIS
}
}

270
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RedisLock.java Normal file
View File

@ -0,0 +1,270 @@
package cn.axzo.foundation.redis.support;
import cn.axzo.foundation.result.ResultCode;
import lombok.extern.slf4j.Slf4j;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
/**
* 使用redis实现的分布式锁. 为了不依赖redis, 这里需要调用者实现{@link RedisWrapper}. 如下:
* <pre>{@code
* @Bean
* RedisLock.RedisWrapper redisWrapper(RedisTemplate redisTemplate) {
* return new RedisLock.RedisWrapper() {
* @Override
* public void delete(String key) {
* redisTemplate.delete(key);
* }
*
* @Override
* public boolean lock(String key, String value, long expireMills) {
* return redisTemplate.opsForValue().setIfAbsent(key, value, expireMills, TimeUnit.MILLISECONDS);
* }
* };
* }
* }</pre>
* <p>
* 然后使用{@link #tryAcquireRun(long, long, Supplier)} 来获取lock执行逻辑，明确指定 "等待锁时间" 和 "锁的超时时间".
*/
@Slf4j
public class RedisLock {
public static final String DEFAULT_LOCK_SUFFIX = ":lock";
private RedisWrapper redis;
/**
* 默认超时时间（毫秒）
*/
private static final long DEFAULT_TIME_OUT_MILLIS = 5 * 1000;
private static final Random RANDOM = new Random();
/**
* 锁的超时时间（豪秒），过期删除
*/
public static final int EXPIRE_IN_MILLIS = 1 * 60 * 1000;
private String key;
// 锁状态标志
private boolean locked = false;
private RuntimeException lockFailedException;
public interface RedisWrapper {
/**
* 删除key
*
* @param key
*/
void delete(String key);
/**
* 锁定key. 通常使用SetIfAbsent();
*
* @param key
* @param value
* @param expireMills
* @return
*/
boolean lock(String key, String value, long expireMills);
}
/**
* 关闭锁，该方法不建议外部直接使用，<br>
* 对于加锁执行的操作，建议直接使用 {@link RedisLock#tryAcquireRun(long, long, Supplier)}，会自动执行close操作。
*/
private void close() {
if (this.locked) {
this.redis.delete(this.key);
}
}
/**
* This creates a RedisLock
*
* @param key key
* @param redis 数据源
*/
public RedisLock(String key, RedisWrapper redis) {
this(key, redis, DEFAULT_LOCK_SUFFIX, null);
}
/**
* This creates a RedisLock
*
* @param key key
* @param redis 数据源
*/
public RedisLock(String key, RedisWrapper redis, RuntimeException lockFailedException) {
this(key, redis, DEFAULT_LOCK_SUFFIX, lockFailedException);
}
/**
* This creates a RedisLock
*
* @param key key
* @param redis 数据源
*/
public RedisLock(String key, RedisWrapper redis, String suffix, RuntimeException lockFailedException) {
this.key = key + Optional.ofNullable(suffix).orElse(DEFAULT_LOCK_SUFFIX);
this.redis = redis;
this.lockFailedException = Optional.ofNullable(lockFailedException)
.orElseGet(() -> new AcquireLockFailException("获取锁失败 " + key));
}
/**
* 尝试在timeoutMillis毫秒内获取锁并设置锁的过期时间为expireMillis毫秒，若获取锁成功，则执行supplier的逻辑，并返回supplier执行结果。然后关闭锁<br>
* <pre>
* 锁的释放，由2方面保证：
* 1、supplier方法执行完成后，会主动释放锁。
* 2、设置锁的过期时间
* </pre>
* 如果只是单纯的尝试获取锁并执行，无需等待锁，可以<b>将timeoutMillis参数设置为0。</b>
*
* @param timeoutMillis 等待获取锁的时间 单位毫秒（会在等待时间内不停自旋尝试获取锁。）如果超过该时间还没成功获取到锁，则抛出获取锁失败的BizException
* <b>timeoutMillis=0，则表示只进行一次获取锁的尝试。获取失败，直接抛获取锁失败的异常</b>
* @param expireMillis 锁的过期时间，保证锁最长的持有时间。（如果主动释放锁失败，会有该参数保证锁成功释放）
* @param supplier 需要执行的方法
* @param <T> 返回参数类型
* @return
*/
public <T> T tryAcquireRun(final long timeoutMillis, final long expireMillis, Supplier<T> supplier) {
if (!lock(timeoutMillis, expireMillis)) {
throw lockFailedException;
}
try {
return supplier.get();
} finally {
close();
}
}
/**
* 尝试获取锁，并执行supplier.get()方法，返回结果。<br>
* 该方法使用了默认的锁等待时间和过期时间：<br>
* 等待锁时间={@link #DEFAULT_TIME_OUT_MILLIS 5秒}<br>
* 锁过期时间={@link #EXPIRE_IN_MILLIS 1分钟}<br>
* 调用该方法，效果等同于 {@link #tryAcquireRun(long, long, Supplier)}
* -> tryAcquireRun(DEFAULT_TIME_OUT_MILLIS, EXPIRE_IN_MILLIS, supplier);
*
* @param supplier
* @param <T>
* @return
*/
public <T> T tryAcquireRun(Supplier<T> supplier) {
if (!lock()) {
throw lockFailedException;
}
try {
return supplier.get();
} finally {
close();
}
}
/**
* 尝试获取锁，并执行supplier.get()方法，返回结果。<br>
* 该方法使用了默认的锁过期时间：<br>
* 锁过期时间={@link #EXPIRE_IN_MILLIS 1分钟}<br>
* 调用该方法，效果等同于 {@link #tryAcquireRun(long, long, Supplier)}
* -> tryAcquireRun(timeoutMillis, EXPIRE_IN_MILLIS, supplier);
*
* @param supplier
* @param <T>
* @return
*/
public <T> T tryAcquireRun(long timeoutMillis, Supplier<T> supplier) {
if (!lock(timeoutMillis)) {
throw lockFailedException;
}
try {
return supplier.get();
} finally {
close();
}
}
/**
* 尝试立即获取锁，并执行supplier.get()方法，返回结果。<br>
* timeoutMills = 0, expireMillis = 5分钟
*/
public <T> T acquireImmediatelyRun(Supplier<T> supplier) {
if (!lock(0, TimeUnit.MINUTES.toMillis(5))) {
throw ResultCode.OPERATE_TOO_FREQUENTLY.toException();
}
try {
return supplier.get();
} finally {
close();
}
}
/**
* 加锁 应该以： lock(); try { doSomething(); } finally { close()； } 的方式调用<br>
* 外部不建议直接使用该方法，建议使用{@link #tryAcquireRun(long, long, Supplier)}明确指定锁的等待和过期时间
*
* @param timeoutMillis 超时时间(毫秒)
* @return 成功或失败标志
*/
private boolean lock(long timeoutMillis) {
return lock(timeoutMillis, EXPIRE_IN_MILLIS);
}
/**
* 加锁 应该以： lock(); try { doSomething(); } finally { close()； } 的方式调用<br>
* 外部不建议直接使用该方法，建议使用{@link #tryAcquireRun(long, long, Supplier)}明确指定锁的等待和过期时间
*
* @param timeoutMillis 超时时间(毫秒
* @param expireMillis 锁的超时时间（毫秒），过期删除
* @return 成功或失败标志
*/
private boolean lock(final long timeoutMillis, final long expireMillis) {
long nano = System.nanoTime();
long timeoutNano = TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
try {
do {
boolean ok = redis.lock(key, "true", expireMillis);
if (ok) {
this.locked = true;
return this.locked;
}
// 短暂休眠，避免出现活锁
Thread.sleep(3, RANDOM.nextInt(500));
} while ((System.nanoTime() - nano) < timeoutNano);
} catch (Exception e) {
throw lockFailedException;
}
return false;
}
/**
* 加锁 应该以： lock(); try { doSomething(); } finally { close()； } 的方式调用<br>
* 外部不建议直接使用该方法，建议使用{@link #tryAcquireRun(long, long, Supplier)}明确指定锁的等待和过期时间
*
* @return 成功或失败标志
*/
private boolean lock() {
return lock(DEFAULT_TIME_OUT_MILLIS);
}
/** 当获取锁失败的时候抛出该异常，方便调用方捕获处理 */
public static class AcquireLockFailException extends RuntimeException {
public AcquireLockFailException() {
super();
}
public AcquireLockFailException(String msg) {
super(msg);
}
public AcquireLockFailException(Throwable throwable) {
super(throwable);
}
public AcquireLockFailException(String msg, Throwable throwable) {
super(msg, throwable);
}
}
}

66
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/LocalRateLimiter.java Normal file
View File

@ -0,0 +1,66 @@
package cn.axzo.foundation.redis.support.impl;
import cn.axzo.foundation.redis.support.RateLimiter;
import cn.axzo.foundation.redis.support.RateLimiterFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimap;
import com.google.common.hash.Hashing;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
public class LocalRateLimiter implements RateLimiter {
private RateLimiterFactory.RateLimiterReq rateLimiterReq;
/**
* 使用 static 保持访问的状态
*/
static private Multimap<String, Long> accessLogs = ArrayListMultimap.create();
public LocalRateLimiter(RateLimiterFactory.RateLimiterReq rateLimiterReq) {
this.rateLimiterReq = rateLimiterReq;
}
public void cleanAccessLogs() {
accessLogs = ArrayListMultimap.create();
}
@Override
public Optional<Permit> tryAcquire(Object value, long step) {
Preconditions.checkArgument(value != null);
String hash = Hashing.murmur3_32().hashString(rateLimiterReq.getLimiterKey() + String.valueOf(value), Charset.defaultCharset()).toString();
Long nowSec = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
accessLogs.put(hash, nowSec);
List<Long> accessTimeSecs = ImmutableList.copyOf(accessLogs.get(hash));
List<LimitRule> rules = rateLimiterReq.getRules();
boolean failed = rules.stream().anyMatch(rule -> {
if (rateLimiterReq.getWindowType() == WindowType.SLIDING) {
return accessTimeSecs.stream().filter(t -> t > (nowSec - rule.getSeconds())).count() > rule.getPermits();
} else {
return accessTimeSecs.stream().filter(t -> ((nowSec - t) / rule.getSeconds()) == 0).count() > rule.getPermits();
}
});
if (failed) {
return Optional.empty();
}
return Optional.of(Permit.builder().cancelRunners(ImmutableList.of()).build());
}
@Override
public void reset(Object value) {
String hash = Hashing.murmur3_32().hashString(rateLimiterReq.getLimiterKey() + String.valueOf(value), Charset.defaultCharset()).toString();
accessLogs.removeAll(hash);
}
@Override
public WindowType getWindowType() {
return rateLimiterReq.getWindowType();
}
}

36
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RateLimiterFactoryImpl.java Normal file
View File

@ -0,0 +1,36 @@
package cn.axzo.foundation.redis.support.impl;
import cn.axzo.foundation.redis.support.RateLimiter;
import cn.axzo.foundation.redis.support.RateLimiterFactory;
import cn.axzo.foundation.web.support.AppRuntime;
import lombok.Builder;
import org.springframework.data.redis.core.RedisTemplate;
import java.util.Objects;
public class RateLimiterFactoryImpl implements RateLimiterFactory {
private RedisTemplate redisTemplate;
private AppRuntime appRuntime;
@Builder
public RateLimiterFactoryImpl(RedisTemplate redisTemplate, AppRuntime appRuntime) {
this.redisTemplate = redisTemplate;
this.appRuntime = appRuntime;
}
@Override
public RateLimiter build(RateLimiterFactory.RateLimiterReq rateLimiterReq) {
if (rateLimiterReq.getRateType() == RateType.LOCAL) {
return new LocalRateLimiter(rateLimiterReq);
}
Objects.requireNonNull(redisTemplate);
Objects.requireNonNull(appRuntime);
return RedisRateLimiter.builder()
.windowType(rateLimiterReq.getWindowType())
.limitRules(rateLimiterReq.getRules())
.limiterKey(rateLimiterReq.getLimiterKey())
.redisTemplate(redisTemplate)
.appRuntime(appRuntime)
.build();
}
}

189
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisEventBroadcastImpl.java Normal file
View File

@ -0,0 +1,189 @@
package cn.axzo.foundation.redis.support.impl;
import cn.axzo.foundation.redis.support.EventBroadcast;
import cn.axzo.foundation.web.support.AppRuntime;
import cn.axzo.foundation.web.support.utils.KeyBuilder;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.RedisTemplate;
import java.util.Map;
import java.util.Objects;
import java.util.function.BiConsumer;
@Slf4j
public class RedisEventBroadcastImpl implements EventBroadcast, InitializingBean, DisposableBean {
private RedisTemplate redisTemplate;
private AppRuntime appRuntime;
private String channel;
/**
* 广播监听. 监听channel上的广播. 并根据event dispatch到不同的consumer
*/
@Setter(AccessLevel.PROTECTED)
private BroadcastListener broadcastListener;
@Builder
public RedisEventBroadcastImpl(RedisTemplate redisTemplate, AppRuntime appRuntime) {
Objects.requireNonNull(redisTemplate);
Objects.requireNonNull(appRuntime);
this.redisTemplate = redisTemplate;
this.appRuntime = appRuntime;
}
@Override
public BroadcastQueue build(String queueName, BiConsumer<String, BroadcastEvent> consumer) {
Objects.requireNonNull(queueName);
Objects.requireNonNull(consumer);
Preconditions.checkArgument(queueName.length() <= 32);
RedisBroadcastQueue queue = RedisBroadcastQueue.builder()
.name(queueName)
.channel(channel)
.redisTemplate(redisTemplate)
.appRuntime(appRuntime)
.build();
broadcastListener.register(queueName, consumer);
return queue;
}
/**
* 构建RedisBroadcastListener, 并启动任务
*
* @throws Exception
*/
@Override
public void afterPropertiesSet() throws Exception {
this.channel = KeyBuilder.build(appRuntime, "broadcast", "channel");
broadcastListener = RedisBroadcastListener.builder()
.redisTemplate(redisTemplate)
.channel(channel)
.build();
broadcastListener.start();
}
@Override
public void destroy() throws Exception {
broadcastListener.stop();
}
@Data
@lombok.Builder
@NoArgsConstructor
@AllArgsConstructor
protected static final class RedisBroadcastQueue implements BroadcastQueue {
String name;
@Getter(AccessLevel.NONE)
RedisTemplate redisTemplate;
@Getter(AccessLevel.NONE)
String channel;
@Getter(AccessLevel.NONE)
AppRuntime appRuntime;
@Override
public boolean broadcast(JSONObject data) {
return (Boolean) redisTemplate.execute((RedisCallback<Boolean>) connection -> {
connection.publish(channel.getBytes(),
BroadcastEvent.builder()
.name(name)
.data(data)
.senderRuntime(appRuntime.toJson())
.build()
.toJSONString().getBytes(Charsets.UTF_8));
return true;
});
}
}
protected static final class RedisBroadcastListener implements BroadcastListener {
private RedisTemplate redisTemplate;
private String channel;
private final Map<String, BiConsumer<String, BroadcastEvent>> broadcastConsumer = Maps.newConcurrentMap();
@lombok.Builder
public RedisBroadcastListener(RedisTemplate redisTemplate, String channel) {
this.redisTemplate = redisTemplate;
this.channel = channel;
}
@Override
public boolean start() {
try {
redisTemplate.getConnectionFactory().getConnection()
.subscribe((message, pattern) -> {
BroadcastEvent broadcastEvent = JSONObject.parseObject(new String(message.getBody(), Charsets.UTF_8))
.toJavaObject(BroadcastEvent.class);
onEvent(broadcastEvent);
}, channel.getBytes());
} catch (Exception e) {
log.error("====== start broadcast listener error =====", e);
return false;
}
log.info("====== start broadcast listener =====");
return true;
}
@Override
public boolean register(String queueName, BiConsumer<String, BroadcastEvent> consumer) {
Preconditions.checkArgument(!broadcastConsumer.containsKey(queueName), "duplicate broadcast queue");
broadcastConsumer.put(queueName, consumer);
return true;
}
@Override
public boolean stop() {
//doNothing, 如果定义了线程池可以在这里销毁
log.info("====== stop broadcast listener =====");
return true;
}
private void onEvent(BroadcastEvent event) {
BiConsumer<String, BroadcastEvent> consumer = broadcastConsumer.get(event.getName());
if (consumer == null) {
log.error("event is ready, but no consumer found, event = {}", event.toJSONString());
return;
}
try {
consumer.accept(event.getName(), event);
} catch (Exception ex) {
log.error("consume broadcast error, event = {}", event.toJSONString());
//ignore 忽略业务异常
}
}
}
/**
* 广播监听. 提供开始监听, 结束监听, 注册queue以及对应的BiConsumer
*/
interface BroadcastListener {
boolean start();
/**
* 注册queue对应的consumer
*
* @param queueName
* @param consumer
* @return
*/
boolean register(String queueName, BiConsumer<String, BroadcastEvent> consumer);
boolean stop();
}
}

178
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisLocalCacheCoordinate.java Normal file
View File

@ -0,0 +1,178 @@
package cn.axzo.foundation.redis.support.impl;
import cn.axzo.foundation.enums.AppEnvEnum;
import cn.axzo.foundation.redis.support.EventBroadcast;
import cn.axzo.foundation.redis.support.LocalCacheCoordinate;
import cn.axzo.foundation.web.support.AppRuntime;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import lombok.AccessLevel;
import lombok.Data;
import lombok.Setter;
import lombok.Singular;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.transaction.support.TransactionSynchronizationAdapter;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.function.Supplier;
@Slf4j
public class RedisLocalCacheCoordinate implements LocalCacheCoordinate, InitializingBean {
@Setter(AccessLevel.PROTECTED)
private AfterCommitExecutor executor;
@Setter
private EventBroadcast EventBroadcast;
private EventBroadcast.BroadcastQueue broadcastQueue;
@Singular
private Multimap<String, Consumer<CacheDirtiedEvent>> cacheDirtyHandlers = ArrayListMultimap.create();
private List<CacheDirtiedEventInterceptor> interceptors = Lists.newArrayList();
@Override
public void notifyCacheDirty(CacheDirtiedEvent event) {
executor.execute(() -> {
broadcastQueue.broadcast(JSON.parseObject(JSONObject.toJSONString(event)));
log.info("notify cache dirty, event={}", event);
});
}
@Override
public void registerCacheDirtyListener(String key, Consumer<CacheDirtiedEvent> consumer) {
cacheDirtyHandlers.put(key, consumer);
}
@Override
public void registerCacheDirtyInterceptor(CacheDirtiedEventInterceptor interceptor) {
interceptors.add(interceptor);
}
void onCacheDirtied(String queueName, EventBroadcast.BroadcastEvent e) {
CacheDirtiedEvent cacheDirtiedEvent = CacheDirtiedEvent.builder().key(e.getData().getString("key"))
.data(e.getData().getJSONObject("data"))
.build();
interceptors.forEach(h -> {
try {
if (h.getFilter().test(cacheDirtiedEvent)) {
h.getConsumer().accept(cacheDirtiedEvent);
log.info("interceptor handled cache dirty event={}, interceptor={}", cacheDirtiedEvent, h);
}
} catch (Exception ex) {
log.error("========interceptor handle cacheDirtiedEvent {}", cacheDirtiedEvent, ex);
}
});
cacheDirtyHandlers.get(cacheDirtiedEvent.getKey()).forEach(h -> {
try {
h.accept(cacheDirtiedEvent);
log.info("handler handled cache dirty event={}, handler={}", cacheDirtiedEvent, h);
} catch (Exception ex) {
log.error("========handler handle cacheDirtiedEvent {}", cacheDirtiedEvent, ex);
}
});
}
@Override
public void afterPropertiesSet() throws Exception {
broadcastQueue = EventBroadcast.build("Local_cache-coordinate", this::onCacheDirtied);
}
public static RedisLocalCacheCoordinate.Builder builder() {
return new RedisLocalCacheCoordinate.Builder();
}
@Data
public static class Builder {
EventBroadcast EventBroadcast;
AppRuntime appRuntime;
public RedisLocalCacheCoordinate.Builder EventBroadcast(EventBroadcast EventBroadcast) {
this.EventBroadcast = EventBroadcast;
return this;
}
public RedisLocalCacheCoordinate.Builder appRuntime(AppRuntime appRuntime) {
this.appRuntime = appRuntime;
return this;
}
public LocalCacheCoordinate build() {
if (appRuntime.getEnv() == AppEnvEnum.unittest) {
return new LocalCacheCoordinate() {
private Multimap<String, Consumer<CacheDirtiedEvent>> cacheDirtyHandlers = ArrayListMultimap.create();
private List<CacheDirtiedEventInterceptor> interceptors = Lists.newArrayList();
@Override
public void notifyCacheDirty(CacheDirtiedEvent event) {
cacheDirtyHandlers.get(event.getKey())
.forEach(h -> h.accept(event));
interceptors.stream()
.filter(i -> i.getFilter().test(event))
.forEach(i -> i.getConsumer().accept(event));
}
@Override
public void registerCacheDirtyListener(String key, Consumer<CacheDirtiedEvent> consumer) {
cacheDirtyHandlers.put(key, consumer);
}
@Override
public void registerCacheDirtyInterceptor(CacheDirtiedEventInterceptor interceptor) {
interceptors.add(interceptor);
}
};
}
Objects.requireNonNull(EventBroadcast);
RedisLocalCacheCoordinate client = new RedisLocalCacheCoordinate();
client.setEventBroadcast(EventBroadcast);
client.setExecutor(new AfterCommitExecutor());
return client;
}
}
/**
* stolen from http://azagorneanu.blogspot.jp/2013/06/transaction-synchronization-callbacks.html
* 保证在交易结束后被调用.
*/
protected static class AfterCommitExecutor extends TransactionSynchronizationAdapter {
//大部分情况只有会清除一个缓存. 因此数组初始化为1
private final ThreadLocal<List<Runnable>> contexts = ThreadLocal
.withInitial((Supplier<List<Runnable>>) () -> Lists.newArrayListWithCapacity(1));
public void execute(Runnable runnable) {
if (!TransactionSynchronizationManager.isSynchronizationActive()) {
runnable.run();
return;
}
contexts.get().add(runnable);
TransactionSynchronizationManager.registerSynchronization(this);
}
@Override
public void afterCommit() {
contexts.get().forEach(e -> {
try {
e.run();
} catch (Exception ex) {
log.error("Failed to execute runnable = {} ", e, ex);
}
});
}
@Override
public void afterCompletion(int status) {
contexts.remove();
}
}
}

259
redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisRateLimiter.java Normal file
View File

@ -0,0 +1,259 @@
package cn.axzo.foundation.redis.support.impl;
import cn.axzo.foundation.redis.support.RateLimiter;
import cn.axzo.foundation.web.support.AppRuntime;
import cn.axzo.foundation.web.support.utils.KeyBuilder;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.hash.Hashing;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.BoundValueOperations;
import org.springframework.data.redis.core.BoundZSetOperations;
import org.springframework.data.redis.core.RedisTemplate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@Slf4j
public class RedisRateLimiter implements RateLimiter {
private AppRuntime appRuntime;
private RedisTemplate redisTemplate;
private RateLimiterWorker rateLimiterWorker;
/**
* 自定义的key, 避免redisKey冲突. 必填
*/
private String limiterKey;
private List<LimitRule> limitRules;
/**
* 窗口保存最大时长. 主要针对Sliding方式窗口的zSet过期
*/
private Integer maxWindowDurationHour;
private WindowType windowType;
@Builder
RedisRateLimiter(AppRuntime appRuntime,
RedisTemplate redisTemplate,
WindowType windowType,
String limiterKey,
List<LimitRule> limitRules,
Integer maxWindowDurationHour) {
Objects.requireNonNull(appRuntime);
Objects.requireNonNull(redisTemplate);
Objects.requireNonNull(windowType);
Objects.requireNonNull(limitRules);
Objects.requireNonNull(limiterKey);
Preconditions.checkArgument(!limitRules.isEmpty());
if (limitRules.stream().anyMatch(p -> !p.isValid())) {
throw new RuntimeException(String.format("invalid rate expression, expression = %s", JSONObject.toJSONString(limitRules)));
}
this.windowType = windowType;
this.appRuntime = appRuntime;
this.redisTemplate = redisTemplate;
this.limitRules = limitRules;
this.limiterKey = limiterKey;
this.maxWindowDurationHour = Optional.ofNullable(maxWindowDurationHour).orElse(24);
this.rateLimiterWorker = buildWorker(windowType);
}
@Override
public Optional<Permit> tryAcquire(Object value, long step) {
if (!rateLimiterWorker.tryAcquire(value)) {
return Optional.empty();
}
List<Runnable> cancelRunners = rateLimiterWorker.visit(value, step);
return Optional.of(Permit.builder()
.cancelRunners(cancelRunners)
.build());
}
@Override
public void reset(Object value) {
rateLimiterWorker.reset(value);
}
@Override
public WindowType getWindowType() {
return windowType;
}
private RateLimiterWorker buildWorker(WindowType windowType) {
if (windowType == WindowType.FIXED) {
return new FixedWindowRateLimiter();
}
if (windowType == WindowType.FIXED_BUCKET) {
return new FixedBucketWindowRateLimiter();
}
if (windowType == WindowType.SLIDING) {
return new SlidingWindowRateLimiter();
}
throw new RuntimeException(String.format("unsupported window type, window type = %s", windowType));
}
private String buildRedisKey(Object value) {
String hash = Hashing.murmur3_128().newHasher()
.putString(limiterKey, Charsets.UTF_8)
.putString(String.valueOf(value), Charsets.UTF_8)
.hash()
.toString();
return KeyBuilder.build(appRuntime, "rl", getWindowType().getShortName(), limiterKey, hash);
}
/**
* 固定窗口限流, 窗口起始时间第一次tryAcquire时时间. 窗口大小为WindowDuration
* <pre>
* key = value + WindowDuration.
* 在该窗口被访问时, 计数器+1. 窗口持续时长为WindowDuration. 并依赖redis ttl销毁
* 窗口被销毁后, 重置计数器
* </pre>
*/
class FixedWindowRateLimiter implements RateLimiterWorker {
public List<Runnable> visit(Object value, long step) {
List<Runnable> cancels = new ArrayList<>(limitRules.size());
//根据时间构建key, 避免hash无法删除
limitRules.stream()
.forEach(e -> {
String key = buildLimiterKey(value, e);
BoundValueOperations op = redisTemplate.boundValueOps(key);
Long result = op.increment(step);
//第一次访问时设置过期时间, 以确定窗口
if (result == step) {
redisTemplate.expire(key, e.getSeconds(), TimeUnit.SECONDS);
}
cancels.add(() -> op.decrement(step));
});
return cancels;
}
@Override
public void reset(Object value) {
limitRules.stream().forEach(e -> {
String key = buildLimiterKey(value, e);
redisTemplate.delete(key);
});
}
public boolean tryAcquire(Object value) {
boolean anyMatch = limitRules.stream()
.anyMatch(limitRule -> {
String key = buildLimiterKey(value, limitRule);
return limitRule.getPermits() <= Optional.ofNullable(redisTemplate.opsForValue().get(key))
.map(e -> Long.parseLong(e.toString()))
.orElse(0L);
});
return !anyMatch;
}
protected String buildLimiterKey(Object value, LimitRule limitRule) {
return buildRedisKey(value + ":" + limitRule.getSeconds());
}
}
/**
* 滑动窗口限流, 每次获取令牌成功时间加入到zset. 后续获取令牌时每次检查zset中WindowDuration中已获取令牌数. 并判断是否可以继续获取令牌
* <pre>
* key = value
* zset value = currentMillis. score = currentMillis
* 获取令牌时, 在计算zset中 score = [currentMillis-WindowDuration, currentMillis} 的element数量
* </pre>
*/
class SlidingWindowRateLimiter implements RateLimiterWorker {
//当zset的element达到一定数量时, 清理该zet. 避免redis内存泄露
private static final int CLEAN_KEY_THRESHOLD = 1000;
private AtomicLong visitCounter = new AtomicLong();
public List<Runnable> visit(Object value, long step) {
Preconditions.checkArgument(step == 1, "滑动窗口只支持 step=1");
String key = buildRedisKey(value);
long now = System.currentTimeMillis();
String member = String.valueOf(now);
final BoundZSetOperations op = redisTemplate.boundZSetOps(key);
op.add(member, now);
redisTemplate.expire(key, maxWindowDurationHour, TimeUnit.HOURS);
if (visitCounter.incrementAndGet() > CLEAN_KEY_THRESHOLD) {
//删除过期的访问记录
op.removeRangeByScore(0, now - TimeUnit.HOURS.toMillis(maxWindowDurationHour));
visitCounter.set(0);
}
return ImmutableList.of(() -> op.remove(member));
}
@Override
public void reset(Object value) {
String key = buildRedisKey(value);
redisTemplate.delete(key);
}
public boolean tryAcquire(Object value) {
String key = buildRedisKey(value);
long now = System.currentTimeMillis();
//检查所有的rule, 如果有其中一个失败则跳出
return !limitRules.stream()
.anyMatch(p -> p.getPermits() <= redisTemplate.opsForZSet().count(key, now - TimeUnit.SECONDS.toMillis(p.getSeconds()), now));
}
}
/**
* 固定窗口限流, 窗口根据自然时间向前滚动
* 继承自FixedWindowRateLimiter, 区别在于构建key的方式不一样
* <pre>
* key = value + currentMillis/WindowDuration.
* currentMillis/WindowDuration会把自然时间分割为长为WindowDuration的片段. 片段有效期为WindowDuration
* 获取令牌时在该片段上检查是否有剩余令牌
* </pre>
*/
class FixedBucketWindowRateLimiter extends FixedWindowRateLimiter implements RateLimiterWorker {
@Override
protected String buildLimiterKey(Object value, LimitRule limitRule) {
// System.currentTimeMillis()是当前距离1970-01-01 08:00:00的毫秒数（假设系统为+8时区）
// 这里希望是距离1970-01-01 00:00:00（GMT标准时间）的毫秒数，所以需要+8个小时的毫秒
long currentGMTMillis = System.currentTimeMillis() + TimeUnit.HOURS.toMillis(8);
return buildRedisKey(value + ":" + currentGMTMillis / TimeUnit.SECONDS.toMillis(limitRule.getSeconds()));
}
}
interface RateLimiterWorker {
/**
* 尝试获取令牌
*
* @param value
* @return 如果获取成功则返回true, 失败则为false
*/
boolean tryAcquire(Object value);
/**
* 获取令牌完成后增加步长为 1的计数器
*
* @param value
* @return limit key
*/
default List<Runnable> visit(Object value) {
return visit(value, 1);
}
/**
* 获取令牌完成后增加指定步长的计数器
*
* @param value
* @param step
* @return limit key
*/
List<Runnable> visit(Object value, long step);
/**
* 重置value对应的锁
*/
void reset(Object value);
}
}

2
unittest-support-lib/src/main/java/cn/axzo/foundation/unittest/support/config/RedisConfig.java
View File

@ -17,7 +17,7 @@ public class RedisConfig {
@PostConstruct
public void postConstruct() {
redisServer = new RedisServer(16379);
redisServer = RedisServer.builder().port(16739).setting("maxheap 128m").build();
// redis server会在后台启动一个redis server的进程，默认IP=127.0.0.1
redisServer.start();
}

96
web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java Normal file
View File

@ -0,0 +1,96 @@
package cn.axzo.foundation.web.support;
import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
/**
* 间隔一定时间刷新的缓存。
* <pre>
* 1、初次调用时，直接获取需要缓存的内容，并缓存到cache中
* 2、每间隔intervalMillis尝试刷新缓存，
* 如果刷新缓存成功，更新cache缓存的值
* 如果刷新缓存失败，则不更新
* </pre>
*
* @param <T>
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
@Slf4j
public class TimerRefreshCache<T> {
private ScheduledThreadPoolExecutor executor;
private String name;
private Long initialDelayMillis;
private Long intervalMillis;
private Supplier<T> refresher;
private Cache<String, T> cache;
/** 缓存是否以初始化，如果为false，会进行主动加载进行初始化 */
private AtomicBoolean initialized = new AtomicBoolean(false);
public TimerRefreshCache(String name, Long initialDelayMillis, Long intervalMillis,
ScheduledThreadPoolExecutor executor, Supplier<T> refresher) {
Preconditions.checkArgument(executor != null);
Preconditions.checkArgument(refresher != null);
Preconditions.checkArgument(intervalMillis != null);
this.name = name;
this.initialDelayMillis = initialDelayMillis;
this.intervalMillis = intervalMillis;
this.refresher = refresher;
// 该方法获取的，往往是完整的缓存内容。所以maxSize暂定为2即可
long maximumSize = 2L;
cache = CacheBuilder.newBuilder()
.maximumSize(maximumSize)
.build();
this.executor = executor;
startTimer();
}
private void startTimer() {
executor.scheduleAtFixedRate(this::doRefresh, initialDelayMillis, intervalMillis, TimeUnit.MILLISECONDS);
}
private boolean doRefresh() {
try {
T value = refresher.get();
// 当返回为null，表示Value没有变化，这时不用刷新Cache
if (value == null) {
return true;
}
cache.put(name, value);
if (log.isDebugEnabled()) {
log.debug("{} refreshed, new value={}", name, value);
}
} catch (Throwable e) {
log.error("{} refresh failed", name, e);
return false;
}
return true;
}
public T get() {
// 如果没有加载过且没有缓存，手动加载一次
if (!initialized.getAndSet(true) && cache.size() == 0) {
//如果第一次刷新缓存失败. 则重置initialized. 下次请求进入时再次尝试更新cache
if (!doRefresh()) {
initialized.set(false);
}
}
return cache.getIfPresent(name);
}
// XXX: for unittest
public void put(T configs) {
log.info("{},set->config={}", TimerRefreshCache.this, configs);
cache.put(name, configs);
}
}

9
web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClient.java
View File

@ -14,6 +14,14 @@ public interface AlertClient {
void post(Alert alert);
default void post(String key, Throwable ex, String message, Object... objects) {
post(new Alert(key, ex, message, objects));
}
default void post(String key, Throwable ex, String message) {
post(new Alert(key, ex, message, null));
}
@Data
class Alert {
private static final int MAX_MESSAGE_LENGTH = 8_000;
@ -25,7 +33,6 @@ public interface AlertClient {
String message;
String stack;
@Builder
public Alert(String key, Throwable ex, String message, Object... objects) {
this.key = key;
this.message = message;

10
web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClientImpl.java
View File

@ -6,6 +6,7 @@ import com.google.common.collect.Multimaps;
import com.google.common.util.concurrent.RateLimiter;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import java.time.LocalDateTime;
import java.util.Collection;
@ -26,6 +27,12 @@ public class AlertClientImpl implements AlertClient {
ListMultimap<AlertKey, AlertMessage> alertsMap = Multimaps.synchronizedListMultimap(ArrayListMultimap.create());
/**
* @param consumer
* @param executor
* @param period 默认间隔多少分钟发告警邮件
* @param consumeImmediatelyPreSecond 如果每秒出现多少次异常告警阈值
*/
@Builder
public AlertClientImpl(Consumer<Map<AlertKey, Collection<AlertMessage>>> consumer,
ScheduledThreadPoolExecutor executor,
@ -68,6 +75,9 @@ public class AlertClientImpl implements AlertClient {
private void consume() {
Map<AlertKey, Collection<AlertMessage>> map = alertsMap.asMap();
if (CollectionUtils.isEmpty(map)) {
return;
}
try {
consumer.accept(map);
} catch (Exception ex) {

6
web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/EmailAlertConsumer.java
View File

@ -1,6 +1,8 @@
package cn.axzo.foundation.web.support.alert;
import cn.axzo.foundation.enums.AppEnvEnum;
import cn.axzo.foundation.web.support.AppRuntime;
import com.alibaba.fastjson.JSONObject;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
@ -57,6 +59,10 @@ public class EmailAlertConsumer implements Consumer<Map<AlertClient.AlertKey, Co
@Override
public void accept(Map<AlertClient.AlertKey, Collection<AlertClient.AlertMessage>> alerts) {
if (appRuntime.getEnv() == AppEnvEnum.local) {
log.error("local alerts: {}", JSONObject.toJSONString(alerts));
return;
}
Message message = new MimeMessage(session);
try {
message.setFrom(new InternetAddress(from));

40
web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java Normal file
View File

@ -0,0 +1,40 @@
package cn.axzo.foundation.web.support.apps;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.commons.lang3.RegExUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.Optional;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class App {
Long id;
Long upstreamId;
String upstreamName;
String appName;
Integer port;
String host;
public String getHost() {
return Optional.ofNullable(host).orElse(String.format("http://%s:%s", appName, port));
}
/**
* 根据upstreamId的后4位生产appId, 如果后四位位0则替换为9
* eg 20000000000000124 -> 9124, 20000000000003078 -> 3078
*/
public String getAppId() {
String right = StringUtils.right(String.valueOf(upstreamId), 4);
if (!right.startsWith("0")) {
return right;
}
return RegExUtils.replaceFirst(right, "0", "9");
}
}

9
web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java Normal file
View File

@ -0,0 +1,9 @@
package cn.axzo.foundation.web.support.apps;
import java.util.List;
public interface AppCenter {
List<App> listAll();
App getByName(String appName);
}

91
web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java Normal file
View File

@ -0,0 +1,91 @@
package cn.axzo.foundation.web.support.apps;
import cn.axzo.foundation.page.PageResp;
import cn.axzo.foundation.util.PageUtils;
import cn.axzo.foundation.web.support.TimerRefreshCache;
import cn.axzo.foundation.web.support.rpc.RpcClient;
import cn.axzo.foundation.web.support.rpc.RpcClientImpl;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import java.util.*;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Slf4j
public class AppCenterImpl implements AppCenter {
private static final long INITIAL_DELAY_MILLIS = 0;
private static final long REFRESH_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(60);
private TimerRefreshCache<Map<String, App>> appCache;
private RpcClient rpcClient;
private String debugHost;
private String listAppUrl;
private Map<String, String> debugAppRoutes;
/**
* executor 为刷新的executor
* debugHost: 本地调试时 连接的环境 eg: http://pre-api.axzo.cn
* debugAppRoutes: 本地调试时部分appName 与apiSix配置的路由不一致,
* eg data-collection 对应的测试地址http://test-api.axzo.cn/dataCollection 而非http://test-api.axzo.cn/data-collection
* 因此该类映射需要单独处理
*/
@Builder
public AppCenterImpl(ScheduledThreadPoolExecutor executor,
String debugHost,
Map<String, String> debugAppRoutes) {
Objects.requireNonNull(executor);
this.rpcClient = RpcClientImpl.builder().build();
this.appCache = new TimerRefreshCache<>("appCenterCache", INITIAL_DELAY_MILLIS,
REFRESH_INTERVAL_MILLIS, executor, this::loadApps);
this.debugHost = debugHost;
this.listAppUrl = Optional.ofNullable(Strings.emptyToNull(debugHost)).map(e -> e + "/apisix-plat").orElse("http://apisix-plat:8080")
+ "/api/v1/upstream/list";
this.debugAppRoutes = Optional.ofNullable(debugAppRoutes).orElse(ImmutableMap.of());
}
private Map<String, App> loadApps() {
List<App> apps = PageUtils.drainAll(page -> {
JSONObject result = rpcClient.request()
.url(listAppUrl)
.content(new JSONObject()
.fluentPut("pageNum", page)
//分页接口有问题, 这里设置一个最大值. 一次加载完所有的数据
.fluentPut("pageSize", Integer.MAX_VALUE))
.clz(JSONObject.class)
.post();
//结构不一样, 转换为自己的pageResp
return PageResp.<App>builder()
.total(result.getLong("totalCount"))
.current(result.getLong("page"))
.size(result.getLong("pageSize"))
.data(Optional.ofNullable(result.getJSONArray("list")).orElse(new JSONArray())
.toJavaList(App.class))
.build();
});
if (!Strings.isNullOrEmpty(debugHost)) {
apps.forEach(e -> e.setHost(debugHost + "/" + debugAppRoutes.getOrDefault(e.getAppName(), e.getAppName())));
}
return apps.stream()
.collect(Collectors.toMap(App::getAppName, e -> e, (o, n) -> n));
}
@Override
public List<App> listAll() {
return new ArrayList<>(appCache.get().values());
}
@Override
public App getByName(String appName) {
return appCache.get().get(appName);
}
}

2
web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultJsonConverter.java
View File

@ -12,6 +12,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import lombok.Builder;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.http.HttpOutputMessage;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
@ -121,7 +122,6 @@ public class ApiResultJsonConverter extends FastJsonHttpMessageConverter {
//只有抛出异常或者返回失败时才会设置header
outputMessage.getHeaders().add(HTTP_CODE_HEADER_KEY, result.getHttpCode() + "");
outputMessage.getHeaders().add(ERROR_CODE_HEADER_KEY, result.getCode());
//如果header中没有pretty. 按照定义好的fastJson的输出方式. 否则自定义美化输出
writeJson(result, outputMessage);

18
web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java
View File

@ -3,9 +3,7 @@ package cn.axzo.foundation.web.support.config;
import cn.axzo.foundation.result.ApiResult;
import cn.axzo.foundation.result.ResultCode;
import cn.axzo.foundation.web.support.AppRuntime;
import com.google.common.base.Strings;
import lombok.Builder;
import org.apache.commons.lang3.StringUtils;
import java.util.Optional;
@ -23,20 +21,6 @@ class ApiResultWrapper<T> extends ApiResult<T> {
this.httpCode = Optional.ofNullable(result.getHttpCode()).orElse(ResultCode.DEFAULT_HTTP_ERROR_CODE);
//如果是系统的或者一个完整的errorCode. 直接处理
//目前已在系统中使用的ErrorCode格式都在6个字符以上；业务自定义code在6个字符以内(如果超过6个字符会直接返回)；
// 如:SYS_100001,SSO_100001,op-leads_5001001,10011002,1001X101；
if (code.contains("_") || code.length() > 6) {
return;
}
//没有appId时沿用之前的拼装逻辑"${appName}_${httpCode}${ErrorCode}"
//存在appId时拼装逻辑调整为"${appId}${ErrorCode}"
if (StringUtils.isEmpty(appRuntime.getAppId())) {
this.code = Strings.nullToEmpty(appRuntime.getAppName()).toUpperCase() + "_" + getHttpCode() + result.getCode();
} else {
this.code = appRuntime.getAppId() + result.getCode();
}
this.code = getStandardCode(appRuntime.getAppId(), result.getCode());
}
}

18
web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java
View File

@ -4,9 +4,7 @@ import cn.axzo.foundation.util.FastjsonUtils;
import cn.axzo.foundation.web.support.AppRuntime;
import cn.axzo.foundation.web.support.context.AxContextInterceptor;
import cn.axzo.foundation.web.support.exception.AbstractExceptionHandler;
import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor;
import cn.axzo.foundation.web.support.interceptors.PrettyPrintInterceptor;
import cn.axzo.foundation.web.support.interceptors.PrintVerboseInterceptor;
import cn.axzo.foundation.web.support.interceptors.*;
import cn.axzo.foundation.web.support.resolvers.AxContextResolver;
import cn.axzo.foundation.web.support.resolvers.CallerAppResolver;
import com.alibaba.fastjson.JSONArray;
@ -83,8 +81,8 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement
@Value("${web.serialize.browser-compatible.enabled:true}")
private Boolean browserCompatible;
@Value("${web.context.supplier.url:}")
private String contextSupplierUrl;
@Value("${web.debug.host:}")
private String debugHost;
/**
* 自定义对返回的errorMsg进行处理
@ -187,15 +185,15 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement
registry.addInterceptor(new CorsInterceptor(corsProcessorType));
registry.addInterceptor(new PrettyPrintInterceptor());
registry.addInterceptor(new CallerAppInterceptor(appRuntime));
registry.addInterceptor(new TraceInterceptor());
registry.addInterceptor(new CheckDeathInterceptor());
super.addInterceptors(registry);
if (null != handlerInterceptors) {
Arrays.stream(handlerInterceptors).forEach(e -> {
registry.addInterceptor(e);
});
Arrays.stream(handlerInterceptors).forEach(registry::addInterceptor);
}
registry.addInterceptor(new AxContextInterceptor(appRuntime, contextSupplierUrl));
registry.addInterceptor(new AxContextInterceptor(appRuntime, debugHost));
}
/**
@ -236,6 +234,8 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement
//添加一个空的根实现. 避免爬虫访问到根路径后一直打印PageNotFound. 项目中可以覆盖 / 根路径的实现
registry.addStatusController("/", HttpStatus.NO_CONTENT);
registry.addStatusController("/favicon.ico", HttpStatus.NO_CONTENT);
//这里必须添加一个映射配合CheckDeathInterceptor使用, 否则会出现NoMapping
registry.addStatusController("/checkDeath", HttpStatus.NO_CONTENT);
super.addViewControllers(registry);
}

20
web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java
View File

@ -8,7 +8,7 @@ import cn.axzo.foundation.web.support.rpc.RequestParams;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import lombok.RequiredArgsConstructor;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
@ -24,11 +24,17 @@ import java.util.Optional;
* 3. 非prd环境支持通过token到puge换
*/
@Slf4j
@RequiredArgsConstructor
public class AxContextInterceptor implements HandlerInterceptor {
private final AppRuntime appRuntime;
private final String supplierHost;
private final String supplierUrl;
@Builder
public AxContextInterceptor(AppRuntime appRuntime, String debugHost) {
this.appRuntime = appRuntime;
this.supplierUrl = Optional.ofNullable(Strings.emptyToNull(debugHost)).map(e -> e + "/pudge")
.orElse("http://pudge:10099") + "/webApi/oauth/apisix/authentication";
}
private final static HttpClient HTTP_CLIENT = OkHttpClientImpl.builder().build();
@ -83,18 +89,18 @@ public class AxContextInterceptor implements HandlerInterceptor {
return JSONObject.parseObject(StringUtils.removeStart(authorization, "Raw "), AxContext.class);
}
if (authorization.startsWith("Bearer")) {
if (Strings.isNullOrEmpty(supplierHost)) {
if (Strings.isNullOrEmpty(supplierUrl)) {
return null;
}
String result;
try {
result = HTTP_CLIENT.get(supplierHost, RequestParams.FormParams.builder()
result = HTTP_CLIENT.get(supplierUrl, RequestParams.FormParams.builder()
.headers(ImmutableMap.of("Authorization", authorization,
"terminal", request.getHeader("terminal")))
"terminal", Strings.nullToEmpty(request.getHeader("terminal"))))
.logEnable(true)
.build());
} catch (Exception ex) {
log.error("获取登陆信息错误, url = {}, authorization = {}", supplierHost, authorization, ex);
log.error("获取登陆信息错误, url = {}, authorization = {}", supplierUrl, authorization, ex);
return null;
}
//这里是一个非标准返回

23
web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/CheckDeathInterceptor.java Normal file
View File

@ -0,0 +1,23 @@
package cn.axzo.foundation.web.support.interceptors;
import org.springframework.http.HttpStatus;
import org.springframework.web.servlet.HandlerInterceptor;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
public class CheckDeathInterceptor implements HandlerInterceptor {
private static final String CHECK_DEATH_URL = "/checkDeath";
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
if (CHECK_DEATH_URL.equals(request.getRequestURI())) {
response.setStatus(HttpStatus.OK.value());
return false;
}
return true;
}
}

33
web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/TraceInterceptor.java Normal file
View File

@ -0,0 +1,33 @@
package cn.axzo.foundation.web.support.interceptors;
import cn.axzo.foundation.util.TraceUtils;
import com.google.common.base.Strings;
import org.springframework.web.servlet.HandlerInterceptor;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* 在request上下文中添加 traceId
*/
public class TraceInterceptor implements HandlerInterceptor {
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
String upstreamTraceId = request.getHeader(TraceUtils.TRACE_ID);
if (!Strings.isNullOrEmpty(upstreamTraceId)) {
// 优先使用上游的 traceId
TraceUtils.putTraceId(upstreamTraceId);
}
String traceId = TraceUtils.getOrCreateTraceId();
request.setAttribute(TraceUtils.TRACE_ID_IN_MDC, traceId);
return true;
}
@Override
public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception {
TraceUtils.removeTraceId();
}
}

9
web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/OkHttpClientImpl.java
View File

@ -22,12 +22,11 @@ import java.util.function.Function;
public class OkHttpClientImpl implements HttpClient {
private OkHttpClient okHttpClient;
private Call.Factory callFactory;
private Config config;
private List<Interceptor> interceptorList;
private Set<String> logResponseHeaderNames;
private final List<Interceptor> interceptorList;
private final Set<String> logResponseHeaderNames;
private final static long MAX_RESPONSE_LOG_LENGTH = 10_240L;
private static final long MAX_RESPONSE_LOG_LENGTH = 10_240L;
@Builder
public OkHttpClientImpl(Config config, List<Interceptor> interceptorList,
@ -169,7 +168,7 @@ public class OkHttpClientImpl implements HttpClient {
.connectionPool(new ConnectionPool(config.getMaxIdleConnections(), config.getKeepAliveMinutes(), TimeUnit.MINUTES))
.dispatcher(dispatcher);
interceptorList.forEach(interceptor -> builder.addInterceptor(interceptor));
interceptorList.forEach(builder::addInterceptor);
if (null != clientBuilder) {
clientBuilder.accept(builder);

44
web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java
View File

@ -2,22 +2,18 @@ package cn.axzo.foundation.web.support.rpc;
import cn.axzo.foundation.page.PageResp;
import cn.axzo.foundation.result.ApiResult;
import cn.axzo.foundation.web.support.context.AxContext;
import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import lombok.extern.slf4j.Slf4j;
import java.util.*;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
* Rpc调用客户端接口
@ -101,40 +97,9 @@ public interface RpcClient {
return delete(url, typeReference, requestParams);
}
default <T> ApiResult<T> convert(String body, Class<T> clz) {
return JSONObject.parseObject(body, new TypeReference<ApiResult<T>>(clz) {
});
}
Set<String> AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo");
// XXX: http/2会把所有Header都转成小写, 历史定义的Header都是大写的，在http/2协议下会透传失败。
TreeSet<String> CASE_INSENSITIVE_AXZO_HEADERS = AXZO_HEADERS.stream()
.collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER)));
// 将axzo-开头的header复制到请求的下一跳
String AZXO_HEADER_PREFIX = "axzo-";
List<Supplier<Map<String, String>>> DEFAULT_HEADER_SUPPLIERS = ImmutableList.of(
() -> AxContext.getRequest()
.map(request -> Collections.list(request.getHeaderNames()).stream()
// 通过http2协议的请求，默认会把大写转成小写，如"_SESSION_OBJECT" -> "_session_object"，导致session无法透传，下一跳需要通过session验签会失败
.filter(p -> CASE_INSENSITIVE_AXZO_HEADERS.contains(p) || p.startsWith(AZXO_HEADER_PREFIX))
.collect(Collectors.toMap(e -> e, e -> request.getHeader(e), (oldValue, newValue) -> newValue)))
.orElse(Collections.emptyMap()),
//设置callerApp
() -> AxContext.getRequest().map(e -> {
Object caller = e.getAttribute(CallerAppInterceptor.NEXT_HTTP_REQUEST_ATTRIBUTE);
return ImmutableMap.of(CallerAppInterceptor.HTTP_REQUEST_HEADER, JSONObject.toJSONString(caller));
}).orElse(ImmutableMap.of())
);
/**
* 使用builder模式来发起请求, 先通过request()获得builder对象. 再构建具体的请求方式
* eg: String resp = rpcClient.request().url("/my").content(Map.of("key", "value")).clz(String.class).post();
*
* @return
*/
default RpcRequestBuilder request() {
return new RpcRequestBuilder(this);
@ -142,7 +107,6 @@ public interface RpcClient {
@Slf4j
class RpcRequestBuilder {
private static final long MAX_PER_PAGE_COUNT = 1000;
private String url;
private Object content;
private Class clz;

57
web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java
View File

@ -2,19 +2,22 @@ package cn.axzo.foundation.web.support.rpc;
import cn.axzo.foundation.exception.BusinessException;
import cn.axzo.foundation.result.ApiResult;
import cn.axzo.foundation.web.support.context.AxContext;
import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import lombok.Builder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.*;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@Slf4j
public class RpcClientImpl implements RpcClient {
@ -22,15 +25,36 @@ public class RpcClientImpl implements RpcClient {
@Getter
protected HttpClient httpClient;
protected RequestProxy requestProxy;
private static final Set<String> AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo", "ctxLogId", "traceId");
// XXX: http/2会把所有Header都转成小写, 历史定义的Header都是大写的，在http/2协议下会透传失败。
private static final TreeSet<String> CASE_INSENSITIVE_AXZO_HEADERS = AXZO_HEADERS.stream()
.collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER)));
// 将axzo-开头的header复制到请求的下一跳
private static final String AZXO_HEADER_PREFIX = "axzo-";
private static final List<Supplier<Map<String, String>>> DEFAULT_HEADER_SUPPLIERS = ImmutableList.of(
() -> AxContext.getRequest()
.map(request -> Collections.list(request.getHeaderNames()).stream()
// 通过http2协议的请求，默认会把大写转成小写，如"_SESSION_OBJECT" -> "_session_object"，导致session无法透传，下一跳需要通过session验签会失败
.filter(p -> CASE_INSENSITIVE_AXZO_HEADERS.contains(p) || p.startsWith(AZXO_HEADER_PREFIX))
.collect(Collectors.toMap(e -> e, request::getHeader, (oldValue, newValue) -> newValue)))
.orElse(Collections.emptyMap()),
//设置callerApp
() -> AxContext.getRequest().map(e -> {
Object caller = e.getAttribute(CallerAppInterceptor.NEXT_HTTP_REQUEST_ATTRIBUTE);
return ImmutableMap.of(CallerAppInterceptor.HTTP_REQUEST_HEADER, JSONObject.toJSONString(caller));
}).orElse(ImmutableMap.of())
);
@Builder
public RpcClientImpl(RequestProxy requestProxy, HttpClient.Config config, Supplier<Map<String, String>> requestHeaderSupplier) {
public RpcClientImpl(RequestProxy requestProxy,
Supplier<Map<String, String>> requestHeaderSupplier,
HttpClient httpClient) {
this.requestProxy = Optional.ofNullable(requestProxy).orElse(RequestProxy.SIMPLE_PROXY);
this.httpClient = OkHttpClientImpl.builder()
.config(config)
.build();
this.httpClient = Optional.ofNullable(httpClient).orElseGet(() -> OkHttpClientImpl.builder()
.config(HttpClient.Config.DEFAULT)
.build());
customHeaderSupplier = requestHeaderSupplier;
this.customHeaderSupplier = requestHeaderSupplier;
}
@Override
@ -40,7 +64,7 @@ public class RpcClientImpl implements RpcClient {
Optional<String> resp = requestBySupplier(requestParams, () -> this.getHttpClient().execute(httpMethod, url, requestParams));
ApiResult<T> result = converter.apply(resp.orElse(StringUtils.EMPTY));
if (!result.isSuccess()) {
throw new BusinessException(result.getCode(), result.getMsg());
throw new BusinessException(result.getCode() + "", result.getMsg());
}
return result.getData();
}
@ -56,17 +80,12 @@ public class RpcClientImpl implements RpcClient {
Objects.requireNonNull(requestParams);
//XXX 附加默认的header, 以及自定义的header
DEFAULT_HEADER_SUPPLIERS.stream()
.forEach(headerSupplier -> headerSupplier.get().entrySet()
.forEach(headerEntry ->
requestParams.addHeaderIfAbsent(headerEntry.getKey(), headerEntry.getValue())
));
DEFAULT_HEADER_SUPPLIERS
.forEach(headerSupplier -> headerSupplier.get().forEach(requestParams::addHeaderIfAbsent));
if (customHeaderSupplier != null) {
Map<String, String> map = customHeaderSupplier.get();
map.entrySet().forEach(e -> {
requestParams.addHeader(e.getKey(), e.getValue());
});
map.forEach(requestParams::addHeader);
}
return requestProxy.request(token -> {

89
web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientWrapper.java
View File

@ -1,10 +1,9 @@
package cn.axzo.foundation.web.support.rpc;
import cn.axzo.foundation.exception.BusinessException;
import cn.axzo.foundation.result.ApiResult;
import cn.axzo.foundation.web.support.rpc.exception.RpcNetworkException;
import cn.axzo.foundation.web.support.apps.App;
import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@ -12,13 +11,9 @@ import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
* 一个RpcClient的wrapper. 可以方便的在RpcClient和其他Client切换.
@ -26,88 +21,54 @@ import java.util.stream.Collectors;
*/
@Slf4j
public class RpcClientWrapper implements RpcClient {
private final Supplier<List<String>> hostsResolver;
private Supplier<String> hostResolver;
private final Supplier<App> appResolver;
private final RpcClient normalRpcClient;
@Getter
private volatile RpcClient activeRpcClient;
private AtomicInteger roundRobinIndex = new AtomicInteger(0);
private RpcClient normalRpcClient;
final Cache<String, String> excludeHostCache = CacheBuilder.
newBuilder()
.expireAfterWrite(1, TimeUnit.MINUTES).build();
private RpcClient activeRpcClient;
@lombok.Builder
public RpcClientWrapper(RpcClient normalRpcClient,
Supplier<String> hostResolver,
Supplier<List<String>> hostsResolver,
Supplier<App> appResolver,
ClientType defaultClientType) {
Preconditions.checkArgument(normalRpcClient != null, "normalRpcClient不能为空");
if (normalRpcClient != null) {
Preconditions.checkArgument(hostResolver != null || hostsResolver != null, "如果是normalRpcClient, hostResolver必须有值");
Preconditions.checkArgument(normalRpcClient instanceof RpcClientImpl, "normalRpcClient必须是RpcClientImpl.class实现");
}
Preconditions.checkArgument(appResolver != null, "如果是normalRpcClient, appResolver必须有值");
Preconditions.checkArgument(normalRpcClient instanceof RpcClientImpl, "normalRpcClient必须是RpcClientImpl.class实现");
this.normalRpcClient = normalRpcClient;
this.hostResolver = hostResolver;
this.hostsResolver = hostsResolver;
this.appResolver = appResolver;
activate(Optional.ofNullable(defaultClientType).orElse(ClientType.NORMAL));
}
@Override
public <T> T execute(HttpClient.HttpMethod httpMethod, String path, RequestParams requestParams, Function<String, ApiResult<T>> converter) {
String host = resolveHost();
try {
return activeRpcClient.execute(httpMethod, resolvePath(host, path), requestParams, converter);
} catch (RpcNetworkException e) {
excludeHostCache.put(host, "1");
log.warn("get network exception, add host {} to invalid pool and cooldown 1 minutes", host);
return activeRpcClient.execute(httpMethod, resolvePath(resolveHost(), path), requestParams, converter);
} catch (BusinessException e) {
int code = Integer.parseInt(e.getErrorCode());
Integer standardCode = ApiResult.getStandardCode(appResolver.get().getAppId(), code);
if (standardCode != code) {
throw new BusinessException(standardCode + "", e.getErrorMsg());
}
throw e;
}
}
@Override
public <R> R execute(HttpClient.HttpMethod httpMethod, String path, RequestParams requestParams, BiFunction<byte[], Map<String, List<String>>, R> responder) {
String host = resolveHost();
try {
return activeRpcClient.execute(httpMethod, resolvePath(host, path), requestParams, responder);
} catch (RpcNetworkException e) {
excludeHostCache.put(host, "1");
log.warn("get network exception, add host {} to invalid pool and cooldown 1 minutes", host);
return activeRpcClient.execute(httpMethod, resolvePath(resolveHost(), path), requestParams, responder);
} catch (BusinessException e) {
int code = Integer.parseInt(e.getErrorCode());
Integer standardCode = ApiResult.getStandardCode(appResolver.get().getAppId(), code);
if (standardCode != code) {
throw new BusinessException(standardCode + "", e.getErrorMsg());
}
throw e;
}
}
protected String resolveHost() {
if (hostResolver == null && hostsResolver == null) {
return StringUtils.EMPTY;
}
// 如果hostResolver不为空, 且path没有包含protocol与host. 则尝试将host与path拼接
if (hostResolver != null) {
return hostResolver.get();
}
List<String> hosts = hostsResolver.get();
// 如果只有 1 个 host，没必要做选择。
if (hosts.size() == 1) {
return hosts.get(0);
}
ConcurrentMap<String, String> excludeHosts = excludeHostCache.asMap();
List<String> availables = excludeHosts.size() == 0 ? hosts : hosts.stream().filter(i -> !excludeHosts.containsKey(i)).collect(Collectors.toList());
if (availables.size() == 0) {
return hosts.get(0);
} else if (availables.size() == 1) {
return availables.get(0);
}
// 使用 round robin 算法，选择可用节点。
int index = roundRobinIndex.getAndAccumulate(1, (x, y) -> {
if ((x + y) >= availables.size()) {
return 0;
}
return x + y;
});
return availables.get(index);
return appResolver.get().getHost();
}
protected String resolvePath(String host, String path) {
@ -126,7 +87,7 @@ public class RpcClientWrapper implements RpcClient {
public RpcClient activate(ClientType type) {
if (type == ClientType.NORMAL) {
Preconditions.checkState(normalRpcClient != null);
Preconditions.checkState(hostResolver != null || hostsResolver != null);
Preconditions.checkState(appResolver != null);
activeRpcClient = normalRpcClient;
} else {
throw new UnsupportedOperationException("unsupported clientType");

12
web-support-lib/src/test/java/cn/axzo/foundation/web/support/alert/AlertClientImplTest.java
View File

@ -12,20 +12,16 @@ class AlertClientImplTest {
.consumer(EmailAlertConsumer.builder()
.host("smtp.qiye.aliyun.com")
.port(465)
.username("zengxiaobo@axzo.cn")
.password("Zxb19861109")
.from("zengxiaobo@axzo.cn")
.username("notice@axzo.cn")
.password("HIKwTkBgSrHRAyBF")
.from("notice@axzo.cn")
.to("wangsiqian@axzo.cn")
.build())
.executor(new ScheduledThreadPoolExecutor(1))
.build();
for (int i = 0; i < 15; i++) {
alertClient.post(AlertClient.Alert.builder()
.ex(new RuntimeException())
.key("keykeykey")
.message("messagemessagemessagemessage")
.build());
alertClient.post("key", new RuntimeException(), "message");
}
}