From 223bb5cfeb361d7def0dab037bab0efbd34abc17 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Wed, 5 Jun 2024 16:28:41 +0800 Subject: [PATCH 01/41] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E4=BA=8B?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../event/support/consumer/DefaultRocketMQListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/DefaultRocketMQListener.java b/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/DefaultRocketMQListener.java index d1f2c13..04f1fba 100644 --- a/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/DefaultRocketMQListener.java +++ b/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/DefaultRocketMQListener.java @@ -14,7 +14,7 @@ import java.util.Optional; @Slf4j public class DefaultRocketMQListener implements RocketMQListener { - EventConsumer eventConsumer; + protected EventConsumer eventConsumer; @Builder public DefaultRocketMQListener(EventConsumer eventConsumer) { From d43fedf5ba2484e8f89d2838c8bc137106490c0b Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Wed, 5 Jun 2024 16:31:02 +0800 Subject: [PATCH 02/41] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E4=BA=8B?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../event/support/consumer/DefaultRocketMQListener.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/DefaultRocketMQListener.java b/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/DefaultRocketMQListener.java index 04f1fba..866fc2b 100644 --- a/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/DefaultRocketMQListener.java +++ b/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/DefaultRocketMQListener.java @@ -3,6 +3,7 @@ package cn.axzo.foundation.event.support.consumer; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import lombok.Builder; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; @@ -14,6 +15,7 @@ import java.util.Optional; @Slf4j public class DefaultRocketMQListener implements RocketMQListener { + @Getter protected EventConsumer eventConsumer; @Builder From a5a4d4c7bb9eafee30cea35ca8510a79cdf47de2 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Wed, 12 Jun 2024 20:08:32 +0800 Subject: [PATCH 03/41] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E7=BD=91?= =?UTF-8?q?=E5=85=B3=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../exception/ApiNotFoundException.java | 8 +++-- .../exception/ApiUnauthorizedException.java | 8 +++-- .../support/exception/GateResultCode.java | 31 +++++++++++++++++++ .../exception/InputFieldAbsentException.java | 8 +++-- .../exception/OutputFieldAbsentException.java | 8 +++-- 5 files changed, 51 insertions(+), 12 deletions(-) create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/GateResultCode.java diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiNotFoundException.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiNotFoundException.java index 70f4155..843676d 100755 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiNotFoundException.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiNotFoundException.java @@ -1,22 +1,24 @@ package cn.axzo.foundation.gateway.support.exception; +import cn.axzo.foundation.exception.BusinessException; + /** * API不存在导致代理失败的异常. *

* 实施时GateServer的ExceptionResolver可以拦截该异常做特殊处理, 如统一为HTTP标准状态码返回: ModelAndView.setStatus(HttpStatus.NOT_FOUND). *

*/ -public class ApiNotFoundException extends RuntimeException { +public class ApiNotFoundException extends BusinessException { private static final long serialVersionUID = 8821155936941903240L; @SuppressWarnings("unused") public ApiNotFoundException(String message) { - super(message); + super(GateResultCode.API_NOT_FOUND.getErrorCode(), message); } @SuppressWarnings("unused") public ApiNotFoundException(String message, Throwable cause) { - super(message, cause); + super(GateResultCode.API_NOT_FOUND.getErrorCode(), message, cause); } } diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiUnauthorizedException.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiUnauthorizedException.java index 58ae939..0e75c34 100755 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiUnauthorizedException.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/ApiUnauthorizedException.java @@ -1,21 +1,23 @@ package cn.axzo.foundation.gateway.support.exception; +import cn.axzo.foundation.exception.BusinessException; + /** * API未经授权访问的异常. *

* 实施时GateServer的ExceptionResolver可以拦截该异常做特殊处理, 如统一为HTTP标准状态码返回: ModelAndView.setStatus(HttpStatus.UNAUTHORIZED). *

*/ -public class ApiUnauthorizedException extends RuntimeException { +public class ApiUnauthorizedException extends BusinessException { private static final long serialVersionUID = -7679511760689237798L; public ApiUnauthorizedException(String message) { - super(message); + super(GateResultCode.API_UNAUTHORIZED.getErrorCode(), message); } @SuppressWarnings("unused") public ApiUnauthorizedException(String message, Throwable cause) { - super(message, cause); + super(GateResultCode.API_UNAUTHORIZED.getErrorCode(), message, cause); } } diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/GateResultCode.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/GateResultCode.java new file mode 100644 index 0000000..cae4177 --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/GateResultCode.java @@ -0,0 +1,31 @@ +package cn.axzo.foundation.gateway.support.exception; + +import cn.axzo.foundation.result.IResultCode; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor(access = AccessLevel.PRIVATE) +public enum GateResultCode implements IResultCode { + API_NOT_FOUND("050", "请求路径不存在", 500), + API_UNAUTHORIZED("051", "请求未授权", 403), + INPUT_FIELD_ABSENT("052", "参数错误", 400), + OUTPUT_FIELD_ABSENT("053", "返回结果错误", 500); + + final private String code; + + final private String message; + + final private Integer httpCode; + + @Override + public String getErrorCode() { + return code; + } + + @Override + public String getErrorMessage() { + return message; + } +} diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/InputFieldAbsentException.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/InputFieldAbsentException.java index 95a4b02..dcf2afa 100755 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/InputFieldAbsentException.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/InputFieldAbsentException.java @@ -1,17 +1,19 @@ package cn.axzo.foundation.gateway.support.exception; +import cn.axzo.foundation.exception.BusinessException; + /** * ParameterFilter处理输入参数时输入参数中指定的字段不存在 */ -public class InputFieldAbsentException extends RuntimeException { +public class InputFieldAbsentException extends BusinessException { private static final long serialVersionUID = -5646417429309513569L; public InputFieldAbsentException(String message) { - super(message); + super(GateResultCode.INPUT_FIELD_ABSENT.getErrorCode(), message); } public InputFieldAbsentException(String message, Throwable cause) { - super(message, cause); + super(GateResultCode.INPUT_FIELD_ABSENT.getErrorCode(), message, cause); } } diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/OutputFieldAbsentException.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/OutputFieldAbsentException.java index 81b0e75..f3fa2f1 100755 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/OutputFieldAbsentException.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/exception/OutputFieldAbsentException.java @@ -1,17 +1,19 @@ package cn.axzo.foundation.gateway.support.exception; +import cn.axzo.foundation.exception.BusinessException; + /** * ParameterFilter处理返回对象时返回对象中指定的字段不存在 */ -public class OutputFieldAbsentException extends RuntimeException { +public class OutputFieldAbsentException extends BusinessException { private static final long serialVersionUID = 8830968633694688099L; public OutputFieldAbsentException(String message) { - super(message); + super(GateResultCode.OUTPUT_FIELD_ABSENT.getErrorCode(), message); } public OutputFieldAbsentException(String message, Throwable cause) { - super(message, cause); + super(GateResultCode.OUTPUT_FIELD_ABSENT.getErrorCode(), message, cause); } } From c67c7192e30867393dd682551eed743a29bdb46b Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 13 Jun 2024 19:12:24 +0800 Subject: [PATCH 04/41] =?UTF-8?q?feat:=20=E8=BF=94=E5=9B=9Ecode=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E4=B8=BAinteger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/result/ApiResult.java | 18 ++++++++++++++---- .../support/config/ApiResultJsonConverter.java | 2 +- .../web/support/config/ApiResultWrapper.java | 8 ++++---- .../web/support/rpc/RpcClientImpl.java | 2 +- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java index 713d677..002e063 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java +++ b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java @@ -1,10 +1,14 @@ package cn.axzo.foundation.result; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Strings; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.Optional; @Data @Builder @@ -18,7 +22,7 @@ public class ApiResult { protected Integer httpCode; - protected String code; + protected transient String errCode; protected String msg; @@ -31,11 +35,17 @@ public class ApiResult { public static ApiResult success(T data) { return ApiResult.builder() .httpCode(SUCCESS_HTTP_CODE) - .code(SUCCESS_CODE) + .errCode(SUCCESS_CODE) .data(data) .build(); } + //为了兼容前端需要的int类型, 这里返回一个单独的code + public Integer getCode() { + return Optional.ofNullable(Strings.emptyToNull(StringUtils.getDigits(errCode))) + .map(Integer::parseInt).orElse(SUCCESS_HTTP_CODE); + } + public static ApiResult error(IResultCode resultCode) { return error(resultCode.getHttpCode(), resultCode.getErrorCode(), resultCode.getErrorMessage()); } @@ -59,13 +69,13 @@ public class ApiResult { public static ApiResult error(Integer httpCode, String errorCode, String errorMsg) { return ApiResult.builder() .httpCode(httpCode) - .code(errorCode) + .errCode(errorCode) .msg(errorMsg) .build(); } @JsonIgnore public boolean isSuccess() { - return SUCCESS_CODE.equals(getCode()); + return SUCCESS_CODE.equals(getErrCode()); } } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultJsonConverter.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultJsonConverter.java index 7f08564..a6aa149 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultJsonConverter.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultJsonConverter.java @@ -121,7 +121,7 @@ public class ApiResultJsonConverter extends FastJsonHttpMessageConverter { //只有抛出异常或者返回失败时才会设置header outputMessage.getHeaders().add(HTTP_CODE_HEADER_KEY, result.getHttpCode() + ""); - outputMessage.getHeaders().add(ERROR_CODE_HEADER_KEY, result.getCode()); + outputMessage.getHeaders().add(ERROR_CODE_HEADER_KEY, result.getErrCode()); //如果header中没有pretty. 按照定义好的fastJson的输出方式. 否则自定义美化输出 writeJson(result, outputMessage); diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java index e59ebb5..a23526f 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java @@ -18,7 +18,7 @@ class ApiResultWrapper extends ApiResult { @Builder(builderMethodName = "wrapperBuilder") public ApiResultWrapper(ApiResult result, AppRuntime appRuntime, ApiResultJsonConverter.ErrorMsgResolver msgResolver) { - this.code = result.getCode(); + this.errCode = result.getErrCode(); this.msg = result.getMsg(); this.httpCode = Optional.ofNullable(result.getHttpCode()).orElse(ResultCode.DEFAULT_HTTP_ERROR_CODE); @@ -26,16 +26,16 @@ class ApiResultWrapper extends ApiResult { //如果是系统的或者一个完整的errorCode. 直接处理 //目前已在系统中使用的ErrorCode格式都在6个字符以上；业务自定义code在6个字符以内(如果超过6个字符会直接返回)； // 如:SYS_100001,SSO_100001,op-leads_5001001,10011002,1001X101； - if (code.contains("_") || code.length() > 6) { + if (errCode.contains("_") || errCode.length() > 6) { return; } //没有appId时沿用之前的拼装逻辑"${appName}_${httpCode}${ErrorCode}" //存在appId时拼装逻辑调整为"${appId}${ErrorCode}" if (StringUtils.isEmpty(appRuntime.getAppId())) { - this.code = Strings.nullToEmpty(appRuntime.getAppName()).toUpperCase() + "_" + getHttpCode() + result.getCode(); + this.errCode = Strings.nullToEmpty(appRuntime.getAppName()).toUpperCase() + "_" + getHttpCode() + result.getErrCode(); } else { - this.code = appRuntime.getAppId() + result.getCode(); + this.errCode = appRuntime.getAppId() + result.getErrCode(); } } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java index ae07ec3..40133a4 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java @@ -40,7 +40,7 @@ public class RpcClientImpl implements RpcClient { Optional resp = requestBySupplier(requestParams, () -> this.getHttpClient().execute(httpMethod, url, requestParams)); ApiResult result = converter.apply(resp.orElse(StringUtils.EMPTY)); if (!result.isSuccess()) { - throw new BusinessException(result.getCode(), result.getMsg()); + throw new BusinessException(result.getErrCode(), result.getMsg()); } return result.getData(); } From 53e1e9d9ffc183a93029f3ea94b49ca6cc7bfcb4 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 13 Jun 2024 19:26:47 +0800 Subject: [PATCH 05/41] =?UTF-8?q?feat:=20=E8=BF=94=E5=9B=9Ecode=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E4=B8=BAinteger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/cn/axzo/foundation/result/ApiResult.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java index 002e063..2db16e7 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java +++ b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java @@ -22,7 +22,7 @@ public class ApiResult { protected Integer httpCode; - protected transient String errCode; + protected String errCode; protected String msg; From 8cbc76eb0478489949a136701742bea247c64557 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 13 Jun 2024 20:02:42 +0800 Subject: [PATCH 06/41] =?UTF-8?q?feat:=20=E8=BF=94=E5=9B=9Ecode=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E4=B8=BAinteger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/result/ApiResult.java | 20 +++++-------------- .../config/ApiResultJsonConverter.java | 2 +- .../web/support/config/ApiResultWrapper.java | 17 +++------------- .../web/support/rpc/RpcClientImpl.java | 8 ++------ 4 files changed, 11 insertions(+), 36 deletions(-) diff --git a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java index 2db16e7..aca42fc 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java +++ b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java @@ -1,14 +1,10 @@ package cn.axzo.foundation.result; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.base.Strings; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; - -import java.util.Optional; @Data @Builder @@ -16,13 +12,13 @@ import java.util.Optional; @AllArgsConstructor public class ApiResult { public static final Integer SUCCESS_HTTP_CODE = 200; - private static final String SUCCESS_CODE = "200"; + private static final Integer SUCCESS_CODE = 200; public static final Integer FAILED_HTTP_CODE = 400; private static final String FAILED_CODE = "400"; protected Integer httpCode; - protected String errCode; + protected Integer code; protected String msg; @@ -35,17 +31,11 @@ public class ApiResult { public static ApiResult success(T data) { return ApiResult.builder() .httpCode(SUCCESS_HTTP_CODE) - .errCode(SUCCESS_CODE) + .code(SUCCESS_CODE) .data(data) .build(); } - //为了兼容前端需要的int类型, 这里返回一个单独的code - public Integer getCode() { - return Optional.ofNullable(Strings.emptyToNull(StringUtils.getDigits(errCode))) - .map(Integer::parseInt).orElse(SUCCESS_HTTP_CODE); - } - public static ApiResult error(IResultCode resultCode) { return error(resultCode.getHttpCode(), resultCode.getErrorCode(), resultCode.getErrorMessage()); } @@ -69,13 +59,13 @@ public class ApiResult { public static ApiResult error(Integer httpCode, String errorCode, String errorMsg) { return ApiResult.builder() .httpCode(httpCode) - .errCode(errorCode) + .code(Integer.parseInt(errorCode)) .msg(errorMsg) .build(); } @JsonIgnore public boolean isSuccess() { - return SUCCESS_CODE.equals(getErrCode()); + return SUCCESS_CODE.equals(getCode()); } } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultJsonConverter.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultJsonConverter.java index a6aa149..5aa1af1 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultJsonConverter.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultJsonConverter.java @@ -12,6 +12,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import lombok.Builder; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpOutputMessage; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -121,7 +122,6 @@ public class ApiResultJsonConverter extends FastJsonHttpMessageConverter { //只有抛出异常或者返回失败时才会设置header outputMessage.getHeaders().add(HTTP_CODE_HEADER_KEY, result.getHttpCode() + ""); - outputMessage.getHeaders().add(ERROR_CODE_HEADER_KEY, result.getErrCode()); //如果header中没有pretty. 按照定义好的fastJson的输出方式. 否则自定义美化输出 writeJson(result, outputMessage); diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java index a23526f..d61f8e3 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java @@ -3,7 +3,6 @@ package cn.axzo.foundation.web.support.config; import cn.axzo.foundation.result.ApiResult; import cn.axzo.foundation.result.ResultCode; import cn.axzo.foundation.web.support.AppRuntime; -import com.google.common.base.Strings; import lombok.Builder; import org.apache.commons.lang3.StringUtils; @@ -18,25 +17,15 @@ class ApiResultWrapper extends ApiResult { @Builder(builderMethodName = "wrapperBuilder") public ApiResultWrapper(ApiResult result, AppRuntime appRuntime, ApiResultJsonConverter.ErrorMsgResolver msgResolver) { - this.errCode = result.getErrCode(); + this.code = result.getCode(); this.msg = result.getMsg(); this.httpCode = Optional.ofNullable(result.getHttpCode()).orElse(ResultCode.DEFAULT_HTTP_ERROR_CODE); - //如果是系统的或者一个完整的errorCode. 直接处理 - //目前已在系统中使用的ErrorCode格式都在6个字符以上；业务自定义code在6个字符以内(如果超过6个字符会直接返回)； - // 如:SYS_100001,SSO_100001,op-leads_5001001,10011002,1001X101； - if (errCode.contains("_") || errCode.length() > 6) { - return; - } - //没有appId时沿用之前的拼装逻辑"${appName}_${httpCode}${ErrorCode}" //存在appId时拼装逻辑调整为"${appId}${ErrorCode}" - if (StringUtils.isEmpty(appRuntime.getAppId())) { - this.errCode = Strings.nullToEmpty(appRuntime.getAppName()).toUpperCase() + "_" + getHttpCode() + result.getErrCode(); - } else { - this.errCode = appRuntime.getAppId() + result.getErrCode(); + if (!StringUtils.isEmpty(appRuntime.getAppId())) { + this.code = Integer.parseInt(StringUtils.getDigits(appRuntime.getAppId() + StringUtils.left(result.getCode() + "", 3))); } - } } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java index 40133a4..2c0a738 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java @@ -7,11 +7,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; +import java.util.*; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -40,7 +36,7 @@ public class RpcClientImpl implements RpcClient { Optional resp = requestBySupplier(requestParams, () -> this.getHttpClient().execute(httpMethod, url, requestParams)); ApiResult result = converter.apply(resp.orElse(StringUtils.EMPTY)); if (!result.isSuccess()) { - throw new BusinessException(result.getErrCode(), result.getMsg()); + throw new BusinessException(result.getCode() + "", result.getMsg()); } return result.getData(); } From 22b96eb6989ff33182118fc65faa5a8ce824b969 Mon Sep 17 00:00:00 2001 From: lilong Date: Tue, 18 Jun 2024 16:25:23 +0800 Subject: [PATCH 07/41] =?UTF-8?q?feat:(REQ-2545)=20pageResp=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0hasNext=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/axzo/foundation/page/PageResp.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java b/common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java index 00cf528..825d5da 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java +++ b/common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java @@ -25,4 +25,22 @@ public class PageResp { public List getData() { return Optional.ofNullable(data).orElse(ImmutableList.of()); } + + public boolean hasNext() { + return this.current < this.getPages(); + } + + public long getPages() { + if (this.getSize() == 0L) { + return 0L; + } + + + long pages = this.getTotal() / this.getSize(); + if (this.getTotal() % this.getSize() != 0L) { + ++pages; + } + return pages; + } + } From bf66aedda54f7273087043f570d2f6b3dc6fd5cf Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Mon, 17 Jun 2024 20:45:44 +0800 Subject: [PATCH 08/41] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E7=9A=84code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/web/support/config/ApiResultWrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java index d61f8e3..07f2317 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java @@ -25,7 +25,7 @@ class ApiResultWrapper extends ApiResult { //没有appId时沿用之前的拼装逻辑"${appName}_${httpCode}${ErrorCode}" //存在appId时拼装逻辑调整为"${appId}${ErrorCode}" if (!StringUtils.isEmpty(appRuntime.getAppId())) { - this.code = Integer.parseInt(StringUtils.getDigits(appRuntime.getAppId() + StringUtils.left(result.getCode() + "", 3))); + this.code = Integer.parseInt(StringUtils.getDigits(appRuntime.getAppId() + StringUtils.leftPad(result.getCode() + "", 3, "0"))); } } } From 4df3579a34e75e5dca4ded3c169f5c1de12e5292 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 20 Jun 2024 10:23:09 +0800 Subject: [PATCH 09/41] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0redis=E7=9A=84?= =?UTF-8?q?=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/result/ResultCode.java | 4 +- pom.xml | 1 + redis-support-lib/pom.xml | 26 ++ .../redis/support/EventBroadcast.java | 75 +++++ .../redis/support/LocalCacheCoordinate.java | 49 ++++ .../foundation/redis/support/RedisLock.java | 270 ++++++++++++++++++ .../support/impl/RedisEventBroadcastImpl.java | 189 ++++++++++++ .../impl/RedisLocalCacheCoordinate.java | 178 ++++++++++++ 8 files changed, 791 insertions(+), 1 deletion(-) create mode 100644 redis-support-lib/pom.xml create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/EventBroadcast.java create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/LocalCacheCoordinate.java create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RedisLock.java create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisEventBroadcastImpl.java create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisLocalCacheCoordinate.java diff --git a/common-lib/src/main/java/cn/axzo/foundation/result/ResultCode.java b/common-lib/src/main/java/cn/axzo/foundation/result/ResultCode.java index 5f41de3..0267fbc 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/result/ResultCode.java +++ b/common-lib/src/main/java/cn/axzo/foundation/result/ResultCode.java @@ -11,7 +11,9 @@ public enum ResultCode implements IResultCode { INVALID_PARAMS("002", "请求参数格式错误", 400), NETWORK_FAILURE("003", "内部网络错误", 500), PROCESS_TIMEOUT("004", "内部处理超时", 500), - APP_CONFIG_ERROR("005", "服务配置错误", 500); + APP_CONFIG_ERROR("005", "服务配置错误", 500), + OPERATE_TOO_FREQUENTLY("006", "操作过于频繁", 500), + ; final private String code; diff --git a/pom.xml b/pom.xml index afc7af4..9df7ec1 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,7 @@ web-support-lib gateway-support-lib event-support-lib + redis-support-lib diff --git a/redis-support-lib/pom.xml b/redis-support-lib/pom.xml new file mode 100644 index 0000000..0a63193 --- /dev/null +++ b/redis-support-lib/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + cn.axzo.foundation + axzo-lib-box + 2.0.0-SNAPSHOT + + + redis-support-lib + + + cn.axzo.foundation + web-support-lib + 2.0.0-SNAPSHOT + + + + org.springframework.boot + spring-boot-starter-data-redis + + + + \ No newline at end of file diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/EventBroadcast.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/EventBroadcast.java new file mode 100644 index 0000000..b706636 --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/EventBroadcast.java @@ -0,0 +1,75 @@ +package cn.axzo.foundation.redis.support; + +import com.alibaba.fastjson.JSONObject; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.function.BiConsumer; + +/** + * 提供简单的广播Client. 通过build获得的BroadcastQueue发送广播. 所有节点都会收到广播并回调consumer + *
    + *
  • 通过build获得BroadcastQueue, 调用queue的broadcast方法. 发送广播
    • + *
  • 收到广播会会主动回调注册时的BiConsumer
    • + *
  • 目前支持通过redis的pub/sub实现, 因此需要依赖redisTemplate
    • + *
  • lettuce来处理回调, 不建议在回调用做比较重的业务
    • + *
+ */ +public interface EventBroadcast { + + + /** + * 通过queueName, 回调consumer来构建广播队列 + * + * @param queueName + * @param consumer + * @return + */ + BroadcastQueue build(String queueName, BiConsumer consumer); + + /** + * 广播队列. 向队列中广播消息. 所有的节点(包含自身)都会收到广播消息 + */ + interface BroadcastQueue { + /** + * 获得队列名称, 与注册时的queueName相同 + * + * @return + */ + String getName(); + + /** + * 广播, data为需要广播的内容 + * + * @param data + * @return + */ + boolean broadcast(JSONObject data); + } + + /** + * 广播事件 + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + class BroadcastEvent { + String name; + JSONObject data; + /** + * 发起广播的节点信息 + */ + JSONObject senderRuntime; + + public String toJSONString() { + return new JSONObject() + .fluentPut("name", name) + .fluentPut("data", data) + .fluentPut("senderRuntime", senderRuntime) + .toJSONString(); + } + } +} diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/LocalCacheCoordinate.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/LocalCacheCoordinate.java new file mode 100644 index 0000000..a9659d9 --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/LocalCacheCoordinate.java @@ -0,0 +1,49 @@ +package cn.axzo.foundation.redis.support; + +import com.alibaba.fastjson.JSONObject; +import lombok.*; + +import java.util.function.Consumer; +import java.util.function.Predicate; + +/** + * 简单的本地缓存协调客户端. 主要的目的是通过事件在不同节点间处理缓存dirty的场景. + * 提供了2中机制来处理缓存dirty的场景 + * registerCacheDirtyListener 精确处理key匹配 + * registerCacheDirtyInterceptor 统一的事件匹配. 可以通过检查 CacheDirtiedEvent.data中的数据是否包含特定的json-path来处理缓存. + */ +public interface LocalCacheCoordinate { + + + void notifyCacheDirty(CacheDirtiedEvent event); + + /** + * @param key {@link CacheDirtiedEvent#key} 中定义数据 + * @param consumer + */ + void registerCacheDirtyListener(String key, Consumer consumer); + + void registerCacheDirtyInterceptor(CacheDirtiedEventInterceptor interceptor); + + @NoArgsConstructor + @AllArgsConstructor + @Data + @Builder + final class CacheDirtiedEvent { + @NonNull + String key; + @NonNull + JSONObject data; + } + + @NoArgsConstructor + @AllArgsConstructor + @Data + @Builder + final class CacheDirtiedEventInterceptor { + @NonNull + Predicate filter; + @NonNull + Consumer consumer; + } +} diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RedisLock.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RedisLock.java new file mode 100644 index 0000000..38dd164 --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RedisLock.java @@ -0,0 +1,270 @@ +package cn.axzo.foundation.redis.support; + +import cn.axzo.foundation.result.ResultCode; +import lombok.extern.slf4j.Slf4j; + +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +/** + * 使用redis实现的分布式锁. 为了不依赖redis, 这里需要调用者实现{@link RedisWrapper}. 如下: + * 
{@code
+ *     @Bean
+ *     RedisLock.RedisWrapper redisWrapper(RedisTemplate redisTemplate) {
+ *         return new RedisLock.RedisWrapper() {
+ *             @Override
+ *             public void delete(String key) {
+ *                 redisTemplate.delete(key);
+ *             }
+ *
+ *             @Override
+ *             public boolean lock(String key, String value, long expireMills) {
+ *                 return redisTemplate.opsForValue().setIfAbsent(key, value, expireMills, TimeUnit.MILLISECONDS);
+ *             }
+ *         };
+ *     }
+ * }
+ *

+ * 然后使用{@link #tryAcquireRun(long, long, Supplier)} 来获取lock执行逻辑，明确指定 "等待锁时间" 和 "锁的超时时间". + */ +@Slf4j +public class RedisLock { + + public static final String DEFAULT_LOCK_SUFFIX = ":lock"; + private RedisWrapper redis; + + /** + * 默认超时时间（毫秒） + */ + private static final long DEFAULT_TIME_OUT_MILLIS = 5 * 1000; + private static final Random RANDOM = new Random(); + /** + * 锁的超时时间（豪秒），过期删除 + */ + public static final int EXPIRE_IN_MILLIS = 1 * 60 * 1000; + + private String key; + // 锁状态标志 + private boolean locked = false; + + private RuntimeException lockFailedException; + + public interface RedisWrapper { + /** + * 删除key + * + * @param key + */ + void delete(String key); + + /** + * 锁定key. 通常使用SetIfAbsent(); + * + * @param key + * @param value + * @param expireMills + * @return + */ + boolean lock(String key, String value, long expireMills); + } + + /** + * 关闭锁，该方法不建议外部直接使用，
+ * 对于加锁执行的操作，建议直接使用 {@link RedisLock#tryAcquireRun(long, long, Supplier)}，会自动执行close操作。 + */ + private void close() { + if (this.locked) { + this.redis.delete(this.key); + } + } + + /** + * This creates a RedisLock + * + * @param key key + * @param redis 数据源 + */ + public RedisLock(String key, RedisWrapper redis) { + this(key, redis, DEFAULT_LOCK_SUFFIX, null); + } + + /** + * This creates a RedisLock + * + * @param key key + * @param redis 数据源 + */ + public RedisLock(String key, RedisWrapper redis, RuntimeException lockFailedException) { + this(key, redis, DEFAULT_LOCK_SUFFIX, lockFailedException); + } + + /** + * This creates a RedisLock + * + * @param key key + * @param redis 数据源 + */ + public RedisLock(String key, RedisWrapper redis, String suffix, RuntimeException lockFailedException) { + this.key = key + Optional.ofNullable(suffix).orElse(DEFAULT_LOCK_SUFFIX); + this.redis = redis; + this.lockFailedException = Optional.ofNullable(lockFailedException) + .orElseGet(() -> new AcquireLockFailException("获取锁失败 " + key)); + } + + /** + * 尝试在timeoutMillis毫秒内获取锁并设置锁的过期时间为expireMillis毫秒，若获取锁成功，则执行supplier的逻辑，并返回supplier执行结果。然后关闭锁
+ * 

+     * 锁的释放，由2方面保证：
+     * 1、supplier方法执行完成后，会主动释放锁。
+     * 2、设置锁的过期时间
+     *
+ * 如果只是单纯的尝试获取锁并执行，无需等待锁，可以将timeoutMillis参数设置为0。 + * + * @param timeoutMillis 等待获取锁的时间 单位毫秒（会在等待时间内不停自旋尝试获取锁。）如果超过该时间还没成功获取到锁，则抛出获取锁失败的BizException + * timeoutMillis=0，则表示只进行一次获取锁的尝试。获取失败，直接抛获取锁失败的异常 + * @param expireMillis 锁的过期时间，保证锁最长的持有时间。（如果主动释放锁失败，会有该参数保证锁成功释放） + * @param supplier 需要执行的方法 + * @param 返回参数类型 + * @return + */ + public T tryAcquireRun(final long timeoutMillis, final long expireMillis, Supplier supplier) { + if (!lock(timeoutMillis, expireMillis)) { + throw lockFailedException; + } + try { + return supplier.get(); + } finally { + close(); + } + } + + /** + * 尝试获取锁，并执行supplier.get()方法，返回结果。
+ * 该方法使用了默认的锁等待时间和过期时间：
+ * 等待锁时间={@link #DEFAULT_TIME_OUT_MILLIS 5秒}
+ * 锁过期时间={@link #EXPIRE_IN_MILLIS 1分钟}
+ * 调用该方法，效果等同于 {@link #tryAcquireRun(long, long, Supplier)} + * -> tryAcquireRun(DEFAULT_TIME_OUT_MILLIS, EXPIRE_IN_MILLIS, supplier); + * + * @param supplier + * @param + * @return + */ + public T tryAcquireRun(Supplier supplier) { + if (!lock()) { + throw lockFailedException; + } + try { + return supplier.get(); + } finally { + close(); + } + } + + /** + * 尝试获取锁，并执行supplier.get()方法，返回结果。
+ * 该方法使用了默认的锁过期时间：
+ * 锁过期时间={@link #EXPIRE_IN_MILLIS 1分钟}
+ * 调用该方法，效果等同于 {@link #tryAcquireRun(long, long, Supplier)} + * -> tryAcquireRun(timeoutMillis, EXPIRE_IN_MILLIS, supplier); + * + * @param supplier + * @param + * @return + */ + public T tryAcquireRun(long timeoutMillis, Supplier supplier) { + if (!lock(timeoutMillis)) { + throw lockFailedException; + } + try { + return supplier.get(); + } finally { + close(); + } + } + + /** + * 尝试立即获取锁，并执行supplier.get()方法，返回结果。
+ * timeoutMills = 0, expireMillis = 5分钟 + */ + public T acquireImmediatelyRun(Supplier supplier) { + if (!lock(0, TimeUnit.MINUTES.toMillis(5))) { + throw ResultCode.OPERATE_TOO_FREQUENTLY.toException(); + } + try { + return supplier.get(); + } finally { + close(); + } + } + + /** + * 加锁 应该以： lock(); try { doSomething(); } finally { close()； } 的方式调用
+ * 外部不建议直接使用该方法，建议使用{@link #tryAcquireRun(long, long, Supplier)}明确指定锁的等待和过期时间 + * + * @param timeoutMillis 超时时间(毫秒) + * @return 成功或失败标志 + */ + private boolean lock(long timeoutMillis) { + return lock(timeoutMillis, EXPIRE_IN_MILLIS); + } + + /** + * 加锁 应该以： lock(); try { doSomething(); } finally { close()； } 的方式调用
+ * 外部不建议直接使用该方法，建议使用{@link #tryAcquireRun(long, long, Supplier)}明确指定锁的等待和过期时间 + * + * @param timeoutMillis 超时时间(毫秒 + * @param expireMillis 锁的超时时间（毫秒），过期删除 + * @return 成功或失败标志 + */ + private boolean lock(final long timeoutMillis, final long expireMillis) { + long nano = System.nanoTime(); + long timeoutNano = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + try { + do { + boolean ok = redis.lock(key, "true", expireMillis); + if (ok) { + this.locked = true; + return this.locked; + } + // 短暂休眠，避免出现活锁 + Thread.sleep(3, RANDOM.nextInt(500)); + } while ((System.nanoTime() - nano) < timeoutNano); + } catch (Exception e) { + throw lockFailedException; + } + return false; + } + + /** + * 加锁 应该以： lock(); try { doSomething(); } finally { close()； } 的方式调用
+ * 外部不建议直接使用该方法，建议使用{@link #tryAcquireRun(long, long, Supplier)}明确指定锁的等待和过期时间 + * + * @return 成功或失败标志 + */ + private boolean lock() { + return lock(DEFAULT_TIME_OUT_MILLIS); + } + + /** 当获取锁失败的时候抛出该异常，方便调用方捕获处理 */ + public static class AcquireLockFailException extends RuntimeException { + public AcquireLockFailException() { + super(); + } + + public AcquireLockFailException(String msg) { + super(msg); + } + + public AcquireLockFailException(Throwable throwable) { + super(throwable); + } + + public AcquireLockFailException(String msg, Throwable throwable) { + super(msg, throwable); + } + } + +} diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisEventBroadcastImpl.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisEventBroadcastImpl.java new file mode 100644 index 0000000..6c9470b --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisEventBroadcastImpl.java @@ -0,0 +1,189 @@ +package cn.axzo.foundation.redis.support.impl; + +import cn.axzo.foundation.redis.support.EventBroadcast; +import cn.axzo.foundation.web.support.AppRuntime; +import cn.axzo.foundation.web.support.utils.KeyBuilder; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.Map; +import java.util.Objects; +import java.util.function.BiConsumer; + +@Slf4j +public class RedisEventBroadcastImpl implements EventBroadcast, InitializingBean, DisposableBean { + + private RedisTemplate redisTemplate; + + private AppRuntime appRuntime; + + private String channel; + + /** + * 广播监听. 监听channel上的广播. 并根据event dispatch到不同的consumer + */ + @Setter(AccessLevel.PROTECTED) + private BroadcastListener broadcastListener; + + @Builder + public RedisEventBroadcastImpl(RedisTemplate redisTemplate, AppRuntime appRuntime) { + Objects.requireNonNull(redisTemplate); + Objects.requireNonNull(appRuntime); + this.redisTemplate = redisTemplate; + this.appRuntime = appRuntime; + } + + @Override + public BroadcastQueue build(String queueName, BiConsumer consumer) { + Objects.requireNonNull(queueName); + Objects.requireNonNull(consumer); + Preconditions.checkArgument(queueName.length() <= 32); + + RedisBroadcastQueue queue = RedisBroadcastQueue.builder() + .name(queueName) + .channel(channel) + .redisTemplate(redisTemplate) + .appRuntime(appRuntime) + .build(); + broadcastListener.register(queueName, consumer); + + return queue; + } + + /** + * 构建RedisBroadcastListener, 并启动任务 + * + * @throws Exception + */ + @Override + public void afterPropertiesSet() throws Exception { + this.channel = KeyBuilder.build(appRuntime, "broadcast", "channel"); + + broadcastListener = RedisBroadcastListener.builder() + .redisTemplate(redisTemplate) + .channel(channel) + .build(); + broadcastListener.start(); + } + + @Override + public void destroy() throws Exception { + broadcastListener.stop(); + } + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + protected static final class RedisBroadcastQueue implements BroadcastQueue { + String name; + + @Getter(AccessLevel.NONE) + RedisTemplate redisTemplate; + + @Getter(AccessLevel.NONE) + String channel; + + @Getter(AccessLevel.NONE) + AppRuntime appRuntime; + + @Override + public boolean broadcast(JSONObject data) { + return (Boolean) redisTemplate.execute((RedisCallback) connection -> { + connection.publish(channel.getBytes(), + BroadcastEvent.builder() + .name(name) + .data(data) + .senderRuntime(appRuntime.toJson()) + .build() + .toJSONString().getBytes(Charsets.UTF_8)); + return true; + }); + } + } + + protected static final class RedisBroadcastListener implements BroadcastListener { + private RedisTemplate redisTemplate; + private String channel; + + private final Map> broadcastConsumer = Maps.newConcurrentMap(); + + @lombok.Builder + public RedisBroadcastListener(RedisTemplate redisTemplate, String channel) { + this.redisTemplate = redisTemplate; + this.channel = channel; + } + + @Override + public boolean start() { + try { + redisTemplate.getConnectionFactory().getConnection() + .subscribe((message, pattern) -> { + BroadcastEvent broadcastEvent = JSONObject.parseObject(new String(message.getBody(), Charsets.UTF_8)) + .toJavaObject(BroadcastEvent.class); + onEvent(broadcastEvent); + }, channel.getBytes()); + } catch (Exception e) { + log.error("====== start broadcast listener error =====", e); + return false; + } + log.info("====== start broadcast listener ====="); + return true; + } + + @Override + public boolean register(String queueName, BiConsumer consumer) { + Preconditions.checkArgument(!broadcastConsumer.containsKey(queueName), "duplicate broadcast queue"); + broadcastConsumer.put(queueName, consumer); + return true; + } + + @Override + public boolean stop() { + //doNothing, 如果定义了线程池可以在这里销毁 + log.info("====== stop broadcast listener ====="); + return true; + } + + private void onEvent(BroadcastEvent event) { + BiConsumer consumer = broadcastConsumer.get(event.getName()); + if (consumer == null) { + log.error("event is ready, but no consumer found, event = {}", event.toJSONString()); + return; + } + try { + consumer.accept(event.getName(), event); + } catch (Exception ex) { + log.error("consume broadcast error, event = {}", event.toJSONString()); + //ignore 忽略业务异常 + } + } + } + + /** + * 广播监听. 提供开始监听, 结束监听, 注册queue以及对应的BiConsumer + */ + interface BroadcastListener { + boolean start(); + + /** + * 注册queue对应的consumer + * + * @param queueName + * @param consumer + * @return + */ + boolean register(String queueName, BiConsumer consumer); + + boolean stop(); + } +} + diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisLocalCacheCoordinate.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisLocalCacheCoordinate.java new file mode 100644 index 0000000..4e3a41b --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisLocalCacheCoordinate.java @@ -0,0 +1,178 @@ +package cn.axzo.foundation.redis.support.impl; + +import cn.axzo.foundation.enums.AppEnvEnum; +import cn.axzo.foundation.redis.support.EventBroadcast; +import cn.axzo.foundation.redis.support.LocalCacheCoordinate; +import cn.axzo.foundation.web.support.AppRuntime; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import lombok.AccessLevel; +import lombok.Data; +import lombok.Setter; +import lombok.Singular; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.transaction.support.TransactionSynchronizationAdapter; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Supplier; + +@Slf4j +public class RedisLocalCacheCoordinate implements LocalCacheCoordinate, InitializingBean { + @Setter(AccessLevel.PROTECTED) + private AfterCommitExecutor executor; + + @Setter + private EventBroadcast EventBroadcast; + private EventBroadcast.BroadcastQueue broadcastQueue; + + @Singular + private Multimap> cacheDirtyHandlers = ArrayListMultimap.create(); + private List interceptors = Lists.newArrayList(); + + @Override + public void notifyCacheDirty(CacheDirtiedEvent event) { + executor.execute(() -> { + broadcastQueue.broadcast(JSON.parseObject(JSONObject.toJSONString(event))); + log.info("notify cache dirty, event={}", event); + }); + } + + @Override + public void registerCacheDirtyListener(String key, Consumer consumer) { + cacheDirtyHandlers.put(key, consumer); + } + + @Override + public void registerCacheDirtyInterceptor(CacheDirtiedEventInterceptor interceptor) { + interceptors.add(interceptor); + } + + void onCacheDirtied(String queueName, EventBroadcast.BroadcastEvent e) { + CacheDirtiedEvent cacheDirtiedEvent = CacheDirtiedEvent.builder().key(e.getData().getString("key")) + .data(e.getData().getJSONObject("data")) + .build(); + interceptors.forEach(h -> { + try { + if (h.getFilter().test(cacheDirtiedEvent)) { + h.getConsumer().accept(cacheDirtiedEvent); + log.info("interceptor handled cache dirty event={}, interceptor={}", cacheDirtiedEvent, h); + } + } catch (Exception ex) { + log.error("========interceptor handle cacheDirtiedEvent {}", cacheDirtiedEvent, ex); + } + }); + + cacheDirtyHandlers.get(cacheDirtiedEvent.getKey()).forEach(h -> { + try { + h.accept(cacheDirtiedEvent); + log.info("handler handled cache dirty event={}, handler={}", cacheDirtiedEvent, h); + } catch (Exception ex) { + log.error("========handler handle cacheDirtiedEvent {}", cacheDirtiedEvent, ex); + } + }); + + } + + @Override + public void afterPropertiesSet() throws Exception { + broadcastQueue = EventBroadcast.build("Local_cache-coordinate", this::onCacheDirtied); + } + + public static RedisLocalCacheCoordinate.Builder builder() { + return new RedisLocalCacheCoordinate.Builder(); + } + + @Data + public static class Builder { + EventBroadcast EventBroadcast; + AppRuntime appRuntime; + + public RedisLocalCacheCoordinate.Builder EventBroadcast(EventBroadcast EventBroadcast) { + this.EventBroadcast = EventBroadcast; + return this; + } + + public RedisLocalCacheCoordinate.Builder appRuntime(AppRuntime appRuntime) { + this.appRuntime = appRuntime; + return this; + } + + public LocalCacheCoordinate build() { + if (appRuntime.getEnv() == AppEnvEnum.unittest) { + return new LocalCacheCoordinate() { + private Multimap> cacheDirtyHandlers = ArrayListMultimap.create(); + private List interceptors = Lists.newArrayList(); + + @Override + public void notifyCacheDirty(CacheDirtiedEvent event) { + cacheDirtyHandlers.get(event.getKey()) + .forEach(h -> h.accept(event)); + + interceptors.stream() + .filter(i -> i.getFilter().test(event)) + .forEach(i -> i.getConsumer().accept(event)); + } + + @Override + public void registerCacheDirtyListener(String key, Consumer consumer) { + cacheDirtyHandlers.put(key, consumer); + } + + @Override + public void registerCacheDirtyInterceptor(CacheDirtiedEventInterceptor interceptor) { + interceptors.add(interceptor); + } + }; + } + + Objects.requireNonNull(EventBroadcast); + RedisLocalCacheCoordinate client = new RedisLocalCacheCoordinate(); + client.setEventBroadcast(EventBroadcast); + client.setExecutor(new AfterCommitExecutor()); + + return client; + } + } + + /** + * stolen from http://azagorneanu.blogspot.jp/2013/06/transaction-synchronization-callbacks.html + * 保证在交易结束后被调用. + */ + protected static class AfterCommitExecutor extends TransactionSynchronizationAdapter { + //大部分情况只有会清除一个缓存. 因此数组初始化为1 + private final ThreadLocal> contexts = ThreadLocal + .withInitial((Supplier>) () -> Lists.newArrayListWithCapacity(1)); + + public void execute(Runnable runnable) { + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + runnable.run(); + return; + } + contexts.get().add(runnable); + TransactionSynchronizationManager.registerSynchronization(this); + } + + @Override + public void afterCommit() { + contexts.get().forEach(e -> { + try { + e.run(); + } catch (Exception ex) { + log.error("Failed to execute runnable = {} ", e, ex); + } + }); + } + + @Override + public void afterCompletion(int status) { + contexts.remove(); + } + } +} From e793983d6f25b4b1fbf59d016014ffe41a5d199f Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 20 Jun 2024 10:36:54 +0800 Subject: [PATCH 10/41] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E4=BE=9D?= =?UTF-8?q?=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis-support-lib/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/redis-support-lib/pom.xml b/redis-support-lib/pom.xml index 0a63193..1126c03 100644 --- a/redis-support-lib/pom.xml +++ b/redis-support-lib/pom.xml @@ -21,6 +21,11 @@ org.springframework.boot spring-boot-starter-data-redis + + + org.apache.commons + commons-pool2 + \ No newline at end of file From 336257a62b00f766256b2fb866c06f0605d2ee0e Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 21 Jun 2024 17:04:01 +0800 Subject: [PATCH 11/41] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/web/support/config/ApiResultWrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java index 07f2317..e6be516 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java @@ -24,7 +24,7 @@ class ApiResultWrapper extends ApiResult { //没有appId时沿用之前的拼装逻辑"${appName}_${httpCode}${ErrorCode}" //存在appId时拼装逻辑调整为"${appId}${ErrorCode}" - if (!StringUtils.isEmpty(appRuntime.getAppId())) { + if (!StringUtils.isEmpty(appRuntime.getAppId()) && StringUtils.length(result.getCode() + "") < 6) { this.code = Integer.parseInt(StringUtils.getDigits(appRuntime.getAppId() + StringUtils.leftPad(result.getCode() + "", 3, "0"))); } } From ed2590a4da5a466d4343c68228c5b6a90c5bc05c Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 27 Jun 2024 17:26:25 +0800 Subject: [PATCH 12/41] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96header=E6=89=93?= =?UTF-8?q?=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gateway/support/plugin/impl/ApiLogHook.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/ApiLogHook.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/ApiLogHook.java index 92d1c85..46e4095 100644 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/ApiLogHook.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/ApiLogHook.java @@ -12,16 +12,14 @@ import cn.axzo.foundation.web.support.rpc.RpcClient; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import lombok.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.util.AntPathMatcher; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -33,7 +31,8 @@ public class ApiLogHook implements ProxyHook { private static final ThreadLocal REQUEST_PARAMS_THREAD_LOCAL = new ThreadLocal<>(); private static final int API_RESPONSE_CONTENT_PRINT_LENGTH = 2048; private final AntPathMatcher antPathMatcher = new AntPathMatcher(); - private static final Set DEFAULT_IGNORE_HEADER_NAMES = ImmutableSet.of("User-Agent", "Authorization", "_EMPLOYEE_PRINCIPAL", "Bfs-Authorization"); + private static final Set DEFAULT_IGNORE_HEADER_NAMES = ImmutableSet.of("User-Agent", "Authorization", + "Axzo-Authorization", "Bfs-Authorization", "referer", "sec-fetch-site","sec-ch-ua-platform","sec-fetch-dest"); private final int maxLogLength; private final Predicate logEnableHook; @@ -52,7 +51,12 @@ public class ApiLogHook implements ProxyHook { this.logEnableHook = Optional.ofNullable(logEnableHook).orElse(hookContext -> true); this.maxLogLength = Optional.ofNullable(maxLogLength).orElse(API_RESPONSE_CONTENT_PRINT_LENGTH); this.ignorePathPatterns = Optional.ofNullable(ignorePathPatterns).orElse(ImmutableList.of()); - this.ignoreHeaderNames = Optional.ofNullable(ignoreHeaderNames).orElse(DEFAULT_IGNORE_HEADER_NAMES); + //忽略大小写 + this.ignoreHeaderNames = Optional.ofNullable(ignoreHeaderNames).orElseGet(() -> { + TreeSet sets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + sets.addAll(DEFAULT_IGNORE_HEADER_NAMES); + return sets; + }); this.printOriginalParams = BooleanUtils.isTrue(printOriginalParams); // 默认 BusinessException 异常不打印ERROR级别日志. this.warningExceptions = Optional.ofNullable(warningExceptions).orElse(ImmutableSet.of(BusinessException.class)); From e520e2d550f7805dfd296949dbfec5f7a9992192 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Mon, 1 Jul 2024 20:57:51 +0800 Subject: [PATCH 13/41] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9=E5=8D=95?= =?UTF-8?q?=E5=85=83=E6=B5=8B=E8=AF=95=E5=86=85=E5=AD=98=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/unittest/support/config/RedisConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittest-support-lib/src/main/java/cn/axzo/foundation/unittest/support/config/RedisConfig.java b/unittest-support-lib/src/main/java/cn/axzo/foundation/unittest/support/config/RedisConfig.java index 922c827..dbc8929 100644 --- a/unittest-support-lib/src/main/java/cn/axzo/foundation/unittest/support/config/RedisConfig.java +++ b/unittest-support-lib/src/main/java/cn/axzo/foundation/unittest/support/config/RedisConfig.java @@ -17,7 +17,7 @@ public class RedisConfig { @PostConstruct public void postConstruct() { - redisServer = new RedisServer(16379); + redisServer = RedisServer.builder().port(16739).setting("maxheap 128m").build(); // redis server会在后台启动一个redis server的进程，默认IP=127.0.0.1 redisServer.start(); } From 1f8c2eb5d045d0a49fe463ede550111b70ea0db2 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 4 Jul 2024 15:48:19 +0800 Subject: [PATCH 14/41] =?UTF-8?q?feat:=20=E8=8E=B7=E5=8F=96app=20&=20?= =?UTF-8?q?=E4=BC=98=E5=8C=96apiResult.code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/result/ApiResult.java | 21 ++++ .../web/support/TimerRefreshCache.java | 96 +++++++++++++++++++ .../axzo/foundation/web/support/apps/App.java | 40 ++++++++ .../web/support/apps/AppCenter.java | 9 ++ .../web/support/apps/AppCenterImpl.java | 77 +++++++++++++++ .../web/support/config/ApiResultWrapper.java | 7 +- .../support/context/AxContextInterceptor.java | 18 ++-- .../foundation/web/support/rpc/RpcClient.java | 44 +-------- .../web/support/rpc/RpcClientImpl.java | 41 +++++++- 9 files changed, 298 insertions(+), 55 deletions(-) create mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java create mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java create mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java create mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java diff --git a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java index aca42fc..dbdcee7 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java +++ b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java @@ -1,10 +1,12 @@ package cn.axzo.foundation.result; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Strings; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; @Data @Builder @@ -68,4 +70,23 @@ public class ApiResult { public boolean isSuccess() { return SUCCESS_CODE.equals(getCode()); } + + /** + * 根据appId 获取标准的code + * 如果code > 100000 则认为可能已经带了appId + * 否则拼接当前的appId 到appCode中 + */ + public Integer getStandardCode(String appId) { + if (code == null || Strings.isNullOrEmpty(appId) || isSuccess()) { + return code; + } + if (code >= 1000000) { + return code; + } + try { + return Integer.parseInt(StringUtils.right(StringUtils.getDigits(appId), 4) + StringUtils.leftPad(code + "", 3, "0")); + } catch (Exception ex) { + return code; + } + } } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java new file mode 100644 index 0000000..f286b35 --- /dev/null +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java @@ -0,0 +1,96 @@ +package cn.axzo.foundation.web.support; + +import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +/** + * 间隔一定时间刷新的缓存。 + * 
+ * 1、初次调用时，直接获取需要缓存的内容，并缓存到cache中
+ * 2、每间隔intervalMillis尝试刷新缓存，
+ *      如果刷新缓存成功，更新cache缓存的值
+ *      如果刷新缓存失败，则不更新
+ *
+ * + * @param + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@Slf4j +public class TimerRefreshCache { + private ScheduledThreadPoolExecutor executor; + private String name; + private Long initialDelayMillis; + private Long intervalMillis; + private Supplier refresher; + private Cache cache; + /** 缓存是否以初始化，如果为false，会进行主动加载进行初始化 */ + private AtomicBoolean initialized = new AtomicBoolean(false); + + public TimerRefreshCache(String name, Long initialDelayMillis, Long intervalMillis, + ScheduledThreadPoolExecutor executor, Supplier refresher) { + Preconditions.checkArgument(executor != null); + Preconditions.checkArgument(refresher != null); + Preconditions.checkArgument(intervalMillis != null); + + this.name = name; + this.initialDelayMillis = initialDelayMillis; + this.intervalMillis = intervalMillis; + this.refresher = refresher; + // 该方法获取的，往往是完整的缓存内容。所以maxSize暂定为2即可 + long maximumSize = 2L; + cache = CacheBuilder.newBuilder() + .maximumSize(maximumSize) + .build(); + this.executor = executor; + startTimer(); + } + + private void startTimer() { + executor.scheduleAtFixedRate(this::doRefresh, initialDelayMillis, intervalMillis, TimeUnit.MILLISECONDS); + } + + private boolean doRefresh() { + try { + T value = refresher.get(); + // 当返回为null，表示Value没有变化，这时不用刷新Cache + if (value == null) { + return true; + } + + cache.put(name, value); + if (log.isDebugEnabled()) { + log.debug("{} refreshed, new value={}", name, value); + } + } catch (Throwable e) { + log.error("{} refresh failed", name, e); + return false; + } + return true; + } + + public T get() { + // 如果没有加载过且没有缓存，手动加载一次 + if (!initialized.getAndSet(true) && cache.size() == 0) { + //如果第一次刷新缓存失败. 则重置initialized. 下次请求进入时再次尝试更新cache + if (!doRefresh()) { + initialized.set(false); + } + } + return cache.getIfPresent(name); + } + + // XXX: for unittest + public void put(T configs) { + log.info("{},set->config={}", TimerRefreshCache.this, configs); + cache.put(name, configs); + } +} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java new file mode 100644 index 0000000..bb3edf1 --- /dev/null +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java @@ -0,0 +1,40 @@ +package cn.axzo.foundation.web.support.apps; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.RegExUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.Optional; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class App { + Long id; + Long upstreamId; + String upstreamName; + String appName; + Integer port; + + String host; + + public String getHost() { + return Optional.ofNullable(host).orElse(String.format("http://%s:%s", appName, port)); + } + + /** + * 根据upstreamId的后4位生产appId, 如果后四位位0则替换为9 + * eg 20000000000000124 -> 9124, 20000000000003078 -> 3078 + */ + public String getAppId() { + String right = StringUtils.right(String.valueOf(upstreamId), 4); + if (!right.startsWith("0")) { + return right; + } + return RegExUtils.replaceFirst(right, "0", "9"); + } +} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java new file mode 100644 index 0000000..d98b5d8 --- /dev/null +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java @@ -0,0 +1,9 @@ +package cn.axzo.foundation.web.support.apps; + +import java.util.List; + +public interface AppCenter { + List listAll(); + + App getByName(String appName); +} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java new file mode 100644 index 0000000..c047a71 --- /dev/null +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java @@ -0,0 +1,77 @@ +package cn.axzo.foundation.web.support.apps; + +import cn.axzo.foundation.util.PageUtils; +import cn.axzo.foundation.web.support.TimerRefreshCache; +import cn.axzo.foundation.web.support.rpc.RequestProxy; +import cn.axzo.foundation.web.support.rpc.RpcClient; +import cn.axzo.foundation.web.support.rpc.RpcClientImpl; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Slf4j +public class AppCenterImpl implements AppCenter { + private static final long INITIAL_DELAY_MILLIS = 0; + private static final long REFRESH_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(60); + private TimerRefreshCache> appCache; + private RpcClient rpcClient; + + private String debugHost; + private Map debugAppRoutes; + + /** + * executor 为刷新的executor + * debugHost: 本地调试时 连接的环境 eg: http://pre-api.axzo.cn + * debugAppRoutes: 本地调试时部分appName 与apiSix配置的路由不一致, + * eg data-collection 对应的测试地址http://test-api.axzo.cn/dataCollection 而非http://test-api.axzo.cn/data-collection + * 因此该类映射需要单独处理 + */ + @Builder + public AppCenterImpl(ScheduledThreadPoolExecutor executor, + String debugHost, + Map debugAppRoutes) { + + Objects.requireNonNull(executor); + + this.rpcClient = RpcClientImpl.builder().requestProxy(RequestProxy.SIMPLE_PROXY).build(); + this.appCache = new TimerRefreshCache<>("appCenterCache", INITIAL_DELAY_MILLIS, + REFRESH_INTERVAL_MILLIS, executor, this::loadAllAppHosts); + this.debugHost = debugHost; + this.debugAppRoutes = Optional.ofNullable(debugAppRoutes).orElse(ImmutableMap.of()); + } + + private Map loadAllAppHosts() { + String host = Optional.ofNullable(debugHost).map(e -> e + "/apisix-plat").orElse("http://apisix-plat:8080"); + List apps = PageUtils.drainAll(page -> rpcClient.request() + .url(host + "/api/v1/upstream/list") + .content(new JSONObject() + .fluentPut("pageNum", page) + .fluentPut("pageSize", 50)) + .clz(App.class) + .postAndGetPage()); + + if (!Strings.isNullOrEmpty(debugHost)) { + apps.forEach(e -> e.setHost(debugHost + "/" + debugAppRoutes.getOrDefault(e.getAppName(), e.getAppName()))); + } + + return apps.stream() + .collect(Collectors.toMap(App::getAppName, e -> e, (o, n) -> n)); + } + + @Override + public List listAll() { + return new ArrayList<>(appCache.get().values()); + } + + @Override + public App getByName(String appName) { + return appCache.get().get(appName); + } +} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java index e6be516..3e8d531 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java @@ -4,7 +4,6 @@ import cn.axzo.foundation.result.ApiResult; import cn.axzo.foundation.result.ResultCode; import cn.axzo.foundation.web.support.AppRuntime; import lombok.Builder; -import org.apache.commons.lang3.StringUtils; import java.util.Optional; @@ -22,10 +21,6 @@ class ApiResultWrapper extends ApiResult { this.httpCode = Optional.ofNullable(result.getHttpCode()).orElse(ResultCode.DEFAULT_HTTP_ERROR_CODE); - //没有appId时沿用之前的拼装逻辑"${appName}_${httpCode}${ErrorCode}" - //存在appId时拼装逻辑调整为"${appId}${ErrorCode}" - if (!StringUtils.isEmpty(appRuntime.getAppId()) && StringUtils.length(result.getCode() + "") < 6) { - this.code = Integer.parseInt(StringUtils.getDigits(appRuntime.getAppId() + StringUtils.leftPad(result.getCode() + "", 3, "0"))); - } + this.code = result.getStandardCode(appRuntime.getAppId()); } } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java index 33db5c9..190a7f6 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java @@ -8,7 +8,7 @@ import cn.axzo.foundation.web.support.rpc.RequestParams; import com.alibaba.fastjson.JSONObject; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; -import lombok.RequiredArgsConstructor; +import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; @@ -24,11 +24,17 @@ import java.util.Optional; * 3. 非prd环境支持通过token到puge换 */ @Slf4j -@RequiredArgsConstructor public class AxContextInterceptor implements HandlerInterceptor { private final AppRuntime appRuntime; - private final String supplierHost; + private final String supplierUrl; + + @Builder + public AxContextInterceptor(AppRuntime appRuntime, String debugHost) { + this.appRuntime = appRuntime; + this.supplierUrl = Optional.ofNullable(Strings.emptyToNull(debugHost)) + .orElse("http://pudge:10099") + "/webApi/oauth/apisix/authentication"; + } private final static HttpClient HTTP_CLIENT = OkHttpClientImpl.builder().build(); @@ -83,18 +89,18 @@ public class AxContextInterceptor implements HandlerInterceptor { return JSONObject.parseObject(StringUtils.removeStart(authorization, "Raw "), AxContext.class); } if (authorization.startsWith("Bearer")) { - if (Strings.isNullOrEmpty(supplierHost)) { + if (Strings.isNullOrEmpty(supplierUrl)) { return null; } String result; try { - result = HTTP_CLIENT.get(supplierHost, RequestParams.FormParams.builder() + result = HTTP_CLIENT.get(supplierUrl, RequestParams.FormParams.builder() .headers(ImmutableMap.of("Authorization", authorization, "terminal", request.getHeader("terminal"))) .logEnable(true) .build()); } catch (Exception ex) { - log.error("获取登陆信息错误, url = {}, authorization = {}", supplierHost, authorization, ex); + log.error("获取登陆信息错误, url = {}, authorization = {}", supplierUrl, authorization, ex); return null; } //这里是一个非标准返回 diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java index 45f3119..f6e182b 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java @@ -2,22 +2,18 @@ package cn.axzo.foundation.web.support.rpc; import cn.axzo.foundation.page.PageResp; import cn.axzo.foundation.result.ApiResult; -import cn.axzo.foundation.web.support.context.AxContext; -import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import lombok.extern.slf4j.Slf4j; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; /** * Rpc调用客户端接口 @@ -101,40 +97,9 @@ public interface RpcClient { return delete(url, typeReference, requestParams); } - default ApiResult convert(String body, Class clz) { - return JSONObject.parseObject(body, new TypeReference>(clz) { - }); - } - - Set AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo"); - - // XXX: http/2会把所有Header都转成小写, 历史定义的Header都是大写的，在http/2协议下会透传失败。 - TreeSet CASE_INSENSITIVE_AXZO_HEADERS = AXZO_HEADERS.stream() - .collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER))); - - // 将axzo-开头的header复制到请求的下一跳 - String AZXO_HEADER_PREFIX = "axzo-"; - - List>> DEFAULT_HEADER_SUPPLIERS = ImmutableList.of( - () -> AxContext.getRequest() - .map(request -> Collections.list(request.getHeaderNames()).stream() - // 通过http2协议的请求，默认会把大写转成小写，如"_SESSION_OBJECT" -> "_session_object"，导致session无法透传，下一跳需要通过session验签会失败 - .filter(p -> CASE_INSENSITIVE_AXZO_HEADERS.contains(p) || p.startsWith(AZXO_HEADER_PREFIX)) - .collect(Collectors.toMap(e -> e, e -> request.getHeader(e), (oldValue, newValue) -> newValue))) - .orElse(Collections.emptyMap()), - //设置callerApp - () -> AxContext.getRequest().map(e -> { - Object caller = e.getAttribute(CallerAppInterceptor.NEXT_HTTP_REQUEST_ATTRIBUTE); - return ImmutableMap.of(CallerAppInterceptor.HTTP_REQUEST_HEADER, JSONObject.toJSONString(caller)); - }).orElse(ImmutableMap.of()) - ); - - /** * 使用builder模式来发起请求, 先通过request()获得builder对象. 再构建具体的请求方式 * eg: String resp = rpcClient.request().url("/my").content(Map.of("key", "value")).clz(String.class).post(); - * - * @return */ default RpcRequestBuilder request() { return new RpcRequestBuilder(this); @@ -142,7 +107,6 @@ public interface RpcClient { @Slf4j class RpcRequestBuilder { - private static final long MAX_PER_PAGE_COUNT = 1000; private String url; private Object content; private Class clz; diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java index 2c0a738..881f0c6 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java @@ -2,6 +2,12 @@ package cn.axzo.foundation.web.support.rpc; import cn.axzo.foundation.exception.BusinessException; import cn.axzo.foundation.result.ApiResult; +import cn.axzo.foundation.web.support.context.AxContext; +import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor; +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -11,6 +17,7 @@ import java.util.*; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; @Slf4j public class RpcClientImpl implements RpcClient { @@ -19,14 +26,41 @@ public class RpcClientImpl implements RpcClient { protected HttpClient httpClient; protected RequestProxy requestProxy; + /** 外部服务的appId */ + private String appId; + + private static final Set AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo"); + // XXX: http/2会把所有Header都转成小写, 历史定义的Header都是大写的，在http/2协议下会透传失败。 + private static final TreeSet CASE_INSENSITIVE_AXZO_HEADERS = AXZO_HEADERS.stream() + .collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER))); + // 将axzo-开头的header复制到请求的下一跳 + private static final String AZXO_HEADER_PREFIX = "axzo-"; + private static final List>> DEFAULT_HEADER_SUPPLIERS = ImmutableList.of( + () -> AxContext.getRequest() + .map(request -> Collections.list(request.getHeaderNames()).stream() + // 通过http2协议的请求，默认会把大写转成小写，如"_SESSION_OBJECT" -> "_session_object"，导致session无法透传，下一跳需要通过session验签会失败 + .filter(p -> CASE_INSENSITIVE_AXZO_HEADERS.contains(p) || p.startsWith(AZXO_HEADER_PREFIX)) + .collect(Collectors.toMap(e -> e, e -> request.getHeader(e), (oldValue, newValue) -> newValue))) + .orElse(Collections.emptyMap()), + //设置callerApp + () -> AxContext.getRequest().map(e -> { + Object caller = e.getAttribute(CallerAppInterceptor.NEXT_HTTP_REQUEST_ATTRIBUTE); + return ImmutableMap.of(CallerAppInterceptor.HTTP_REQUEST_HEADER, JSONObject.toJSONString(caller)); + }).orElse(ImmutableMap.of()) + ); + @Builder - public RpcClientImpl(RequestProxy requestProxy, HttpClient.Config config, Supplier> requestHeaderSupplier) { + public RpcClientImpl(RequestProxy requestProxy, + HttpClient.Config config, + Supplier> requestHeaderSupplier, + String appId) { this.requestProxy = Optional.ofNullable(requestProxy).orElse(RequestProxy.SIMPLE_PROXY); this.httpClient = OkHttpClientImpl.builder() .config(config) .build(); - customHeaderSupplier = requestHeaderSupplier; + this.customHeaderSupplier = requestHeaderSupplier; + this.appId = appId; } @Override @@ -36,7 +70,8 @@ public class RpcClientImpl implements RpcClient { Optional resp = requestBySupplier(requestParams, () -> this.getHttpClient().execute(httpMethod, url, requestParams)); ApiResult result = converter.apply(resp.orElse(StringUtils.EMPTY)); if (!result.isSuccess()) { - throw new BusinessException(result.getCode() + "", result.getMsg()); + Integer standardCode = result.getStandardCode(appId); + throw new BusinessException(standardCode + "", result.getMsg()); } return result.getData(); } From 8f3915283493ea3d605a12efedae5500bd575362 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 4 Jul 2024 16:08:25 +0800 Subject: [PATCH 15/41] =?UTF-8?q?feat:=20=E8=8E=B7=E5=8F=96app=20step=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../foundation/web/support/config/DefaultWebMvcConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java index dfc2cf3..2f859fd 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java @@ -83,8 +83,8 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement @Value("${web.serialize.browser-compatible.enabled:true}") private Boolean browserCompatible; - @Value("${web.context.supplier.url:}") - private String contextSupplierUrl; + @Value("${web.debug.host:}") + private String contextSupplierHost; /** * 自定义对返回的errorMsg进行处理 @@ -195,7 +195,7 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement registry.addInterceptor(e); }); } - registry.addInterceptor(new AxContextInterceptor(appRuntime, contextSupplierUrl)); + registry.addInterceptor(new AxContextInterceptor(appRuntime, contextSupplierHost)); } /** From af7aaf7f13d468ec6bafaaa17b0e9ed5556b5848 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 4 Jul 2024 16:13:37 +0800 Subject: [PATCH 16/41] =?UTF-8?q?Revert=20"feat:=20=E8=8E=B7=E5=8F=96app?= =?UTF-8?q?=20&=20=E4=BC=98=E5=8C=96apiResult.code"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 1f8c2eb5d045d0a49fe463ede550111b70ea0db2. --- .../cn/axzo/foundation/result/ApiResult.java | 21 ---- .../web/support/TimerRefreshCache.java | 96 ------------------- .../axzo/foundation/web/support/apps/App.java | 40 -------- .../web/support/apps/AppCenter.java | 9 -- .../web/support/apps/AppCenterImpl.java | 77 --------------- .../web/support/config/ApiResultWrapper.java | 7 +- .../support/context/AxContextInterceptor.java | 18 ++-- .../foundation/web/support/rpc/RpcClient.java | 44 ++++++++- .../web/support/rpc/RpcClientImpl.java | 41 +------- 9 files changed, 55 insertions(+), 298 deletions(-) delete mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java delete mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java delete mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java delete mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java diff --git a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java index dbdcee7..aca42fc 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java +++ b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java @@ -1,12 +1,10 @@ package cn.axzo.foundation.result; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.base.Strings; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; @Data @Builder @@ -70,23 +68,4 @@ public class ApiResult { public boolean isSuccess() { return SUCCESS_CODE.equals(getCode()); } - - /** - * 根据appId 获取标准的code - * 如果code > 100000 则认为可能已经带了appId - * 否则拼接当前的appId 到appCode中 - */ - public Integer getStandardCode(String appId) { - if (code == null || Strings.isNullOrEmpty(appId) || isSuccess()) { - return code; - } - if (code >= 1000000) { - return code; - } - try { - return Integer.parseInt(StringUtils.right(StringUtils.getDigits(appId), 4) + StringUtils.leftPad(code + "", 3, "0")); - } catch (Exception ex) { - return code; - } - } } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java deleted file mode 100644 index f286b35..0000000 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java +++ /dev/null @@ -1,96 +0,0 @@ -package cn.axzo.foundation.web.support; - -import com.google.common.base.Preconditions; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; - -/** - * 间隔一定时间刷新的缓存。 - * 
- * 1、初次调用时，直接获取需要缓存的内容，并缓存到cache中
- * 2、每间隔intervalMillis尝试刷新缓存，
- *      如果刷新缓存成功，更新cache缓存的值
- *      如果刷新缓存失败，则不更新
- *
- * - * @param - */ -@NoArgsConstructor(access = AccessLevel.PRIVATE) -@Slf4j -public class TimerRefreshCache { - private ScheduledThreadPoolExecutor executor; - private String name; - private Long initialDelayMillis; - private Long intervalMillis; - private Supplier refresher; - private Cache cache; - /** 缓存是否以初始化，如果为false，会进行主动加载进行初始化 */ - private AtomicBoolean initialized = new AtomicBoolean(false); - - public TimerRefreshCache(String name, Long initialDelayMillis, Long intervalMillis, - ScheduledThreadPoolExecutor executor, Supplier refresher) { - Preconditions.checkArgument(executor != null); - Preconditions.checkArgument(refresher != null); - Preconditions.checkArgument(intervalMillis != null); - - this.name = name; - this.initialDelayMillis = initialDelayMillis; - this.intervalMillis = intervalMillis; - this.refresher = refresher; - // 该方法获取的，往往是完整的缓存内容。所以maxSize暂定为2即可 - long maximumSize = 2L; - cache = CacheBuilder.newBuilder() - .maximumSize(maximumSize) - .build(); - this.executor = executor; - startTimer(); - } - - private void startTimer() { - executor.scheduleAtFixedRate(this::doRefresh, initialDelayMillis, intervalMillis, TimeUnit.MILLISECONDS); - } - - private boolean doRefresh() { - try { - T value = refresher.get(); - // 当返回为null，表示Value没有变化，这时不用刷新Cache - if (value == null) { - return true; - } - - cache.put(name, value); - if (log.isDebugEnabled()) { - log.debug("{} refreshed, new value={}", name, value); - } - } catch (Throwable e) { - log.error("{} refresh failed", name, e); - return false; - } - return true; - } - - public T get() { - // 如果没有加载过且没有缓存，手动加载一次 - if (!initialized.getAndSet(true) && cache.size() == 0) { - //如果第一次刷新缓存失败. 则重置initialized. 下次请求进入时再次尝试更新cache - if (!doRefresh()) { - initialized.set(false); - } - } - return cache.getIfPresent(name); - } - - // XXX: for unittest - public void put(T configs) { - log.info("{},set->config={}", TimerRefreshCache.this, configs); - cache.put(name, configs); - } -} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java deleted file mode 100644 index bb3edf1..0000000 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java +++ /dev/null @@ -1,40 +0,0 @@ -package cn.axzo.foundation.web.support.apps; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.lang3.RegExUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.Optional; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class App { - Long id; - Long upstreamId; - String upstreamName; - String appName; - Integer port; - - String host; - - public String getHost() { - return Optional.ofNullable(host).orElse(String.format("http://%s:%s", appName, port)); - } - - /** - * 根据upstreamId的后4位生产appId, 如果后四位位0则替换为9 - * eg 20000000000000124 -> 9124, 20000000000003078 -> 3078 - */ - public String getAppId() { - String right = StringUtils.right(String.valueOf(upstreamId), 4); - if (!right.startsWith("0")) { - return right; - } - return RegExUtils.replaceFirst(right, "0", "9"); - } -} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java deleted file mode 100644 index d98b5d8..0000000 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java +++ /dev/null @@ -1,9 +0,0 @@ -package cn.axzo.foundation.web.support.apps; - -import java.util.List; - -public interface AppCenter { - List listAll(); - - App getByName(String appName); -} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java deleted file mode 100644 index c047a71..0000000 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java +++ /dev/null @@ -1,77 +0,0 @@ -package cn.axzo.foundation.web.support.apps; - -import cn.axzo.foundation.util.PageUtils; -import cn.axzo.foundation.web.support.TimerRefreshCache; -import cn.axzo.foundation.web.support.rpc.RequestProxy; -import cn.axzo.foundation.web.support.rpc.RpcClient; -import cn.axzo.foundation.web.support.rpc.RpcClientImpl; -import com.alibaba.fastjson.JSONObject; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; -import lombok.Builder; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -@Slf4j -public class AppCenterImpl implements AppCenter { - private static final long INITIAL_DELAY_MILLIS = 0; - private static final long REFRESH_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(60); - private TimerRefreshCache> appCache; - private RpcClient rpcClient; - - private String debugHost; - private Map debugAppRoutes; - - /** - * executor 为刷新的executor - * debugHost: 本地调试时 连接的环境 eg: http://pre-api.axzo.cn - * debugAppRoutes: 本地调试时部分appName 与apiSix配置的路由不一致, - * eg data-collection 对应的测试地址http://test-api.axzo.cn/dataCollection 而非http://test-api.axzo.cn/data-collection - * 因此该类映射需要单独处理 - */ - @Builder - public AppCenterImpl(ScheduledThreadPoolExecutor executor, - String debugHost, - Map debugAppRoutes) { - - Objects.requireNonNull(executor); - - this.rpcClient = RpcClientImpl.builder().requestProxy(RequestProxy.SIMPLE_PROXY).build(); - this.appCache = new TimerRefreshCache<>("appCenterCache", INITIAL_DELAY_MILLIS, - REFRESH_INTERVAL_MILLIS, executor, this::loadAllAppHosts); - this.debugHost = debugHost; - this.debugAppRoutes = Optional.ofNullable(debugAppRoutes).orElse(ImmutableMap.of()); - } - - private Map loadAllAppHosts() { - String host = Optional.ofNullable(debugHost).map(e -> e + "/apisix-plat").orElse("http://apisix-plat:8080"); - List apps = PageUtils.drainAll(page -> rpcClient.request() - .url(host + "/api/v1/upstream/list") - .content(new JSONObject() - .fluentPut("pageNum", page) - .fluentPut("pageSize", 50)) - .clz(App.class) - .postAndGetPage()); - - if (!Strings.isNullOrEmpty(debugHost)) { - apps.forEach(e -> e.setHost(debugHost + "/" + debugAppRoutes.getOrDefault(e.getAppName(), e.getAppName()))); - } - - return apps.stream() - .collect(Collectors.toMap(App::getAppName, e -> e, (o, n) -> n)); - } - - @Override - public List listAll() { - return new ArrayList<>(appCache.get().values()); - } - - @Override - public App getByName(String appName) { - return appCache.get().get(appName); - } -} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java index 3e8d531..e6be516 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java @@ -4,6 +4,7 @@ import cn.axzo.foundation.result.ApiResult; import cn.axzo.foundation.result.ResultCode; import cn.axzo.foundation.web.support.AppRuntime; import lombok.Builder; +import org.apache.commons.lang3.StringUtils; import java.util.Optional; @@ -21,6 +22,10 @@ class ApiResultWrapper extends ApiResult { this.httpCode = Optional.ofNullable(result.getHttpCode()).orElse(ResultCode.DEFAULT_HTTP_ERROR_CODE); - this.code = result.getStandardCode(appRuntime.getAppId()); + //没有appId时沿用之前的拼装逻辑"${appName}_${httpCode}${ErrorCode}" + //存在appId时拼装逻辑调整为"${appId}${ErrorCode}" + if (!StringUtils.isEmpty(appRuntime.getAppId()) && StringUtils.length(result.getCode() + "") < 6) { + this.code = Integer.parseInt(StringUtils.getDigits(appRuntime.getAppId() + StringUtils.leftPad(result.getCode() + "", 3, "0"))); + } } } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java index 190a7f6..33db5c9 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java @@ -8,7 +8,7 @@ import cn.axzo.foundation.web.support.rpc.RequestParams; import com.alibaba.fastjson.JSONObject; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; -import lombok.Builder; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; @@ -24,17 +24,11 @@ import java.util.Optional; * 3. 非prd环境支持通过token到puge换 */ @Slf4j +@RequiredArgsConstructor public class AxContextInterceptor implements HandlerInterceptor { private final AppRuntime appRuntime; - private final String supplierUrl; - - @Builder - public AxContextInterceptor(AppRuntime appRuntime, String debugHost) { - this.appRuntime = appRuntime; - this.supplierUrl = Optional.ofNullable(Strings.emptyToNull(debugHost)) - .orElse("http://pudge:10099") + "/webApi/oauth/apisix/authentication"; - } + private final String supplierHost; private final static HttpClient HTTP_CLIENT = OkHttpClientImpl.builder().build(); @@ -89,18 +83,18 @@ public class AxContextInterceptor implements HandlerInterceptor { return JSONObject.parseObject(StringUtils.removeStart(authorization, "Raw "), AxContext.class); } if (authorization.startsWith("Bearer")) { - if (Strings.isNullOrEmpty(supplierUrl)) { + if (Strings.isNullOrEmpty(supplierHost)) { return null; } String result; try { - result = HTTP_CLIENT.get(supplierUrl, RequestParams.FormParams.builder() + result = HTTP_CLIENT.get(supplierHost, RequestParams.FormParams.builder() .headers(ImmutableMap.of("Authorization", authorization, "terminal", request.getHeader("terminal"))) .logEnable(true) .build()); } catch (Exception ex) { - log.error("获取登陆信息错误, url = {}, authorization = {}", supplierUrl, authorization, ex); + log.error("获取登陆信息错误, url = {}, authorization = {}", supplierHost, authorization, ex); return null; } //这里是一个非标准返回 diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java index f6e182b..45f3119 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java @@ -2,18 +2,22 @@ package cn.axzo.foundation.web.support.rpc; import cn.axzo.foundation.page.PageResp; import cn.axzo.foundation.result.ApiResult; +import cn.axzo.foundation.web.support.context.AxContext; +import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import lombok.extern.slf4j.Slf4j; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Rpc调用客户端接口 @@ -97,9 +101,40 @@ public interface RpcClient { return delete(url, typeReference, requestParams); } + default ApiResult convert(String body, Class clz) { + return JSONObject.parseObject(body, new TypeReference>(clz) { + }); + } + + Set AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo"); + + // XXX: http/2会把所有Header都转成小写, 历史定义的Header都是大写的，在http/2协议下会透传失败。 + TreeSet CASE_INSENSITIVE_AXZO_HEADERS = AXZO_HEADERS.stream() + .collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER))); + + // 将axzo-开头的header复制到请求的下一跳 + String AZXO_HEADER_PREFIX = "axzo-"; + + List>> DEFAULT_HEADER_SUPPLIERS = ImmutableList.of( + () -> AxContext.getRequest() + .map(request -> Collections.list(request.getHeaderNames()).stream() + // 通过http2协议的请求，默认会把大写转成小写，如"_SESSION_OBJECT" -> "_session_object"，导致session无法透传，下一跳需要通过session验签会失败 + .filter(p -> CASE_INSENSITIVE_AXZO_HEADERS.contains(p) || p.startsWith(AZXO_HEADER_PREFIX)) + .collect(Collectors.toMap(e -> e, e -> request.getHeader(e), (oldValue, newValue) -> newValue))) + .orElse(Collections.emptyMap()), + //设置callerApp + () -> AxContext.getRequest().map(e -> { + Object caller = e.getAttribute(CallerAppInterceptor.NEXT_HTTP_REQUEST_ATTRIBUTE); + return ImmutableMap.of(CallerAppInterceptor.HTTP_REQUEST_HEADER, JSONObject.toJSONString(caller)); + }).orElse(ImmutableMap.of()) + ); + + /** * 使用builder模式来发起请求, 先通过request()获得builder对象. 再构建具体的请求方式 * eg: String resp = rpcClient.request().url("/my").content(Map.of("key", "value")).clz(String.class).post(); + * + * @return */ default RpcRequestBuilder request() { return new RpcRequestBuilder(this); @@ -107,6 +142,7 @@ public interface RpcClient { @Slf4j class RpcRequestBuilder { + private static final long MAX_PER_PAGE_COUNT = 1000; private String url; private Object content; private Class clz; diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java index 881f0c6..2c0a738 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java @@ -2,12 +2,6 @@ package cn.axzo.foundation.web.support.rpc; import cn.axzo.foundation.exception.BusinessException; import cn.axzo.foundation.result.ApiResult; -import cn.axzo.foundation.web.support.context.AxContext; -import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor; -import com.alibaba.fastjson.JSONObject; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -17,7 +11,6 @@ import java.util.*; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; @Slf4j public class RpcClientImpl implements RpcClient { @@ -26,41 +19,14 @@ public class RpcClientImpl implements RpcClient { protected HttpClient httpClient; protected RequestProxy requestProxy; - /** 外部服务的appId */ - private String appId; - - private static final Set AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo"); - // XXX: http/2会把所有Header都转成小写, 历史定义的Header都是大写的，在http/2协议下会透传失败。 - private static final TreeSet CASE_INSENSITIVE_AXZO_HEADERS = AXZO_HEADERS.stream() - .collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER))); - // 将axzo-开头的header复制到请求的下一跳 - private static final String AZXO_HEADER_PREFIX = "axzo-"; - private static final List>> DEFAULT_HEADER_SUPPLIERS = ImmutableList.of( - () -> AxContext.getRequest() - .map(request -> Collections.list(request.getHeaderNames()).stream() - // 通过http2协议的请求，默认会把大写转成小写，如"_SESSION_OBJECT" -> "_session_object"，导致session无法透传，下一跳需要通过session验签会失败 - .filter(p -> CASE_INSENSITIVE_AXZO_HEADERS.contains(p) || p.startsWith(AZXO_HEADER_PREFIX)) - .collect(Collectors.toMap(e -> e, e -> request.getHeader(e), (oldValue, newValue) -> newValue))) - .orElse(Collections.emptyMap()), - //设置callerApp - () -> AxContext.getRequest().map(e -> { - Object caller = e.getAttribute(CallerAppInterceptor.NEXT_HTTP_REQUEST_ATTRIBUTE); - return ImmutableMap.of(CallerAppInterceptor.HTTP_REQUEST_HEADER, JSONObject.toJSONString(caller)); - }).orElse(ImmutableMap.of()) - ); - @Builder - public RpcClientImpl(RequestProxy requestProxy, - HttpClient.Config config, - Supplier> requestHeaderSupplier, - String appId) { + public RpcClientImpl(RequestProxy requestProxy, HttpClient.Config config, Supplier> requestHeaderSupplier) { this.requestProxy = Optional.ofNullable(requestProxy).orElse(RequestProxy.SIMPLE_PROXY); this.httpClient = OkHttpClientImpl.builder() .config(config) .build(); - this.customHeaderSupplier = requestHeaderSupplier; - this.appId = appId; + customHeaderSupplier = requestHeaderSupplier; } @Override @@ -70,8 +36,7 @@ public class RpcClientImpl implements RpcClient { Optional resp = requestBySupplier(requestParams, () -> this.getHttpClient().execute(httpMethod, url, requestParams)); ApiResult result = converter.apply(resp.orElse(StringUtils.EMPTY)); if (!result.isSuccess()) { - Integer standardCode = result.getStandardCode(appId); - throw new BusinessException(standardCode + "", result.getMsg()); + throw new BusinessException(result.getCode() + "", result.getMsg()); } return result.getData(); } From 8431e71fdec4bfb49573b8640bc8b0fc7d8a8aa7 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 4 Jul 2024 16:33:50 +0800 Subject: [PATCH 17/41] =?UTF-8?q?feat:=20=E8=8E=B7=E5=8F=96app=20step=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/result/ApiResult.java | 4 +- .../web/support/apps/AppCenterImpl.java | 30 +++++-- .../web/support/config/ApiResultWrapper.java | 2 +- .../support/config/DefaultWebMvcConfig.java | 4 +- .../web/support/rpc/RpcClientImpl.java | 11 +-- .../web/support/rpc/RpcClientWrapper.java | 89 ++++++------------- 6 files changed, 53 insertions(+), 87 deletions(-) diff --git a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java index dbdcee7..9e175ad 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java +++ b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java @@ -76,8 +76,8 @@ public class ApiResult { * 如果code > 100000 则认为可能已经带了appId * 否则拼接当前的appId 到appCode中 */ - public Integer getStandardCode(String appId) { - if (code == null || Strings.isNullOrEmpty(appId) || isSuccess()) { + public static Integer getStandardCode(String appId, Integer code) { + if (code == null || Strings.isNullOrEmpty(appId) || SUCCESS_CODE.equals(code)) { return code; } if (code >= 1000000) { diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java index c047a71..a86df17 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java @@ -1,10 +1,12 @@ package cn.axzo.foundation.web.support.apps; +import cn.axzo.foundation.page.PageResp; import cn.axzo.foundation.util.PageUtils; import cn.axzo.foundation.web.support.TimerRefreshCache; import cn.axzo.foundation.web.support.rpc.RequestProxy; import cn.axzo.foundation.web.support.rpc.RpcClient; import cn.axzo.foundation.web.support.rpc.RpcClientImpl; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; @@ -42,20 +44,30 @@ public class AppCenterImpl implements AppCenter { this.rpcClient = RpcClientImpl.builder().requestProxy(RequestProxy.SIMPLE_PROXY).build(); this.appCache = new TimerRefreshCache<>("appCenterCache", INITIAL_DELAY_MILLIS, - REFRESH_INTERVAL_MILLIS, executor, this::loadAllAppHosts); + REFRESH_INTERVAL_MILLIS, executor, this::loadApps); this.debugHost = debugHost; this.debugAppRoutes = Optional.ofNullable(debugAppRoutes).orElse(ImmutableMap.of()); } - private Map loadAllAppHosts() { + private Map loadApps() { String host = Optional.ofNullable(debugHost).map(e -> e + "/apisix-plat").orElse("http://apisix-plat:8080"); - List apps = PageUtils.drainAll(page -> rpcClient.request() - .url(host + "/api/v1/upstream/list") - .content(new JSONObject() - .fluentPut("pageNum", page) - .fluentPut("pageSize", 50)) - .clz(App.class) - .postAndGetPage()); + List apps = PageUtils.drainAll(page -> { + JSONObject result = rpcClient.request() + .url(host + "/api/v1/upstream/list") + .content(new JSONObject() + .fluentPut("pageNum", page) + .fluentPut("pageSize", 50)) + .clz(JSONObject.class) + .post(); + //结构不一样, 转换为自己的pageResp + return PageResp.builder() + .total(result.getLong("totalCount")) + .current(result.getLong("page")) + .size(result.getLong("pageSize")) + .data(Optional.ofNullable(result.getJSONArray("list")).orElse(new JSONArray()) + .toJavaList(App.class)) + .build(); + }); if (!Strings.isNullOrEmpty(debugHost)) { apps.forEach(e -> e.setHost(debugHost + "/" + debugAppRoutes.getOrDefault(e.getAppName(), e.getAppName()))); diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java index 3e8d531..9c1e41c 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/ApiResultWrapper.java @@ -21,6 +21,6 @@ class ApiResultWrapper extends ApiResult { this.httpCode = Optional.ofNullable(result.getHttpCode()).orElse(ResultCode.DEFAULT_HTTP_ERROR_CODE); - this.code = result.getStandardCode(appRuntime.getAppId()); + this.code = getStandardCode(appRuntime.getAppId(), result.getCode()); } } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java index 2f859fd..1c767d9 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java @@ -84,7 +84,7 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement private Boolean browserCompatible; @Value("${web.debug.host:}") - private String contextSupplierHost; + private String debugHost; /** * 自定义对返回的errorMsg进行处理 @@ -195,7 +195,7 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement registry.addInterceptor(e); }); } - registry.addInterceptor(new AxContextInterceptor(appRuntime, contextSupplierHost)); + registry.addInterceptor(new AxContextInterceptor(appRuntime, debugHost)); } /** diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java index 881f0c6..80a4237 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java @@ -25,10 +25,6 @@ public class RpcClientImpl implements RpcClient { @Getter protected HttpClient httpClient; protected RequestProxy requestProxy; - - /** 外部服务的appId */ - private String appId; - private static final Set AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo"); // XXX: http/2会把所有Header都转成小写, 历史定义的Header都是大写的，在http/2协议下会透传失败。 private static final TreeSet CASE_INSENSITIVE_AXZO_HEADERS = AXZO_HEADERS.stream() @@ -52,15 +48,13 @@ public class RpcClientImpl implements RpcClient { @Builder public RpcClientImpl(RequestProxy requestProxy, HttpClient.Config config, - Supplier> requestHeaderSupplier, - String appId) { + Supplier> requestHeaderSupplier) { this.requestProxy = Optional.ofNullable(requestProxy).orElse(RequestProxy.SIMPLE_PROXY); this.httpClient = OkHttpClientImpl.builder() .config(config) .build(); this.customHeaderSupplier = requestHeaderSupplier; - this.appId = appId; } @Override @@ -70,8 +64,7 @@ public class RpcClientImpl implements RpcClient { Optional resp = requestBySupplier(requestParams, () -> this.getHttpClient().execute(httpMethod, url, requestParams)); ApiResult result = converter.apply(resp.orElse(StringUtils.EMPTY)); if (!result.isSuccess()) { - Integer standardCode = result.getStandardCode(appId); - throw new BusinessException(standardCode + "", result.getMsg()); + throw new BusinessException(result.getCode() + "", result.getMsg()); } return result.getData(); } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientWrapper.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientWrapper.java index 731b45e..ebc3651 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientWrapper.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientWrapper.java @@ -1,10 +1,9 @@ package cn.axzo.foundation.web.support.rpc; +import cn.axzo.foundation.exception.BusinessException; import cn.axzo.foundation.result.ApiResult; -import cn.axzo.foundation.web.support.rpc.exception.RpcNetworkException; +import cn.axzo.foundation.web.support.apps.App; import com.google.common.base.Preconditions; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -12,13 +11,9 @@ import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; /** * 一个RpcClient的wrapper. 可以方便的在RpcClient和其他Client切换. @@ -26,88 +21,54 @@ import java.util.stream.Collectors; */ @Slf4j public class RpcClientWrapper implements RpcClient { - private final Supplier> hostsResolver; - private Supplier hostResolver; + private final Supplier appResolver; + private final RpcClient normalRpcClient; @Getter - private volatile RpcClient activeRpcClient; - private AtomicInteger roundRobinIndex = new AtomicInteger(0); - - private RpcClient normalRpcClient; - - final Cache excludeHostCache = CacheBuilder. - newBuilder() - .expireAfterWrite(1, TimeUnit.MINUTES).build(); - + private RpcClient activeRpcClient; @lombok.Builder public RpcClientWrapper(RpcClient normalRpcClient, - Supplier hostResolver, - Supplier> hostsResolver, + Supplier appResolver, ClientType defaultClientType) { Preconditions.checkArgument(normalRpcClient != null, "normalRpcClient不能为空"); - if (normalRpcClient != null) { - Preconditions.checkArgument(hostResolver != null || hostsResolver != null, "如果是normalRpcClient, hostResolver必须有值"); - Preconditions.checkArgument(normalRpcClient instanceof RpcClientImpl, "normalRpcClient必须是RpcClientImpl.class实现"); - } + Preconditions.checkArgument(appResolver != null, "如果是normalRpcClient, appResolver必须有值"); + Preconditions.checkArgument(normalRpcClient instanceof RpcClientImpl, "normalRpcClient必须是RpcClientImpl.class实现"); this.normalRpcClient = normalRpcClient; - this.hostResolver = hostResolver; - this.hostsResolver = hostsResolver; + this.appResolver = appResolver; activate(Optional.ofNullable(defaultClientType).orElse(ClientType.NORMAL)); } @Override public T execute(HttpClient.HttpMethod httpMethod, String path, RequestParams requestParams, Function> converter) { - String host = resolveHost(); try { - return activeRpcClient.execute(httpMethod, resolvePath(host, path), requestParams, converter); - } catch (RpcNetworkException e) { - excludeHostCache.put(host, "1"); - log.warn("get network exception, add host {} to invalid pool and cooldown 1 minutes", host); + return activeRpcClient.execute(httpMethod, resolvePath(resolveHost(), path), requestParams, converter); + } catch (BusinessException e) { + int code = Integer.parseInt(e.getErrorCode()); + Integer standardCode = ApiResult.getStandardCode(appResolver.get().getAppId(), code); + if (standardCode != code) { + throw new BusinessException(standardCode + "", e.getErrorMsg()); + } throw e; } } @Override public R execute(HttpClient.HttpMethod httpMethod, String path, RequestParams requestParams, BiFunction>, R> responder) { - String host = resolveHost(); try { - return activeRpcClient.execute(httpMethod, resolvePath(host, path), requestParams, responder); - } catch (RpcNetworkException e) { - excludeHostCache.put(host, "1"); - log.warn("get network exception, add host {} to invalid pool and cooldown 1 minutes", host); + return activeRpcClient.execute(httpMethod, resolvePath(resolveHost(), path), requestParams, responder); + } catch (BusinessException e) { + int code = Integer.parseInt(e.getErrorCode()); + Integer standardCode = ApiResult.getStandardCode(appResolver.get().getAppId(), code); + if (standardCode != code) { + throw new BusinessException(standardCode + "", e.getErrorMsg()); + } throw e; } } protected String resolveHost() { - if (hostResolver == null && hostsResolver == null) { - return StringUtils.EMPTY; - } - // 如果hostResolver不为空, 且path没有包含protocol与host. 则尝试将host与path拼接 - if (hostResolver != null) { - return hostResolver.get(); - } - List hosts = hostsResolver.get(); - // 如果只有 1 个 host，没必要做选择。 - if (hosts.size() == 1) { - return hosts.get(0); - } - ConcurrentMap excludeHosts = excludeHostCache.asMap(); - List availables = excludeHosts.size() == 0 ? hosts : hosts.stream().filter(i -> !excludeHosts.containsKey(i)).collect(Collectors.toList()); - if (availables.size() == 0) { - return hosts.get(0); - } else if (availables.size() == 1) { - return availables.get(0); - } - // 使用 round robin 算法，选择可用节点。 - int index = roundRobinIndex.getAndAccumulate(1, (x, y) -> { - if ((x + y) >= availables.size()) { - return 0; - } - return x + y; - }); - return availables.get(index); + return appResolver.get().getHost(); } protected String resolvePath(String host, String path) { @@ -126,7 +87,7 @@ public class RpcClientWrapper implements RpcClient { public RpcClient activate(ClientType type) { if (type == ClientType.NORMAL) { Preconditions.checkState(normalRpcClient != null); - Preconditions.checkState(hostResolver != null || hostsResolver != null); + Preconditions.checkState(appResolver != null); activeRpcClient = normalRpcClient; } else { throw new UnsupportedOperationException("unsupported clientType"); From 105481f94bb207f9c5c28f270727907cf104d690 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 4 Jul 2024 18:57:36 +0800 Subject: [PATCH 18/41] =?UTF-8?q?feat:=20=E8=8E=B7=E5=8F=96app=20step=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/web/support/apps/AppCenterImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java index a86df17..61de035 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java @@ -26,6 +26,7 @@ public class AppCenterImpl implements AppCenter { private RpcClient rpcClient; private String debugHost; + private String listAppUrl; private Map debugAppRoutes; /** @@ -46,14 +47,15 @@ public class AppCenterImpl implements AppCenter { this.appCache = new TimerRefreshCache<>("appCenterCache", INITIAL_DELAY_MILLIS, REFRESH_INTERVAL_MILLIS, executor, this::loadApps); this.debugHost = debugHost; + this.listAppUrl = Optional.ofNullable(Strings.emptyToNull(debugHost)).map(e -> e + "/apisix-plat").orElse("http://apisix-plat:8080") + + "/api/v1/upstream/list"; this.debugAppRoutes = Optional.ofNullable(debugAppRoutes).orElse(ImmutableMap.of()); } private Map loadApps() { - String host = Optional.ofNullable(debugHost).map(e -> e + "/apisix-plat").orElse("http://apisix-plat:8080"); List apps = PageUtils.drainAll(page -> { JSONObject result = rpcClient.request() - .url(host + "/api/v1/upstream/list") + .url(listAppUrl) .content(new JSONObject() .fluentPut("pageNum", page) .fluentPut("pageSize", 50)) From 077bf1cad4327325cf49c28577d0e115e3168247 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 4 Jul 2024 19:00:33 +0800 Subject: [PATCH 19/41] =?UTF-8?q?feat:=20=E8=8E=B7=E5=8F=96app=20step=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../foundation/web/support/context/AxContextInterceptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java index 190a7f6..e495dab 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java @@ -32,7 +32,7 @@ public class AxContextInterceptor implements HandlerInterceptor { @Builder public AxContextInterceptor(AppRuntime appRuntime, String debugHost) { this.appRuntime = appRuntime; - this.supplierUrl = Optional.ofNullable(Strings.emptyToNull(debugHost)) + this.supplierUrl = Optional.ofNullable(Strings.emptyToNull(debugHost)).map(e -> e + "/pudge") .orElse("http://pudge:10099") + "/webApi/oauth/apisix/authentication"; } From 02d9503133746892b9382c3095f4e12cc7e51a4e Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 5 Jul 2024 09:20:54 +0800 Subject: [PATCH 20/41] =?UTF-8?q?feat:=20=E8=8E=B7=E5=8F=96app=20step=206?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web/support/TimerRefreshCache.java | 96 +++++++++++++++++++ .../axzo/foundation/web/support/apps/App.java | 40 ++++++++ .../web/support/apps/AppCenter.java | 9 ++ .../foundation/web/support/rpc/RpcClient.java | 44 +-------- 4 files changed, 149 insertions(+), 40 deletions(-) create mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java create mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java create mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java new file mode 100644 index 0000000..f286b35 --- /dev/null +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/TimerRefreshCache.java @@ -0,0 +1,96 @@ +package cn.axzo.foundation.web.support; + +import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +/** + * 间隔一定时间刷新的缓存。 + * 
+ * 1、初次调用时，直接获取需要缓存的内容，并缓存到cache中
+ * 2、每间隔intervalMillis尝试刷新缓存，
+ *      如果刷新缓存成功，更新cache缓存的值
+ *      如果刷新缓存失败，则不更新
+ *
+ * + * @param + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@Slf4j +public class TimerRefreshCache { + private ScheduledThreadPoolExecutor executor; + private String name; + private Long initialDelayMillis; + private Long intervalMillis; + private Supplier refresher; + private Cache cache; + /** 缓存是否以初始化，如果为false，会进行主动加载进行初始化 */ + private AtomicBoolean initialized = new AtomicBoolean(false); + + public TimerRefreshCache(String name, Long initialDelayMillis, Long intervalMillis, + ScheduledThreadPoolExecutor executor, Supplier refresher) { + Preconditions.checkArgument(executor != null); + Preconditions.checkArgument(refresher != null); + Preconditions.checkArgument(intervalMillis != null); + + this.name = name; + this.initialDelayMillis = initialDelayMillis; + this.intervalMillis = intervalMillis; + this.refresher = refresher; + // 该方法获取的，往往是完整的缓存内容。所以maxSize暂定为2即可 + long maximumSize = 2L; + cache = CacheBuilder.newBuilder() + .maximumSize(maximumSize) + .build(); + this.executor = executor; + startTimer(); + } + + private void startTimer() { + executor.scheduleAtFixedRate(this::doRefresh, initialDelayMillis, intervalMillis, TimeUnit.MILLISECONDS); + } + + private boolean doRefresh() { + try { + T value = refresher.get(); + // 当返回为null，表示Value没有变化，这时不用刷新Cache + if (value == null) { + return true; + } + + cache.put(name, value); + if (log.isDebugEnabled()) { + log.debug("{} refreshed, new value={}", name, value); + } + } catch (Throwable e) { + log.error("{} refresh failed", name, e); + return false; + } + return true; + } + + public T get() { + // 如果没有加载过且没有缓存，手动加载一次 + if (!initialized.getAndSet(true) && cache.size() == 0) { + //如果第一次刷新缓存失败. 则重置initialized. 下次请求进入时再次尝试更新cache + if (!doRefresh()) { + initialized.set(false); + } + } + return cache.getIfPresent(name); + } + + // XXX: for unittest + public void put(T configs) { + log.info("{},set->config={}", TimerRefreshCache.this, configs); + cache.put(name, configs); + } +} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java new file mode 100644 index 0000000..bb3edf1 --- /dev/null +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/App.java @@ -0,0 +1,40 @@ +package cn.axzo.foundation.web.support.apps; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.RegExUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.Optional; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class App { + Long id; + Long upstreamId; + String upstreamName; + String appName; + Integer port; + + String host; + + public String getHost() { + return Optional.ofNullable(host).orElse(String.format("http://%s:%s", appName, port)); + } + + /** + * 根据upstreamId的后4位生产appId, 如果后四位位0则替换为9 + * eg 20000000000000124 -> 9124, 20000000000003078 -> 3078 + */ + public String getAppId() { + String right = StringUtils.right(String.valueOf(upstreamId), 4); + if (!right.startsWith("0")) { + return right; + } + return RegExUtils.replaceFirst(right, "0", "9"); + } +} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java new file mode 100644 index 0000000..d98b5d8 --- /dev/null +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenter.java @@ -0,0 +1,9 @@ +package cn.axzo.foundation.web.support.apps; + +import java.util.List; + +public interface AppCenter { + List listAll(); + + App getByName(String appName); +} diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java index 45f3119..f6e182b 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClient.java @@ -2,22 +2,18 @@ package cn.axzo.foundation.web.support.rpc; import cn.axzo.foundation.page.PageResp; import cn.axzo.foundation.result.ApiResult; -import cn.axzo.foundation.web.support.context.AxContext; -import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import lombok.extern.slf4j.Slf4j; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; /** * Rpc调用客户端接口 @@ -101,40 +97,9 @@ public interface RpcClient { return delete(url, typeReference, requestParams); } - default ApiResult convert(String body, Class clz) { - return JSONObject.parseObject(body, new TypeReference>(clz) { - }); - } - - Set AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo"); - - // XXX: http/2会把所有Header都转成小写, 历史定义的Header都是大写的，在http/2协议下会透传失败。 - TreeSet CASE_INSENSITIVE_AXZO_HEADERS = AXZO_HEADERS.stream() - .collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER))); - - // 将axzo-开头的header复制到请求的下一跳 - String AZXO_HEADER_PREFIX = "axzo-"; - - List>> DEFAULT_HEADER_SUPPLIERS = ImmutableList.of( - () -> AxContext.getRequest() - .map(request -> Collections.list(request.getHeaderNames()).stream() - // 通过http2协议的请求，默认会把大写转成小写，如"_SESSION_OBJECT" -> "_session_object"，导致session无法透传，下一跳需要通过session验签会失败 - .filter(p -> CASE_INSENSITIVE_AXZO_HEADERS.contains(p) || p.startsWith(AZXO_HEADER_PREFIX)) - .collect(Collectors.toMap(e -> e, e -> request.getHeader(e), (oldValue, newValue) -> newValue))) - .orElse(Collections.emptyMap()), - //设置callerApp - () -> AxContext.getRequest().map(e -> { - Object caller = e.getAttribute(CallerAppInterceptor.NEXT_HTTP_REQUEST_ATTRIBUTE); - return ImmutableMap.of(CallerAppInterceptor.HTTP_REQUEST_HEADER, JSONObject.toJSONString(caller)); - }).orElse(ImmutableMap.of()) - ); - - /** * 使用builder模式来发起请求, 先通过request()获得builder对象. 再构建具体的请求方式 * eg: String resp = rpcClient.request().url("/my").content(Map.of("key", "value")).clz(String.class).post(); - * - * @return */ default RpcRequestBuilder request() { return new RpcRequestBuilder(this); @@ -142,7 +107,6 @@ public interface RpcClient { @Slf4j class RpcRequestBuilder { - private static final long MAX_PER_PAGE_COUNT = 1000; private String url; private Object content; private Class clz; From a949372904d9f96bd24d0c0a467c9c9d9141dffc Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 5 Jul 2024 10:11:08 +0800 Subject: [PATCH 21/41] =?UTF-8?q?feat:=20=E8=8E=B7=E5=8F=96app=20step=207?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/web/support/apps/AppCenterImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java index 61de035..87fc58b 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java @@ -58,7 +58,8 @@ public class AppCenterImpl implements AppCenter { .url(listAppUrl) .content(new JSONObject() .fluentPut("pageNum", page) - .fluentPut("pageSize", 50)) + //分页接口有问题, 这里设置一个最大值. 一次加载完所有的数据 + .fluentPut("pageSize", Integer.MAX_VALUE)) .clz(JSONObject.class) .post(); //结构不一样, 转换为自己的pageResp From c5e235872bc86c31c0e45f7878a5f8428d65794a Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 5 Jul 2024 15:22:48 +0800 Subject: [PATCH 22/41] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E9=99=90?= =?UTF-8?q?=E6=B5=81=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/result/ApiResult.java | 9 + .../foundation/redis/support/RateLimiter.java | 116 ++++++++ .../redis/support/RateLimiterFactory.java | 32 +++ .../redis/support/impl/LocalRateLimiter.java | 66 +++++ .../support/impl/RateLimiterFactoryImpl.java | 36 +++ .../redis/support/impl/RedisRateLimiter.java | 259 ++++++++++++++++++ 6 files changed, 518 insertions(+) create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiter.java create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiterFactory.java create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/LocalRateLimiter.java create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RateLimiterFactoryImpl.java create mode 100644 redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisRateLimiter.java diff --git a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java index 9e175ad..9696d5c 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java +++ b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java @@ -1,5 +1,6 @@ package cn.axzo.foundation.result; +import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Strings; import lombok.AllArgsConstructor; @@ -26,6 +27,8 @@ public class ApiResult { protected T data; + protected JSONObject stacks; + public static ApiResult success() { return success(null); } @@ -71,6 +74,12 @@ public class ApiResult { return SUCCESS_CODE.equals(getCode()); } + + public ApiResult setStacks(JSONObject stacks) { + this.stacks = stacks; + return this; + } + /** * 根据appId 获取标准的code * 如果code > 100000 则认为可能已经带了appId diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiter.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiter.java new file mode 100644 index 0000000..ce10452 --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiter.java @@ -0,0 +1,116 @@ +package cn.axzo.foundation.redis.support; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import lombok.*; + +import java.util.*; +import java.util.stream.Collectors; + +public interface RateLimiter { + /** + * 尝试获得锁, 获取失败则返回Optional.empty() + * 如果获取锁成功. 则返回Optional. 同时计数器增加 + * Permit支持取消 + * + * @param value 业务标识 + * @param step 指定步长 + * @return + */ + Optional tryAcquire(Object value, long step); + + default Optional tryAcquire(Object value) { + return tryAcquire(value, 1); + } + + /** + * 重置value对应的锁, 便于特殊场景下重新获取锁 + */ + void reset(Object value); + + /** + * 获取窗口类型 + * + * @return + */ + WindowType getWindowType(); + + class Permit { + private List cancelRunners; + + @Builder + public Permit(List cancelRunners) { + Objects.requireNonNull(cancelRunners); + this.cancelRunners = cancelRunners; + } + + public void cancel() { + if (!cancelRunners.isEmpty()) { + cancelRunners.stream().forEach(e -> e.run()); + } + } + } + + @Getter + @AllArgsConstructor + enum WindowType { + /** + * 固定窗口, 窗口范围: start = visitTim, end = start + WindowDuration + */ + FIXED("f"), + /** + * 固定窗口, 窗口范围: start = currentMillis/WindowDuration, end = currentMillis/WindowDuration + */ + FIXED_BUCKET("fb"), + /** + * 滑动窗口, 窗口范围: start = currentMillis - WindowDuration, end = currentMillis + */ + SLIDING("s"); + + //减少redisKey长度 + private final String shortName; + } + + /** + * 限流规则 + * 
+     *     seconds: 窗口时长
+     *     permits: 允许发放的令牌数量
+     *
+ */ + @Data + @Builder + @AllArgsConstructor + @ToString + class LimitRule { + long seconds; + int permits; + + public boolean isValid() { + return seconds > 0 && permits > 0; + } + + /** + * 根据约定规则创建Rules + * eg: express 10/1,20/1... 代表10秒1次 & 20秒1次... + * note: seconds不可重复 + */ + public static List fromExpression(String expression) { + if (Strings.isNullOrEmpty(expression)) { + return Collections.emptyList(); + } + Map rulesMap = Splitter.on(",") + .omitEmptyStrings() + .trimResults() + .withKeyValueSeparator("/") + .split(expression); + + return rulesMap.entrySet().stream() + .map(e -> LimitRule.builder() + .seconds(Long.parseLong(e.getKey())) + .permits(Integer.parseInt(e.getValue())) + .build()) + .collect(Collectors.toList()); + } + } +} diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiterFactory.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiterFactory.java new file mode 100644 index 0000000..91c6303 --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/RateLimiterFactory.java @@ -0,0 +1,32 @@ +package cn.axzo.foundation.redis.support; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +public interface RateLimiterFactory { + + /** + * 构建一个基于RateLimiter + */ + RateLimiter build(RateLimiterReq rateLimiterReq); + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + class RateLimiterReq { + RateLimiter.WindowType windowType; + List rules; + String limiterKey; + RateType rateType; + } + + enum RateType { + LOCAL, + REDIS + } +} diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/LocalRateLimiter.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/LocalRateLimiter.java new file mode 100644 index 0000000..dcf2b25 --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/LocalRateLimiter.java @@ -0,0 +1,66 @@ +package cn.axzo.foundation.redis.support.impl; + +import cn.axzo.foundation.redis.support.RateLimiter; +import cn.axzo.foundation.redis.support.RateLimiterFactory; +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multimap; +import com.google.common.hash.Hashing; + +import java.nio.charset.Charset; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +public class LocalRateLimiter implements RateLimiter { + + private RateLimiterFactory.RateLimiterReq rateLimiterReq; + + /** + * 使用 static 保持访问的状态 + */ + static private Multimap accessLogs = ArrayListMultimap.create(); + + public LocalRateLimiter(RateLimiterFactory.RateLimiterReq rateLimiterReq) { + this.rateLimiterReq = rateLimiterReq; + } + + public void cleanAccessLogs() { + accessLogs = ArrayListMultimap.create(); + } + + @Override + public Optional tryAcquire(Object value, long step) { + Preconditions.checkArgument(value != null); + + String hash = Hashing.murmur3_32().hashString(rateLimiterReq.getLimiterKey() + String.valueOf(value), Charset.defaultCharset()).toString(); + Long nowSec = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()); + accessLogs.put(hash, nowSec); + + List accessTimeSecs = ImmutableList.copyOf(accessLogs.get(hash)); + List rules = rateLimiterReq.getRules(); + boolean failed = rules.stream().anyMatch(rule -> { + if (rateLimiterReq.getWindowType() == WindowType.SLIDING) { + return accessTimeSecs.stream().filter(t -> t > (nowSec - rule.getSeconds())).count() > rule.getPermits(); + } else { + return accessTimeSecs.stream().filter(t -> ((nowSec - t) / rule.getSeconds()) == 0).count() > rule.getPermits(); + } + }); + if (failed) { + return Optional.empty(); + } + return Optional.of(Permit.builder().cancelRunners(ImmutableList.of()).build()); + } + + @Override + public void reset(Object value) { + String hash = Hashing.murmur3_32().hashString(rateLimiterReq.getLimiterKey() + String.valueOf(value), Charset.defaultCharset()).toString(); + accessLogs.removeAll(hash); + } + + @Override + public WindowType getWindowType() { + return rateLimiterReq.getWindowType(); + } +} \ No newline at end of file diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RateLimiterFactoryImpl.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RateLimiterFactoryImpl.java new file mode 100644 index 0000000..07815af --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RateLimiterFactoryImpl.java @@ -0,0 +1,36 @@ +package cn.axzo.foundation.redis.support.impl; + +import cn.axzo.foundation.redis.support.RateLimiter; +import cn.axzo.foundation.redis.support.RateLimiterFactory; +import cn.axzo.foundation.web.support.AppRuntime; +import lombok.Builder; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.Objects; + +public class RateLimiterFactoryImpl implements RateLimiterFactory { + private RedisTemplate redisTemplate; + private AppRuntime appRuntime; + + @Builder + public RateLimiterFactoryImpl(RedisTemplate redisTemplate, AppRuntime appRuntime) { + this.redisTemplate = redisTemplate; + this.appRuntime = appRuntime; + } + + @Override + public RateLimiter build(RateLimiterFactory.RateLimiterReq rateLimiterReq) { + if (rateLimiterReq.getRateType() == RateType.LOCAL) { + return new LocalRateLimiter(rateLimiterReq); + } + Objects.requireNonNull(redisTemplate); + Objects.requireNonNull(appRuntime); + return RedisRateLimiter.builder() + .windowType(rateLimiterReq.getWindowType()) + .limitRules(rateLimiterReq.getRules()) + .limiterKey(rateLimiterReq.getLimiterKey()) + .redisTemplate(redisTemplate) + .appRuntime(appRuntime) + .build(); + } +} diff --git a/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisRateLimiter.java b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisRateLimiter.java new file mode 100644 index 0000000..a37b177 --- /dev/null +++ b/redis-support-lib/src/main/java/cn/axzo/foundation/redis/support/impl/RedisRateLimiter.java @@ -0,0 +1,259 @@ +package cn.axzo.foundation.redis.support.impl; + +import cn.axzo.foundation.redis.support.RateLimiter; +import cn.axzo.foundation.web.support.AppRuntime; +import cn.axzo.foundation.web.support.utils.KeyBuilder; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.hash.Hashing; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.BoundValueOperations; +import org.springframework.data.redis.core.BoundZSetOperations; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +@Slf4j +public class RedisRateLimiter implements RateLimiter { + private AppRuntime appRuntime; + private RedisTemplate redisTemplate; + private RateLimiterWorker rateLimiterWorker; + /** + * 自定义的key, 避免redisKey冲突. 必填 + */ + private String limiterKey; + private List limitRules; + /** + * 窗口保存最大时长. 主要针对Sliding方式窗口的zSet过期 + */ + private Integer maxWindowDurationHour; + private WindowType windowType; + + @Builder + RedisRateLimiter(AppRuntime appRuntime, + RedisTemplate redisTemplate, + WindowType windowType, + String limiterKey, + List limitRules, + Integer maxWindowDurationHour) { + Objects.requireNonNull(appRuntime); + Objects.requireNonNull(redisTemplate); + Objects.requireNonNull(windowType); + Objects.requireNonNull(limitRules); + Objects.requireNonNull(limiterKey); + Preconditions.checkArgument(!limitRules.isEmpty()); + if (limitRules.stream().anyMatch(p -> !p.isValid())) { + throw new RuntimeException(String.format("invalid rate expression, expression = %s", JSONObject.toJSONString(limitRules))); + } + + this.windowType = windowType; + this.appRuntime = appRuntime; + this.redisTemplate = redisTemplate; + this.limitRules = limitRules; + this.limiterKey = limiterKey; + + this.maxWindowDurationHour = Optional.ofNullable(maxWindowDurationHour).orElse(24); + + this.rateLimiterWorker = buildWorker(windowType); + } + + @Override + public Optional tryAcquire(Object value, long step) { + if (!rateLimiterWorker.tryAcquire(value)) { + return Optional.empty(); + } + List cancelRunners = rateLimiterWorker.visit(value, step); + return Optional.of(Permit.builder() + .cancelRunners(cancelRunners) + .build()); + } + + @Override + public void reset(Object value) { + rateLimiterWorker.reset(value); + } + + @Override + public WindowType getWindowType() { + return windowType; + } + + private RateLimiterWorker buildWorker(WindowType windowType) { + if (windowType == WindowType.FIXED) { + return new FixedWindowRateLimiter(); + } + if (windowType == WindowType.FIXED_BUCKET) { + return new FixedBucketWindowRateLimiter(); + } + if (windowType == WindowType.SLIDING) { + return new SlidingWindowRateLimiter(); + } + throw new RuntimeException(String.format("unsupported window type, window type = %s", windowType)); + } + + private String buildRedisKey(Object value) { + String hash = Hashing.murmur3_128().newHasher() + .putString(limiterKey, Charsets.UTF_8) + .putString(String.valueOf(value), Charsets.UTF_8) + .hash() + .toString(); + return KeyBuilder.build(appRuntime, "rl", getWindowType().getShortName(), limiterKey, hash); + } + + /** + * 固定窗口限流, 窗口起始时间第一次tryAcquire时时间. 窗口大小为WindowDuration + * 
+     *     key = value + WindowDuration.
+     *     在该窗口被访问时, 计数器+1. 窗口持续时长为WindowDuration. 并依赖redis ttl销毁
+     *     窗口被销毁后, 重置计数器
+     *
+ */ + class FixedWindowRateLimiter implements RateLimiterWorker { + public List visit(Object value, long step) { + List cancels = new ArrayList<>(limitRules.size()); + //根据时间构建key, 避免hash无法删除 + limitRules.stream() + .forEach(e -> { + String key = buildLimiterKey(value, e); + BoundValueOperations op = redisTemplate.boundValueOps(key); + Long result = op.increment(step); + //第一次访问时设置过期时间, 以确定窗口 + if (result == step) { + redisTemplate.expire(key, e.getSeconds(), TimeUnit.SECONDS); + } + cancels.add(() -> op.decrement(step)); + }); + return cancels; + } + + @Override + public void reset(Object value) { + limitRules.stream().forEach(e -> { + String key = buildLimiterKey(value, e); + redisTemplate.delete(key); + }); + } + + public boolean tryAcquire(Object value) { + boolean anyMatch = limitRules.stream() + .anyMatch(limitRule -> { + String key = buildLimiterKey(value, limitRule); + return limitRule.getPermits() <= Optional.ofNullable(redisTemplate.opsForValue().get(key)) + .map(e -> Long.parseLong(e.toString())) + .orElse(0L); + }); + return !anyMatch; + } + + protected String buildLimiterKey(Object value, LimitRule limitRule) { + return buildRedisKey(value + ":" + limitRule.getSeconds()); + } + } + + /** + * 滑动窗口限流, 每次获取令牌成功时间加入到zset. 后续获取令牌时每次检查zset中WindowDuration中已获取令牌数. 并判断是否可以继续获取令牌 + * 
+     *     key = value
+     *     zset value = currentMillis. score = currentMillis
+     *     获取令牌时, 在计算zset中 score = [currentMillis-WindowDuration, currentMillis} 的element数量
+     *
+ */ + class SlidingWindowRateLimiter implements RateLimiterWorker { + //当zset的element达到一定数量时, 清理该zet. 避免redis内存泄露 + private static final int CLEAN_KEY_THRESHOLD = 1000; + private AtomicLong visitCounter = new AtomicLong(); + + public List visit(Object value, long step) { + Preconditions.checkArgument(step == 1, "滑动窗口只支持 step=1"); + String key = buildRedisKey(value); + long now = System.currentTimeMillis(); + String member = String.valueOf(now); + final BoundZSetOperations op = redisTemplate.boundZSetOps(key); + op.add(member, now); + + redisTemplate.expire(key, maxWindowDurationHour, TimeUnit.HOURS); + if (visitCounter.incrementAndGet() > CLEAN_KEY_THRESHOLD) { + //删除过期的访问记录 + op.removeRangeByScore(0, now - TimeUnit.HOURS.toMillis(maxWindowDurationHour)); + visitCounter.set(0); + } + return ImmutableList.of(() -> op.remove(member)); + } + + @Override + public void reset(Object value) { + String key = buildRedisKey(value); + redisTemplate.delete(key); + } + + public boolean tryAcquire(Object value) { + String key = buildRedisKey(value); + long now = System.currentTimeMillis(); + + //检查所有的rule, 如果有其中一个失败则跳出 + return !limitRules.stream() + .anyMatch(p -> p.getPermits() <= redisTemplate.opsForZSet().count(key, now - TimeUnit.SECONDS.toMillis(p.getSeconds()), now)); + } + } + + /** + * 固定窗口限流, 窗口根据自然时间向前滚动 + * 继承自FixedWindowRateLimiter, 区别在于构建key的方式不一样 + * 
+     *     key = value + currentMillis/WindowDuration.
+     *     currentMillis/WindowDuration会把自然时间分割为长为WindowDuration的片段. 片段有效期为WindowDuration
+     *     获取令牌时在该片段上检查是否有剩余令牌
+     *
+ */ + class FixedBucketWindowRateLimiter extends FixedWindowRateLimiter implements RateLimiterWorker { + @Override + protected String buildLimiterKey(Object value, LimitRule limitRule) { + // System.currentTimeMillis()是当前距离1970-01-01 08:00:00的毫秒数（假设系统为+8时区） + // 这里希望是距离1970-01-01 00:00:00（GMT标准时间）的毫秒数，所以需要+8个小时的毫秒 + long currentGMTMillis = System.currentTimeMillis() + TimeUnit.HOURS.toMillis(8); + return buildRedisKey(value + ":" + currentGMTMillis / TimeUnit.SECONDS.toMillis(limitRule.getSeconds())); + } + } + + interface RateLimiterWorker { + /** + * 尝试获取令牌 + * + * @param value + * @return 如果获取成功则返回true, 失败则为false + */ + boolean tryAcquire(Object value); + + /** + * 获取令牌完成后增加步长为 1的计数器 + * + * @param value + * @return limit key + */ + default List visit(Object value) { + return visit(value, 1); + } + + /** + * 获取令牌完成后增加指定步长的计数器 + * + * @param value + * @param step + * @return limit key + */ + List visit(Object value, long step); + + /** + * 重置value对应的锁 + */ + void reset(Object value); + } +} From d4f01bb227ad884775d043e2c3b081c891eabc0d Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 5 Jul 2024 16:17:28 +0800 Subject: [PATCH 23/41] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=AF=BC=E5=85=A5/=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/axzo/foundation/page/PageReq.java | 18 + excel-support-lib/pom.xml | 44 + .../excel/support/DataSheetClient.java | 918 ++++++++++++++++++ .../support/impl/DataSheetAsyncExporter.java | 326 +++++++ .../support/impl/DataSheetClientImpl.java | 90 ++ .../excel/support/impl/DataSheetExporter.java | 698 +++++++++++++ .../excel/support/impl/DataSheetImporter.java | 595 ++++++++++++ pom.xml | 1 + 8 files changed, 2690 insertions(+) create mode 100644 common-lib/src/main/java/cn/axzo/foundation/page/PageReq.java create mode 100644 excel-support-lib/pom.xml create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/DataSheetClient.java create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetAsyncExporter.java create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetClientImpl.java create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetExporter.java create mode 100644 excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetImporter.java diff --git a/common-lib/src/main/java/cn/axzo/foundation/page/PageReq.java b/common-lib/src/main/java/cn/axzo/foundation/page/PageReq.java new file mode 100644 index 0000000..7dfbc4c --- /dev/null +++ b/common-lib/src/main/java/cn/axzo/foundation/page/PageReq.java @@ -0,0 +1,18 @@ +package cn.axzo.foundation.page; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PageReq implements IPageReq{ + Integer page; + Integer pageSize; + List sort; +} diff --git a/excel-support-lib/pom.xml b/excel-support-lib/pom.xml new file mode 100644 index 0000000..62c6fac --- /dev/null +++ b/excel-support-lib/pom.xml @@ -0,0 +1,44 @@ + + + 4.0.0 + + cn.axzo.foundation + axzo-lib-box + 2.0.0-SNAPSHOT + + + cn.axzo.maokai + excel-support-lib + + + 8 + 8 + UTF-8 + + + + + cn.axzo.foundation + web-support-lib + 2.0.0-SNAPSHOT + + + com.alibaba + easyexcel + 3.3.4 + + + com.opencsv + opencsv + 5.9 + + + org.springframework.boot + spring-boot-starter-data-redis + provided + + + + \ No newline at end of file diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/DataSheetClient.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/DataSheetClient.java new file mode 100644 index 0000000..989c923 --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/DataSheetClient.java @@ -0,0 +1,918 @@ +package cn.axzo.foundation.excel.support; + +import cn.axzo.foundation.exception.BusinessException; +import cn.axzo.foundation.page.IPageReq; +import cn.axzo.foundation.page.PageResp; +import cn.axzo.foundation.util.VarParamFormatter; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSONPath; +import com.alibaba.fastjson.parser.ParserConfig; +import com.alibaba.fastjson.util.TypeUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import lombok.*; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.poi.ss.SpreadsheetVersion; + +import javax.servlet.http.HttpServletResponse; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*; + +/** + * 用于发送sms & email验证码, 或者获取图片验证码. 并提供校验验证码是否正确的能力 + */ +public interface DataSheetClient { + + ExporterBuilder exportBuilder(); + + /** + * 数据导入的builder + * + * @return + */ + ImporterBuilder importBuilder(); + + @Data + @NoArgsConstructor + @AllArgsConstructor + @Accessors(fluent = true) + abstract class ImporterBuilder { + /** + * 导入的场景名称, 必填 + */ + @NonNull + private String scene; + + /** + * 导入格式 + */ + @NonNull + private ImportFormat format; + + /** + * EXCEL格式的sheetName + */ + private String tableName; + + private Meta meta; + + /** + * 是否忽略导入文件表头的原meta + * 默认按false处理，即{@link #meta}为空时尝试使用原文件表头meta + */ + private Boolean ignoreHeaderMeta; + + private boolean debugEnabled; + + /** + * 忽略未知的列(cellMeta中未定义的列) + */ + private boolean ignoreUnknownColumn; + + /** + * 是否自动清除表格内容的前后空白字符 + * 默认true，即会清除前后空白字符 + */ + private Boolean autoTrim; + + /** + * 解析结果ImportResp中的lines是否包含解析失败的行 + * true: 解析失败的时候，不会抛异常，会在每行的JSONObject中添加 + * "errors":[{"columnIndex":1,"columnKey":"","columnName":"","rawValue":"","errorCode":"","errorMsg":""}] + * false: 解析失败的时候，抛第一个异常 + */ + private boolean includeLineErrors; + + /** + * 允许最大的导入行数 + */ + private Integer allowMaxLineCount; + + /** 将excel中的表头转换为cellMeta中的表头 */ + private Function headerConverter; + + /** + * 操作人, 可选，用于导入统计报告 + */ + private String operator; + + public abstract Importer build(); + } + + interface Importer { + /** + * 所有字段读取为String类型 + * + * @param inputStream + * @return + */ + ImportResp readAll(InputStream inputStream); + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class ImportResp { + /** + * 导入的场景名称 + */ + private String scene; + + /** + * 模版对应的code以及版本 + */ + private String templateCode; + private String version; + + /** + * headers + */ + private List headers; + /** + * 每一行的数据 + */ + private List lines; + + /** + * header的行数 + */ + private Integer headerRowCount; + /** + * 导入数据总行数, 不包含header + */ + private Integer rowCount; + /** + * 导入数据列数 + */ + private Integer columnCount; + + /** + * Meta数据 + */ + private Meta meta; + + /** + * 消耗的时间 + */ + private Long elapsedMillis; + + /** + * 操作人 + */ + private String operator; + } + + @AllArgsConstructor + @Getter + enum ImportFormat { + EXCEL("xlsx"), + // TODO: 支持其他类型 + // CSV("csv") + ; + + private String suffix; + } + + @Data + @NoArgsConstructor + @Accessors(fluent = true) + abstract class ExporterBuilder { + /** + * 导出的场景名称, 必填 + */ + @NonNull + private String scene; + + /** + * 导出格式 + */ + @NonNull + private ExportFormat format; + + /** + * 顶部提示内容，主要用于一些提示场景。如导出数据超范围等文案提示等，支持多行。 + */ + private Function, List> topHintsSupplier; + + /** + * 提供分页获取数据的方法 + */ + @NonNull + private Function> rowSupplier; + + /** + * 导出的总数据量限制，不指定时，按照分页返回的结果全量导出 + */ + private Long limit = Long.MAX_VALUE; + + /** + * 获取数据的分页大小, 默认每页200条(MybatisPlus的PaginationInterceptor默认limit为500) + */ + private Long pageSize = 200L; + + /** + * 表名，仅当导出格式为excel时有效 + */ + private String sheetName; + + @Deprecated + public ExporterBuilder tableName(String tableName) { + this.sheetName = tableName; + return this; + } + + /** + * 列映射表, key=列名， value=获取字段值的方法, 方法参数1为当前row的JSONObject, 参数2为当前row的index，返回列字段值 + */ + private ImmutableMap> columnMap; + + /** + * 多行表头映射表, 当表头由多行数据组成时使用. key=表头, value=该表头对应的每行的内容 + */ + private ImmutableMap> multiLineHeadMap; + + /** + * 行数据的转换器，外部可以通过它将原始的行数据根据需要做转换 + */ + private Function> rowConverter; + + /** + * 页数据的转换器，外部可以通过它将也的数据根据需要做转换 + * 它会在 rowconvert() 执行完成 + */ + private Function, List> pageConverter; + + private BiConsumer onProgress; + + /** + * 批注信息，可选 + */ + private Meta meta; + + /** + * 是否开启调试日志 + */ + private Boolean debugEnabled = true; + + /** + * 导出的fields. 优先使用fields来创建column&meta + */ + private List fields; + + /** + * meta的version + */ + private String version; + + /** 文件名 */ + private String fileName; + + /** 操作人, 可选，用于异步导出 */ + private String operator; + + /** 拓展字段，可用于存放请求参数等，该字段会回传给FileLoaderParam */ + private JSONObject ext; + + /** + * 仅对导出excel生效，是否自动清除表格内容的前后空白字符 + * 默认true，即会清除前后空白字符 + */ + private Boolean autoTrim; + + abstract public Exporter build(); + } + + @Data + class ExportField { + private String column; + private String header; + private List headerLines; + private Integer width; + private CellMeta.Type type; + private Boolean mandatory; + private Boolean wrapText; + + private String lowerType; + private String upperType; + private String dateTimeFormat; + private List options; + + private BiFunction reader; + + @Builder + public ExportField(String column, String header, List headerLines, Integer width, CellMeta.Type type, + Boolean mandatory, Boolean wrapText, Function resultConverter, + BiFunction reader, List options) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(column), "column不能为空"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(header), "header不能为空"); + Preconditions.checkArgument(!(resultConverter != null && reader != null), "reader & resultConverter不能同时存在"); + + this.column = column; + this.header = header; + this.headerLines = headerLines; + this.width = width; + this.type = Optional.ofNullable(type).orElse(CellMeta.Type.STRING); + this.mandatory = Optional.ofNullable(mandatory).orElse(Boolean.FALSE); + this.wrapText = wrapText; + if (reader != null) { + this.reader = reader; + } else { + this.reader = Optional.ofNullable(resultConverter) + .map(e -> (BiFunction) (row, integer) -> e.apply(Strings.nullToEmpty(row.getString(column)))) + .orElseGet(() -> DataSheetClient.stringCellReader(column)); + } + this.options = options; + } + + public CellMeta toCellMeta() { + JSONObject params = new JSONObject(); + if (CellMeta.Type.RANGE == type) { + params.put(EXT_KEY_RANGE_LOWER_TYPE, lowerType); + params.put(EXT_KEY_RANGE_UPPER_TYPE, upperType); + } + if (CellMeta.Type.DATE == type || CellMeta.Type.DATETIME == type) { + params.put(EXT_KEY_DATETIME_FORMAT, dateTimeFormat); + } + + return CellMeta.builder() + .key(column).name(header).width(width).mandatory(mandatory).type(type).wrapText(wrapText) + .params(params).options(options).build(); + } + } + + interface Exporter { + /** + * 导出到OutputStream中 + * + * @param outputStream + * @return + */ + ExportResp export(@NonNull OutputStream outputStream); + + /** + * 导出到HttpServletResponse中 + * + * @param response + * @return + */ + ExportResp export(@NonNull HttpServletResponse response); + + /** + * 导出到HttpServletResponse中，可指定导出的文件名 + * + * @param response + * @return + */ + ExportResp export(HttpServletResponse response, String filename); + } + + interface AsyncExporter { + /** + * 异步导出数据 + * + * @param dataSheetExporter 实现导出逻辑的 exporter + * @return batchId + */ + String export(@NonNull ExporterBuilder dataSheetExporter); + + /** + * 多表单数据异步导出至同一文件中, 仅支持导出格式是EXCEL + * + * @param fileName 导出文件名 + * @param dataSheetExporters 实现导出逻辑的 exporter 列表 + * @return batchId + */ + String export(String fileName, @NonNull List dataSheetExporters); + + Progress getProgress(@NonNull String batchId); + + @NoArgsConstructor + @AllArgsConstructor + @Data + @Builder + class Progress { + String fileName; + int progress; + String errorMsg; + String downloadUrl; + + public boolean isSuccess() { + return Strings.isNullOrEmpty(errorMsg); + } + + public boolean isDone() { + return progress < 0 || progress >= 100; + } + } + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + class ExportResp { + /** + * 导出场景 + */ + private String scene; + /** + * 导出数据条数 + */ + private Long exportedCount; + + /** + * 导出数据总行数，包含了顶部提示行及表头行 + */ + private Long rowCount; + + /** + * 导出数据列数 + */ + private Long columnCount; + + /** + * 导出数据行数 + */ + private Long dataRowCount; + + /** + * 耗时 + */ + private Long elapsedMillis; + + /** + * 操作人 + */ + private String operator; + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class Meta { + public static final String PATTERN_VERSION_OPEN = "$V{"; + public static final String PATTERN_CELL_OPEN = "$C{"; + public static final String PATTERN_IGNORE_OPEN = "$I{"; + public static final String PATTERN_EXTRA_OPEN = "$E{"; + public static final String PATTERN_CLOSE = "}"; + + public static final String RANGE_TYPE_OPEN_OPEN = "1"; + public static final String RANGE_TYPE_OPEN_CLOSED = "2"; + public static final String RANGE_TYPE_CLOSED_OPEN = "3"; + public static final String RANGE_TYPE_CLOSED_CLOSED = "4"; + + public static final String IGNORE_ROW_KEY = "r"; + public static final String IGNORE_COLUMN_KEY = "c"; + public static final String IGNORE_ROW_AND_COLUMN_KEY = "b"; + + private String templateCode; + private String version; + private List cellMetas; + /** + * 忽略掉的行号 + */ + private List ignoreRowIndexes; + /** + * 忽略掉的列号 + */ + private List ignoreColumnIndexes; + /** + * 忽略掉的列名 + * 当导入文件中有meta信息时，该ignore信息将会丢失 + */ + private Set ignoreColumnNames; + + public List getIgnoreRowIndexes() { + return Optional.ofNullable(ignoreRowIndexes).orElseGet(ImmutableList::of); + } + + public List getIgnoreColumnIndexes() { + return Optional.ofNullable(ignoreColumnIndexes).orElseGet(ImmutableList::of); + } + + public Set getIgnoreColumnNames() { + return Optional.ofNullable(ignoreColumnNames).orElseGet(ImmutableSet::of); + } + } + + @SuperBuilder + @Data + @NoArgsConstructor + @AllArgsConstructor + class CellMeta { + public static final String EXT_KEY_DATETIME_FORMAT = "dateTimeFormat"; + public static final String EXT_KEY_RANGE_LOWER_TYPE = "lowerType"; + public static final String EXT_KEY_RANGE_UPPER_TYPE = "upperType"; + + public static final String RANGE_BOUND_TYPE_OPEN = "open"; + public static final String RANGE_BOUND_TYPE_CLOSED = "closed"; + public static final Set RANGE_TYPES = ImmutableSet.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED); + + private static final List DEFAULT_DATE_FORMATS = ImmutableList.of( + "yyyy-MM-dd", + "yyyy-MM-dd HH:mm", + "yyyy-MM-dd HH:mm:ss", + "yyyy.MM.dd", + "yyyy.MM.dd HH:mm", + "yyyy.MM.dd HH:mm:ss", + "yyyy年MM月dd日", + "yyyy年MM月dd日 HH:mm", + "yyyy年MM月dd日 HH:mm:ss", + "yyyy年MM月dd日 HH时mm分", + "yyyy年MM月dd日 HH时mm分ss秒", + "yyyy/MM/dd", + "yyyy/MM/dd HH:mm", + "yyyy/MM/dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss'Z'", + "yyyy-MM-dd'T'HH:mm:ssZ", + "yyyy-MM-dd'T'HH:mm:ssz", + "yyyy-MM-dd'T'HH:mm:ss", + "MM/dd/yyyy HH:mm:ss a", + "yyyyMMddHHmmss", + "yyyyMMdd" + ); + + private static final Map BOOLEAN_MAP = ImmutableMap.builder() + .put("是", Boolean.TRUE) + .put("yes", Boolean.TRUE) + .put("y", Boolean.TRUE) + .put("1", Boolean.TRUE) + .put("否", Boolean.FALSE) + .put("no", Boolean.FALSE) + .put("n", Boolean.FALSE) + .put("0", Boolean.FALSE) + .build(); + + /** + * cell的key + */ + private String key; + /** + * cell的header，一般为中文 + */ + private String name; + /** + * cell的宽度 + */ + private Integer width; + /** + * 类型 + */ + private Type type; + /** + * 是否必需, 为false时, 允许该列在Excel中不存在 + */ + private Boolean mandatory; + /** + * 可选值 + */ + private List options; + /** + * 存放一些和type相关的参数，例如DateTime的pattern格式，Range的开闭区间 + */ + private JSONObject params; + /** + * 存放一些额外的信息 + */ + private JSONObject ext; + + /** + * wrap the text automatically + */ + private Boolean wrapText; + + /** + * 导入的时候，将原始的String转换为特定的类型 + */ + private Function importConverter; + + /** + * 字段值额外的校验方法，支持根据当前字段值及整行数据进行自定义校验 + */ + private BiConsumer cellValidator; + + public void validate() { + if (type == Type.RANGE + && (!RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_LOWER_TYPE)) + || !RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_UPPER_TYPE)))) { + throw ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR.toException(); + } + } + + public Object convertType(String value) { + if (importConverter != null) { + return importConverter.apply(value); + } + + if (Boolean.TRUE.equals(mandatory) && Strings.isNullOrEmpty(value)) { + throw ResultCode.IMPORT_CELL_META_MISSING_MANDATORY_VALUE.toException(); + } + // 非必填的字段如果为空，直接返回 + if (value == null) { + return type == Type.STRING ? StringUtils.EMPTY : null; + } + + switch (type) { + case INT: + // 移除千分符 + return Integer.valueOf(value.replace(",", "")); + case LONG: + // 移除千分符 + return Long.valueOf(value.replace(",", "")); + case FLOAT: + // 移除千分符 + return Float.valueOf(value.replace(",", "")); + case DOUBLE: + // 移除千分符 + return Double.valueOf(value.replace(",", "")); + case BIG_DECIMAL: + // 移除千分符 + return new BigDecimal(value.replace(",", "")); + case DATE: + case DATETIME: + return parseDate(value, DEFAULT_DATE_FORMATS); + case RANGE: + // 格式 100-200 + List values = Splitter.on("-").omitEmptyStrings().trimResults().splitToList(value) + .stream() + .map(Integer::valueOf) + .collect(Collectors.toList()); + if (values.size() != 2) { + throw ResultCode.IMPORT_CELL_RANGE_FORMAT_ERROR.toException(); + } + if (values.get(0) >= values.get(1)) { + throw ResultCode.IMPORT_CELL_RANGE_VALUE_ERROR.toException(); + } + return new JSONObject() + .fluentPut("lower", values.get(0)) + .fluentPut("upper", values.get(1)) + .fluentPut(EXT_KEY_RANGE_LOWER_TYPE, params.getString(EXT_KEY_RANGE_LOWER_TYPE)) + .fluentPut(EXT_KEY_RANGE_UPPER_TYPE, params.getString(EXT_KEY_RANGE_UPPER_TYPE)); + case BOOLEAN: + return Optional.ofNullable(BOOLEAN_MAP.get(value.toLowerCase())) + .orElseThrow(ResultCode.IMPORT_CELL_BOOLEAN_VALUE_ERROR::toException); + default: + return Strings.nullToEmpty(value); + } + } + + private LocalDateTime parseDate(String dateStr, List formats) { + try { + Date date = DateUtils.parseDate(dateStr, formats.toArray(new String[0])); + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } catch (Exception e) { + throw ResultCode.IMPORT_CELL_DATETIME_CONVERT_FAILED.toException("不支持的日期格式{}", dateStr); + } + } + + @AllArgsConstructor + public enum Type { + INT("00"), + LONG("01"), + FLOAT("02"), + DOUBLE("03"), + STRING("04"), + BIG_DECIMAL("05"), + DATE("06"), + DATETIME("07"), + RANGE("08"), + BOOLEAN("09"); + + @Getter + private String code; + + private static Map map = Stream.of(Type.values()) + .collect(Collectors.toMap(Type::getCode, Function.identity())); + + public static Type from(String code) { + return map.get(code); + } + } + } + + @AllArgsConstructor + @Getter + enum ExportFormat { + EXCEL("application/vnd.ms-excel", ".xlsx"), + CSV("text/csv", ".csv"); + + private String contentType; + private String suffix; + } + + static BiFunction indexCellReader() { + return (row, index) -> String.valueOf(index + 1); + } + + static BiFunction stringCellReader(String columnName) { + // excel库poi会检查文本格式的单元格内容，当超过32767时，将会抛出IllegalArgumentException异常 + // 参考 {@link org.apache.poi.xssf.streaming.SXSSFCell.setCellValue} + return (row, index) -> StringUtils.left(Strings.nullToEmpty(row.getString(columnName)), + SpreadsheetVersion.EXCEL2007.getMaxTextLength()); + } + + static BiFunction dateTimeCellReader(String columnName, String pattern) { + return (row, index) -> { + Long epochMillis = row.getLong(columnName); + if (epochMillis == null) { + return StringUtils.EMPTY; + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); + Instant instant = Instant.ofEpochMilli(epochMillis); + LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); + return formatter.format(localDateTime); + }; + } + + static BiFunction bigDecimalCellReader(String columnName) { + return bigDecimalCellReader(columnName, 2); + } + + static BiFunction bigDecimalCellReader(String columnName, int scale) { + return (row, index) -> { + BigDecimal number = row.getBigDecimal(columnName); + if (number == null) { + number = BigDecimal.ZERO; + } + + return number.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString(); + }; + } + + static BiFunction jsonPathCellReader(String jsonPath) { + return (row, index) -> { + Object obj = JSONPath.eval(row, jsonPath); + /** + * excel单元格对于数值最大只能保有15位，超过15位的数值(Long)会被置为0，比如在导出uid的场景，会导致数据错误 + * https://docs.microsoft.com/en-us/office/troubleshoot/excel/last-digits-changed-to-zeros + * https://wenku.baidu.com/view/e1d2a497f624ccbff121dd36a32d7375a417c6ad.html + */ + if (obj instanceof Long && obj.toString().length() > 15) { + obj = obj.toString(); + } + if (obj instanceof String) { + // excel库poi会检查文本格式的单元格内容，当超过32767时，将会抛出IllegalArgumentException异常 + // 参考 {@link org.apache.poi.xssf.streaming.SXSSFCell.setCellValue} + return StringUtils.left(obj.toString(), SpreadsheetVersion.EXCEL2007.getMaxTextLength()); + } + return obj; + }; + } + + static BiFunction jsonPathDateTimeCellReader(String jsonPath, String pattern) { + return (row, index) -> { + Object val = JSONPath.eval(row, jsonPath); + Long epochMillis = TypeUtils.cast(val, Long.class, ParserConfig.getGlobalInstance()); + if (epochMillis == null) { + return StringUtils.EMPTY; + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); + Instant instant = Instant.ofEpochMilli(epochMillis); + LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); + return formatter.format(localDateTime); + }; + } + + static BiFunction jsonPathBigDecimalCellReader(String jsonPath) { + return jsonPathBigDecimalCellReader(jsonPath, 2); + } + + static BiFunction jsonPathBigDecimalCellReader(String jsonPath, int scale) { + return (row, index) -> { + Object val = JSONPath.eval(row, jsonPath); + if (val == null) { + val = BigDecimal.ZERO; + } + BigDecimal bigDecimal = TypeUtils.cast(val, BigDecimal.class, ParserConfig.getGlobalInstance()); + return bigDecimal.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString(); + }; + } + + @Getter + class DataSheetException extends BusinessException { + private String subErrorCode; + /** 异常相关的行号，列号，0开始 */ + private Integer rowIndex; + private Integer columnIndex; + /** 异常相关的列名称，中文 */ + private List columnNames; + + public DataSheetException(ResultCode resultCode) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + } + + public DataSheetException(ResultCode resultCode, Integer rowIndex, Integer columnIndex) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + this.rowIndex = rowIndex; + this.columnIndex = columnIndex; + } + + public DataSheetException(ResultCode resultCode, List columnNames) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + this.columnNames = columnNames; + } + + public DataSheetException(ResultCode resultCode, String message) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), message); + this.subErrorCode = resultCode.getSubBizCode(); + } + + public DataSheetException(String subErrorCode, String errorMsg, Integer rowIndex, Integer columnIndex) { + super(cn.axzo.foundation.result.ResultCode.INVALID_PARAMS.getErrorCode(), errorMsg); + this.subErrorCode = subErrorCode; + this.rowIndex = rowIndex; + this.columnIndex = columnIndex; + } + + public DataSheetException(String errorCode, String subErrorCode, String errorMsg) { + super(errorCode, errorMsg); + this.subErrorCode = subErrorCode; + } + } + + @AllArgsConstructor + @Getter + enum ResultCode { + /** 解析Excel的批注时，相关的errorCode */ + IMPORT_PARSE_MISSING_VERSION("C00", "批注中没有找到版本信息"), + IMPORT_PARSE_VERSION_FORMAT_ERROR("C01", "批注中的版本格式不对"), + IMPORT_PARSE_CELL_META_MISSING_TYPE("C02", "批注中的字段没有找到类型信息"), + IMPORT_PARSE_CELL_META_FORMAT_ERROR("C03", "批注中的字段格式不对"), + IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR("C04", "批注中的范围类型字段的格式不对"), + IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR("C05", "批注中的范围类型字段的区间类型不对"), + + /** 导入数据时，相关的errorCode */ + IMPORT_LINES_REACHED_LIMIT("C20", "导入的数据超过了最大行数"), + IMPORT_COLUMN_RANGE_MISSING_TYPES("C21", "范围类型缺少区间类型的定义：开区间还是闭区间"), + IMPORT_COLUMN_DUPLICATED_NAME("C22", "列的名称不能重复"), + IMPORT_COLUMN_MISSING_CELL_META("C23", "字段缺少类型定义"), + IMPORT_COLUMN_NAME_NOT_MATCHED("C24", "字段的名称与类型定义的名称不一致"), + IMPORT_CELL_RANGE_FORMAT_ERROR("C25", "范围类型的格式不对"), + IMPORT_CELL_RANGE_VALUE_ERROR("C26", "范围类型的下限值必须小于上限值"), + IMPORT_CELL_DATETIME_CONVERT_FAILED("C27", "时间类型解析失败"), + IMPORT_CELL_CONVERT_FAILED("C28", "类型转换失败"), + IMPORT_CELL_META_MISSING_MANDATORY_VALUE("C29", "必填字段不能为空"), + IMPORT_CELL_BOOLEAN_VALUE_ERROR("C30", "布尔类型的值不支持"), + ; + + private String subBizCode; + private String message; + + public DataSheetException toException() { + return new DataSheetException(this); + } + + public DataSheetException toException(String message) { + return new DataSheetException(this, message); + } + + public DataSheetException toException(List columnNames) { + return new DataSheetException(this, columnNames); + } + + public DataSheetException toException(Integer rowIndex, Integer columnIndex) { + return new DataSheetException(this, rowIndex, columnIndex); + } + + public DataSheetException toException(String customMsg, Object... objects) { + if (objects == null) { + return toException(customMsg); + } + + String msg = VarParamFormatter.format(customMsg, objects); + //如果最后一个参数是Throwable. 则将SimpleName附加到msg中 + if (objects[objects.length - 1] instanceof Throwable) { + Throwable throwable = (Throwable) objects[objects.length - 1]; + msg = String.format("%s (%s)", msg, throwable.getClass().getSimpleName()); + } + + return toException(msg); + } + } +} diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetAsyncExporter.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetAsyncExporter.java new file mode 100644 index 0000000..7a2f0b3 --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetAsyncExporter.java @@ -0,0 +1,326 @@ +package cn.axzo.foundation.excel.support.impl; + +import cn.axzo.foundation.enums.AppEnvEnum; +import cn.axzo.foundation.excel.support.DataSheetClient; +import cn.axzo.foundation.util.UUIDBuilder; +import cn.axzo.foundation.web.support.AppRuntime; +import cn.axzo.foundation.web.support.utils.KeyBuilder; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.excel.util.IoUtils; +import com.alibaba.excel.util.StringUtils; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.task.TaskExecutor; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.util.CollectionUtils; + +import java.io.*; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * 异步数据导出. + * 将导出的进度防止redis 中，当导出完成后，将文件通过 fileUploader 上传到云存储。 + */ +@Slf4j +public class DataSheetAsyncExporter implements DataSheetClient.AsyncExporter { + + private static final int PROGRESS_EXPIRE_DAYS = 3; + private RedisTemplate redisTemplate; + private AppRuntime appRuntime; + private Function fileUploader; + private TaskExecutor executor; + + public DataSheetAsyncExporter(@NonNull RedisTemplate redisTemplate, + @NonNull AppRuntime appRuntime, + @NonNull Function fileUploader, + @NonNull TaskExecutor executor) { + this.redisTemplate = redisTemplate; + this.appRuntime = appRuntime; + this.fileUploader = fileUploader; + this.executor = executor; + } + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + public static class FileUploadParam { + private byte[] bytes; + private String fileName; + private String operator; + private JSONObject ext; + } + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + public static class FileUploadResult { + private String downloadUrl; + } + + @Override + public String export(@NonNull DataSheetClient.ExporterBuilder dataSheetExporter) { + final String batchId = UUIDBuilder.generateLongUuid(false); + final String scene = dataSheetExporter.scene(); + ExportContext context = ExportContext.builder().batchId(batchId).fileName(dataSheetExporter.fileName()) + .operator(dataSheetExporter.operator()).ext(dataSheetExporter.ext()).build(); + executor.execute(() -> doExport(context, scene, dataSheetExporter)); + return batchId; + } + + @Override + public String export(String fileName, @NonNull List dataSheetExporters) { + checkMultiExporters(fileName, dataSheetExporters); + + final String batchId = UUIDBuilder.generateLongUuid(false); + ExportContext context = ExportContext.builder() + .batchId(batchId) + .fileName(fileName) + .operator(dataSheetExporters.get(0).operator()) + .writer(EasyExcel.write(new ByteArrayOutputStream()) + .registerWriteHandler(new LongestMatchColumnWidthStyleStrategy()) + .build()) + .progressMap(dataSheetExporters.stream().collect(Collectors.toMap(e -> e.sheetName(), e -> 0))) + .build(); + + executor.execute(() -> dataSheetExporters.forEach(e -> doExport(context, e.scene(), e))); + return batchId; + } + + void doExport(final ExportContext context, final String scene, DataSheetClient.ExporterBuilder dataSheetExporter) { + try { + final File tempFile = File.createTempFile(context.getBatchId(), ".tmp"); + tempFile.deleteOnExit(); + log.info("DataSheetAsyncExporter export() scene={} context={}, tempFile->{}", scene, context, tempFile.getAbsolutePath()); + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) { + DataSheetClient.Exporter exporter = dataSheetExporter.onProgress((progress, errorMsg) -> { + if (context.isMultiSheet()) { + onMultiSheetProgress(progress, errorMsg, context, tempFile, dataSheetExporter.sheetName()); + } else { + onProgress(progress, errorMsg, context, tempFile); + } + }).build(); + + Preconditions.checkState(exporter != null); + exporter.export(out); + } catch (Exception e) { + log.error("DataSheetAsyncExporter export() FAIL scene={} context={} ", scene, context, e); + } finally { + if (tempFile != null && tempFile.exists()) { + tempFile.delete(); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void checkMultiExporters(String fileName, List dataSheetExporters) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(fileName), "导出文件名不能为空"); + Preconditions.checkArgument(!CollectionUtils.isEmpty(dataSheetExporters), "导出参数不能为空"); + dataSheetExporters.forEach(e -> Preconditions.checkArgument(e.format() == DataSheetClient.ExportFormat.EXCEL, "导出格式必须为EXCEL")); + dataSheetExporters.forEach(e -> Preconditions.checkArgument(!Strings.isNullOrEmpty(e.sheetName()), "表单名不能为空")); + + long sheetCount = dataSheetExporters.stream().map(DataSheetClient.ExporterBuilder::sheetName).distinct().count(); + Preconditions.checkArgument(sheetCount == dataSheetExporters.size(), "表单名不能重复"); + } + + private String buildRedisKey(String batchId) { + return KeyBuilder.build(appRuntime, "data_export", batchId); + } + + private void onProgress(Integer progress, String errorMsg, ExportContext context, File tempFile) { + log.info("DataSheetAsyncExporter onProgress() progress={} errorMsg={} context={} tempFile={}", progress, errorMsg, context, tempFile.getAbsolutePath()); + String downloadUrl = StringUtils.EMPTY; + if (Strings.isNullOrEmpty(errorMsg) && progress >= 100) { + try { + // 只有导出成功，并完成，才开始上传文件 + byte[] bytes = IoUtils.toByteArray(new BufferedInputStream(new FileInputStream(tempFile))); + downloadUrl = fileUploader.apply(FileUploadParam.builder() + .bytes(bytes).fileName(context.getFileName()) + .operator(context.getOperator()).ext(context.getExt()).build()).getDownloadUrl(); + log.info("DataSheetAsyncExporter onProgress() DONE context={}, downloadUrl={}", context, downloadUrl); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + // 将导出的状态放入 redis 中。后续通过getProgress()来获取。 + String jsonString = JSONObject.toJSONString(Progress.builder().fileName(context.getFileName()).progress(progress) + .downloadUrl(downloadUrl).errorMsg(errorMsg).build()); + redisTemplate.opsForValue().set(buildRedisKey(context.getBatchId()), jsonString, PROGRESS_EXPIRE_DAYS, TimeUnit.DAYS); + } + + private void onMultiSheetProgress(Integer progress, String errorMsg, ExportContext context, File tempFile, String sheetName) { + log.info("DataSheetAsyncExporter onMultiSheetProgress() context={}, progress={} errorMsg={} tempFile={} sheetName={}", context, progress, errorMsg, tempFile.getAbsolutePath(), sheetName); + context.getProgressMap().put(sheetName, progress); + Progress totalProgress = getMultiSheetProgress(context, errorMsg); + + if (!Strings.isNullOrEmpty(errorMsg) && progress < 0) { + context.getWriter().write(ImmutableList.of(ImmutableList.of(errorMsg)), + EasyExcel.writerSheet(sheetName).build()); + } else if (progress >= 100) { + EasyExcel.read(tempFile, new ReadAndWriteListener(context, sheetName)).sheet(sheetName).doRead(); + } + + if (totalProgress.isDone()) { + // 只有所有表单都导出完成，才开始上传文件 + context.getWriter().finish(); + ByteArrayOutputStream outputStream = (ByteArrayOutputStream) context.getWriter().writeContext() + .writeWorkbookHolder().getOutputStream(); + String downloadUrl = fileUploader.apply(FileUploadParam.builder() + .bytes(outputStream.toByteArray()).fileName(context.getFileName()) + .operator(context.getOperator()).build()).getDownloadUrl(); + log.info("DataSheetAsyncExporter onMultiSheetProgress() DONE context={}, downloadUrl={}", context, downloadUrl); + totalProgress.setDownloadUrl(downloadUrl); + } + + // 将导出的状态放入 redis 中。后续通过getProgress()来获取。 + String jsonString = JSONObject.toJSONString(totalProgress); + redisTemplate.opsForValue().set(buildRedisKey(context.getBatchId()), jsonString, PROGRESS_EXPIRE_DAYS, TimeUnit.DAYS); + } + + private Progress getMultiSheetProgress(ExportContext context, String errorMsg) { + Map progressMap = context.getProgressMap(); + int totalProgress = progressMap.values().stream().mapToInt(a -> a).sum() / progressMap.size(); + boolean isDone = progressMap.values().stream().allMatch(a -> a < 0 || a >= 100); + if (isDone && totalProgress != 100) { + // 所有导出都已完成，但其中有失败，将progress设为-1 + totalProgress = -1; + } + + return Progress.builder() + .fileName(context.getFileName()) + .progress(totalProgress) + .errorMsg(errorMsg) + .downloadUrl(StringUtils.EMPTY) + .build(); + } + + /** + * 使用在export()返回的 batchId 来查询导出进度。 + * + * @param batchId + * @return + */ + @Override + public Progress getProgress(String batchId) { + String res = redisTemplate.opsForValue().get(buildRedisKey(batchId)); + if (Strings.isNullOrEmpty(res)) { + return Progress.builder().fileName(StringUtils.EMPTY) + .progress(0).errorMsg(StringUtils.EMPTY).downloadUrl(StringUtils.EMPTY).build(); + } + return JSONObject.parseObject(res, Progress.class); + } + + @Data + public static class Builder { + private RedisTemplate redisTemplate; + private AppRuntime appRuntime; + private Function fileUploader; + private TaskExecutor executor; + + public Builder redisTemplate(RedisTemplate redisTemplate) { + this.redisTemplate = redisTemplate; + return this; + } + + public Builder appRuntime(AppRuntime appRuntime) { + this.appRuntime = appRuntime; + return this; + } + + public Builder fileUploader(Function fileUploader) { + this.fileUploader = fileUploader; + return this; + } + + public Builder executor(TaskExecutor executor) { + this.executor = executor; + return this; + } + + public DataSheetClient.AsyncExporter build() { + if (appRuntime.getEnv() == AppEnvEnum.unittest) { + return new DataSheetClient.AsyncExporter() { + @Override + public String export(DataSheetClient.@NonNull ExporterBuilder dataSheetExporter) { + return "8888"; + } + + @Override + public String export(String fileName, @NonNull List dataSheetExporters) { + return "8888"; + } + + @Override + public Progress getProgress(String batchId) { + return Progress.builder().progress(100).downloadUrl("http://test/test.jpg").build(); + } + }; + } + return new DataSheetAsyncExporter(redisTemplate, this.appRuntime, this.fileUploader, this.executor); + } + } + + private static class ReadAndWriteListener extends AnalysisEventListener> { + private ExportContext context; + private WriteSheet sheet; + + public ReadAndWriteListener(ExportContext context, String sheetName) { + this.context = context; + this.sheet = EasyExcel.writerSheet(sheetName).build(); + } + + @Override + public void invoke(Map data, AnalysisContext context) { + List row = data.entrySet().stream().sorted(Map.Entry.comparingByKey()) + .map(Map.Entry::getValue).collect(Collectors.toList()); + this.context.getWriter().write(ImmutableList.of(row), sheet); + } + + @Override + public void invokeHeadMap(Map headMap, AnalysisContext context) { + List row = headMap.entrySet().stream().sorted(Map.Entry.comparingByKey()) + .map(Map.Entry::getValue).collect(Collectors.toList()); + this.context.getWriter().write(ImmutableList.of(row), sheet); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + } + } + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + static class ExportContext { + private String batchId; + private String fileName; + private String operator; + private ExcelWriter writer; + private Map progressMap; + private JSONObject ext; + + private boolean isMultiSheet() { + return writer != null && !CollectionUtils.isEmpty(progressMap); + } + } +} diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetClientImpl.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetClientImpl.java new file mode 100644 index 0000000..12b6fcd --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetClientImpl.java @@ -0,0 +1,90 @@ +package cn.axzo.foundation.excel.support.impl; + +import cn.axzo.foundation.excel.support.DataSheetClient; +import com.alibaba.fastjson.JSONObject; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cglib.beans.BeanMap; + +import java.util.HashMap; +import java.util.Map; + +@Slf4j +public class DataSheetClientImpl implements DataSheetClient { + + private final static String UNKNOWN_OPERATOR = "unknown@fiture.com"; + + @Override + public ExporterBuilder exportBuilder() { + DataSheetExporter exporter = new DataSheetExporter(); + exporter.setOnCompleted(this::reportExport); + return exporter; + } + + @Override + public ImporterBuilder importBuilder() { + DataSheetImporter importer = new DataSheetImporter(); + importer.setOnCompleted(this::reportImport); + return importer; + } + + public void reportExport(DataSheetClient.ExportResp resp) { + String operator = resp.getOperator(); + + ReportReq req = ReportReq.builder() + .scene(resp.getScene()) + .action(ReportReq.Action.EXPORT) + .resp(BeanMap.create(resp)) + .rowCount(resp.getRowCount()) + .elapsedMillis(resp.getElapsedMillis()) + .operatorId(operator) + .operatorName(operator) + .build(); + report(req); + } + + public void reportImport(DataSheetClient.ImportResp resp) { + String operator = resp.getOperator(); + + JSONObject jsonObject = new JSONObject(new HashMap<>(BeanMap.create(resp))); + ReportReq req = ReportReq.builder() + .scene(resp.getScene()) + .action(ReportReq.Action.IMPORT) + .resp(jsonObject.fluentRemove("lines").fluentRemove("meta")) + .rowCount(resp.getRowCount().longValue()) + .elapsedMillis(resp.getElapsedMillis()) + .operatorId(operator) + .operatorName(operator) + .build(); + report(req); + } + + private boolean report(ReportReq req) { + return true; + } + + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + private static class ReportReq { + String appName; + String scene; + Action action; + Map resp; + Long rowCount; + Long elapsedMillis; + String filePath; + String operatorId; + String operatorName; + String operatorTenantId; + + public enum Action { + IMPORT, + EXPORT; + } + } +} diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetExporter.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetExporter.java new file mode 100644 index 0000000..5e3cf7d --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetExporter.java @@ -0,0 +1,698 @@ +package cn.axzo.foundation.excel.support.impl; + +import cn.axzo.foundation.excel.support.DataSheetClient; +import cn.axzo.foundation.exception.BusinessException; +import cn.axzo.foundation.page.PageReq; +import cn.axzo.foundation.page.PageResp; +import cn.axzo.foundation.result.ResultCode; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.metadata.Head; +import com.alibaba.excel.metadata.data.WriteCellData; +import com.alibaba.excel.util.StringUtils; +import com.alibaba.excel.write.builder.ExcelWriterBuilder; +import com.alibaba.excel.write.handler.AbstractCellWriteHandler; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.excel.write.metadata.holder.WriteSheetHolder; +import com.alibaba.excel.write.metadata.holder.WriteTableHolder; +import com.alibaba.excel.write.style.AbstractCellStyleStrategy; +import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy; +import com.alibaba.excel.write.style.column.SimpleColumnWidthStyleStrategy; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; +import com.google.common.collect.*; +import com.opencsv.CSVWriter; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.CharEncoding; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.ss.util.CellRangeAddressList; +import org.apache.poi.xssf.usermodel.XSSFClientAnchor; +import org.apache.poi.xssf.usermodel.XSSFRichTextString; +import org.springframework.util.CollectionUtils; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*; +import static cn.axzo.foundation.excel.support.DataSheetClient.Meta.*; + + +@Slf4j +public class DataSheetExporter extends DataSheetClient.ExporterBuilder implements DataSheetClient.Exporter { + /** 完成后处理事件. */ + @Setter + private Consumer onCompleted; + + @Override + public DataSheetClient.Exporter build() { + + if (!CollectionUtils.isEmpty(this.fields())) { + + if (this.columnMap() != null) { + throw new RuntimeException("column map is present, please remove column or fields"); + } + if (this.meta() != null) { + throw new RuntimeException("meta is present, please remove meta or fields"); + } + + ImmutableMap> columns = this.fields().stream() + .collect(ImmutableMap.toImmutableMap(DataSheetClient.ExportField::getHeader, DataSheetClient.ExportField::getReader)); + this.columnMap(columns); + + ImmutableMap> multiLineHeadMap = this.fields().stream() + .filter(f -> !CollectionUtils.isEmpty(f.getHeaderLines())) + .collect(ImmutableMap.toImmutableMap(DataSheetClient.ExportField::getHeader, DataSheetClient.ExportField::getHeaderLines)); + this.multiLineHeadMap(multiLineHeadMap); + + DataSheetClient.Meta meta = DataSheetClient.Meta.builder() + .templateCode(this.scene()) + .version(Optional.ofNullable(this.version()).orElse("00")) + .cellMetas(this.fields().stream().map(DataSheetClient.ExportField::toCellMeta).collect(Collectors.toList())) + .build(); + this.meta(meta); + } + + Preconditions.checkArgument(this.scene() != null, "scene不能为空"); + Preconditions.checkArgument(this.format() != null, "format不能为空"); + Preconditions.checkArgument(this.rowSupplier() != null, "rowSupplier不能为空"); + Preconditions.checkArgument(this.columnMap() != null, "columnMap不能为空"); + return this; + } + + @Override + public DataSheetClient.ExportResp export(OutputStream outputStream) { + Preconditions.checkArgument(outputStream != null, "outputStream不能为空"); + + long exportedCount = 0; + Exporter exporter = null; + Stopwatch stopwatch = Stopwatch.createStarted(); + String errorMsg = ""; + + try { + exporter = getDataExporter(outputStream); + + long pageNumber = 1; + while (limit() > exportedCount) { + // 每页最多允许数量 + long pageMaxSize = Math.min(limit() - exportedCount, pageSize()); + if (pageMaxSize <= 0) { + break; + } + PageReq pageParam = PageReq.builder() + .page((int) pageNumber) + // 这里使用pageSize，而不是pageMaxSize，否则会导致最后一次查询的数据和前面的查询数据重复 + .pageSize(pageSize().intValue()) + .build(); + + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, request data, pageParam={}", scene(), pageParam); + } + PageResp result = rowSupplier().apply(pageParam); + + // nextPageMaxSize少于查询的pageSize，说明达到了最大导出数，只提取允许的数量 + if (result.getData().size() > pageMaxSize && pageMaxSize < pageSize()) { + result = PageResp.builder().current(result.getCurrent()) + .size(result.getSize()).total(result.getTotal()) + .data(result.getData().subList(0, (int) pageMaxSize)) + .build(); + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, result data, and split result size to pageMaxSize={}, size={}", + scene(), pageMaxSize, result.getData().size()); + } + } else { + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, result data, size={}", + scene(), result.getData().size()); + } + } + + if (pageNumber == 1) { + if (!CollectionUtils.isEmpty(multiLineHeadMap())) { + List> multiLineHeads = columnMap().keySet().stream() + .map(headName -> multiLineHeadMap().getOrDefault(headName, ImmutableList.of(headName))) + .collect(Collectors.toList()); + exporter.writeMultiLineHeads(multiLineHeads); + } else { + exporter.writeHead(Lists.newArrayList(columnMap().keySet())); + } + + // 第一页数据需要写顶部提示内容及表头 + if (topHintsSupplier() != null) { + exporter.writeTopHints(topHintsSupplier().apply(result)); + } + } + writeData(exporter, result); + + exportedCount += result.getData().size(); + + if (onProgress() != null) { + //XXX： 不知道最大的页数，这里模拟下进度，保证不超过 100% + int progress = Math.min((int) pageNumber, 90); + onProgress().accept(progress, ""); + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, progress={}", scene(), progress); + } + } + + if (result.getData().size() < result.getSize()) { + break; + } + pageNumber += 1; + } + } catch (Exception e) { + errorMsg = e.getMessage(); + if (!StringUtils.isEmpty(errorMsg) && onProgress() != null) { + onProgress().accept(-1, errorMsg); + log.error("-------ExportClient ERROR [{}]------, progress={}", scene(), -1); + } + throw e; + } finally { + if (exporter != null) { + exporter.finish(); + if (StringUtils.isEmpty(errorMsg) && onProgress() != null) { + onProgress().accept(100, ""); + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, progress={}", scene(), 100); + } + } + } + } + + stopwatch.stop(); + DataSheetClient.ExportResp exportResp = DataSheetClient.ExportResp.builder() + .scene(scene()) + .exportedCount(exportedCount) + .rowCount(exporter.getRowCount()) + .columnCount(exporter.getColumnCount()) + .dataRowCount(exporter.getDataRowCount()) + .elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS)) + .operator(operator()) + .build(); + + if (onCompleted != null) { + onCompleted.accept(exportResp); + } + + log.info("-------ExportClient[{}]------, finish export, exportResp={}", scene(), exportResp); + return exportResp; + } + + @Override + public DataSheetClient.ExportResp export(HttpServletResponse response) { + return export(response, null); + } + + @Override + public DataSheetClient.ExportResp export(HttpServletResponse response, String filename) { + Preconditions.checkArgument(response != null, "response不能为空"); + + if (filename == null) { + filename = getFilename(format().getSuffix()); + } + + try { + filename = URLEncoder.encode(filename, CharEncoding.UTF_8); + } catch (UnsupportedEncodingException e) { + throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出失败，请稍后重试"); + } + + response.setCharacterEncoding(CharEncoding.UTF_8); + response.setContentType(format().getContentType()); + + try { + response.setHeader("Content-Disposition", "attachment;filename=" + filename); + + log.info("-------ExportClient[{}]------, write to servlet response, contentType={}, filename={}", + scene(), format().getContentType(), filename); + + return export(response.getOutputStream()); + } catch (Exception ex) { + log.error("导出数据异常，文件名：" + filename, ex); + throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出失败，请稍后重试"); + } + } + + private Exporter getDataExporter(OutputStream outputStream) { + return format() == DataSheetClient.ExportFormat.CSV ? + new CsvExporter(outputStream, scene(), debugEnabled()) : + new ExcelExporter(sheetName(), outputStream, scene(), meta(), debugEnabled(), autoTrim()); + } + + private String getFilename(String suffix) { + if (fileName() != null) { + return fileName(); + } + String formatDatetime = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").format(LocalDateTime.now()); + return (Strings.isNullOrEmpty(sheetName()) ? "" : sheetName() + "-" + formatDatetime) + suffix; + } + + private void writeData(Exporter exporter, PageResp page) { + if (CollectionUtils.isEmpty(page.getData())) { + return; + } + + List rowPage = page.getData(); + if (rowConverter() != null) { + rowPage = page.getData().stream() + .map(row -> { + List converted = rowConverter().apply(row); + + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, row converted, row={}, converted={}", + scene(), row, converted); + } + + return converted; + }) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } + if (pageConverter() != null) { + final List originRowPage = rowPage; + rowPage = pageConverter().apply(rowPage); + if (debugEnabled()) { + log.info("-------ExportClient[{}]------, page converted, old={}, converted={}", + scene(), originRowPage, rowPage); + } + } + + int rowIndex = (int) exporter.getDataRowCount(); + List finalRowPage = rowPage; + List> rows = IntStream.range(0, finalRowPage.size()) + .mapToObj(index -> columnMap().values().stream() + .map(v -> v.apply(finalRowPage.get(index), rowIndex + index)) + .collect(Collectors.toList())) + .collect(Collectors.toList()); + rows.forEach(exporter::writeRow); + } + + private class CsvExporter extends RowColumnCounter implements Exporter { + + private CSVWriter writer; + private String scene; + private boolean debugEnabled; + + private CsvExporter(OutputStream outputStream, String scene, boolean debugEnabled) { + try { + // encoding + outputStream.write(0xef); + outputStream.write(0xbb); + outputStream.write(0xbf); + } catch (IOException e) { + throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出数据失败"); + } + + writer = new CSVWriter(new OutputStreamWriter(outputStream)); + this.scene = scene; + this.debugEnabled = debugEnabled; + } + + @Override + public void writeTopHints(List topHints) { + if (CollectionUtils.isEmpty(topHints)) { + return; + } + topHints.forEach(t -> writer.writeNext(new String[]{t})); + countTopHints(topHints.size()); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write top={}", scene, topHints); + } + } + + @Override + public void writeHead(List head) { + writer.writeNext(head.toArray(new String[0])); + countHead(head.size()); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write head={}", scene, head); + } + } + + @Override + public void writeMultiLineHeads(List> multiLineHeads) { + int lineNum = multiLineHeads.stream().mapToInt(List::size).max().orElse(0); + IntStream.range(0, lineNum).forEach(i -> { + List heads = multiLineHeads.stream().map(head -> { + // 当表头的行数小于最大行数时，最上面的行中表头内容写入空字符串 + int line = i - lineNum + head.size(); + return line < 0 ? StringUtils.EMPTY : head.get(line); + }).collect(Collectors.toList()); + writeHead(heads); + }); + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write multi line heads={}", scene, multiLineHeads); + } + } + + @Override + public void writeRow(List row) { + String[] arrayContents = row.stream() + .map(c -> c == null ? StringUtils.EMPTY : c.toString()) + .toArray(String[]::new); + writer.writeNext(arrayContents); + countRow(row.size()); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write row={}", scene, row); + } + } + + @Override + public void finish() { + try { + writer.close(); + } catch (IOException e) { + throw new BusinessException(ResultCode.RUNTIME_EXCEPTION.getErrorCode(), "导出数据失败"); + } + } + } + + private class ExcelExporter extends RowColumnCounter implements Exporter { + + private final Integer DEFAULT_WIDTH = 6; + + private ExcelWriter writer; + private WriteSheet sheet; + private String scene; + private boolean debugEnabled; + private HeaderCommentWriteHandler commentWriteHandler; + private DataValidationWriteHandler validationWriteHandler; + private int topHintsRowCount = 0; + + private ExcelExporter(String sheetName, OutputStream outputStream, String scene, DataSheetClient.Meta meta, + boolean debugEnabled, Boolean autoTrim) { + + ExcelWriterBuilder excelWriterBuilder = EasyExcel.write(outputStream); + if (Objects.nonNull(meta) && !CollectionUtils.isEmpty(meta.getCellMetas())) { + Integer width = meta.getCellMetas().get(0).getWidth(); + excelWriterBuilder.registerWriteHandler(new SimpleColumnWidthStyleStrategy(Objects.nonNull(width) ? + width : DEFAULT_WIDTH)); + } else { + excelWriterBuilder.registerWriteHandler(new LongestMatchColumnWidthStyleStrategy()); + } + + if (meta != null) { + excelWriterBuilder.registerWriteHandler(new AbstractCellStyleStrategy() { + private CellStyle wrapStyle; + + @Override + protected void setHeadCellStyle(Cell cell, Head head, Integer relativeRowIndex) { + } + + @Override + protected void setContentCellStyle(Cell cell, Head head, Integer relativeRowIndex) { + if (cell.getColumnIndex() >= meta.getCellMetas().size()) { + return; + } + DataSheetClient.CellMeta cellMeta = meta.getCellMetas().get(cell.getColumnIndex()); + if (cellMeta != null && BooleanUtils.isTrue(cellMeta.getWrapText())) { + cell.setCellStyle(wrapStyle); + } + } + }); + commentWriteHandler = new HeaderCommentWriteHandler(meta); + validationWriteHandler = new DataValidationWriteHandler(meta); + + excelWriterBuilder.registerWriteHandler(new AbstractCellWriteHandler() { + @Override + public void afterCellDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, List> cellDataList, Cell cell, Head head, Integer relativeRowIndex, Boolean isHead) { + boolean isHeadRow = cell.getRowIndex() == topHintsRowCount; + commentWriteHandler.afterCellDispose(cell, isHeadRow); + validationWriteHandler.afterCellDispose(cell, isHeadRow); + } + }); + } + + writer = excelWriterBuilder.autoTrim(autoTrim).build(); + sheet = EasyExcel.writerSheet(sheetName).build(); + this.scene = scene; + this.debugEnabled = debugEnabled; + } + + @Override + public void writeTopHints(List topHints) { + if (CollectionUtils.isEmpty(topHints)) { + return; + } + countTopHints(topHints.size()); + topHintsRowCount += topHints.size(); + writer.write(topHints.stream().map(ImmutableList::of).collect(Collectors.toList()), sheet); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write top={}", scene, topHints); + } + } + + @Override + public void writeHead(List head) { + sheet.setHead(head.stream().map(Lists::newArrayList).collect(Collectors.toList())); + countHead(head.size()); + + // XXX 写入head后需要调用一次write()确保excel中有数据，否则数据为空时将出现内容异常 + writer.write(ImmutableList.of(), sheet); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write head={}", scene, head); + } + } + + @Override + public void writeMultiLineHeads(List> multiLineHeads) { + // XXX setHead()方法有副作用，内部逻辑会改变传入的list, 因此需要确保传入的list可操作 + sheet.setHead(multiLineHeads.stream().map(Lists::newArrayList).collect(Collectors.toList())); + int lineNum = multiLineHeads.stream().mapToInt(List::size).max().orElse(0); + IntStream.range(0, lineNum).forEach(i -> countHead(multiLineHeads.size())); + + // XXX 写入head后需要调用一次write()确保excel中有数据，否则数据为空时将出现内容异常 + writer.write(ImmutableList.of(), sheet); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write multi line heads={}", scene, multiLineHeads); + } + } + + @Override + public void writeRow(List row) { + writer.write(ImmutableList.of(row), sheet); + countRow(row.size()); + + if (debugEnabled) { + log.info("-------ExportClient[{}]------, write row={}", scene, row); + } + } + + @Override + public void finish() { + writer.finish(); + } + } + + /** + * Excel添加头部的批注 + * XXX: 此处不能继承AbstractCellWriteHandler 会导致 mybatis typehandler 扫描时被加载 + * 导致老项目没有使用 easyexcel2.2 的项目启动失败。 + */ + private static class HeaderCommentWriteHandler { + private static final Map, String> rangeBoundTypes = ImmutableMap.of( + ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN), RANGE_TYPE_OPEN_OPEN, + ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED), RANGE_TYPE_OPEN_CLOSED, + ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN), RANGE_TYPE_CLOSED_OPEN, + ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED), RANGE_TYPE_CLOSED_CLOSED); + + private String templateCode; + private String version; + private Map cellMetaMap; + + private Set typeWithParams = ImmutableSet.of( + DataSheetClient.CellMeta.Type.RANGE, + DataSheetClient.CellMeta.Type.DATE, + DataSheetClient.CellMeta.Type.DATETIME + ); + + private HeaderCommentWriteHandler(DataSheetClient.Meta meta) { + this.templateCode = meta.getTemplateCode(); + this.version = meta.getVersion(); + + this.cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), DataSheetClient.CellMeta::getName); + } + + public void afterCellDispose(Cell cell, boolean isHead) { + // 只添加表头行的批注 + if (!isHead) { + return; + } + + List comments = Lists.newArrayList(); + if (cell.getColumnIndex() == 0) { + String content = Joiner.on("_").join(ImmutableList.of(templateCode, version)); + comments.add(buildComment(PATTERN_VERSION_OPEN, content)); + } + + DataSheetClient.CellMeta cellMeta = cellMetaMap.get(cell.getStringCellValue()); + if (cellMeta == null) { + return; + } + // 格式： key_type_mandatory + String content = Joiner.on("_").join(ImmutableList.of( + cellMeta.getKey(), + cellMeta.getType().getCode(), + Boolean.TRUE.equals(cellMeta.getMandatory()) ? "1" : "0")); + + if (typeWithParams.contains(cellMeta.getType())) { + content = content.concat("_").concat(getParamsByType(cellMeta)); + } + comments.add(buildComment(PATTERN_CELL_OPEN, content)); + + // 一个Cell只能写入一次Comment + writeComment(cell, Joiner.on("\n").join(comments)); + } + + private String getParamsByType(DataSheetClient.CellMeta cellMeta) { + switch (cellMeta.getType()) { + case RANGE: + String lowerType = cellMeta.getParams().getString(EXT_KEY_RANGE_LOWER_TYPE); + String upperType = cellMeta.getParams().getString(EXT_KEY_RANGE_UPPER_TYPE); + return rangeBoundTypes.get(ImmutableList.of(lowerType, upperType)); + case DATE: + case DATETIME: + return cellMeta.getParams().getString(EXT_KEY_DATETIME_FORMAT); + default: + return StringUtils.EMPTY; + } + } + + private void writeComment(Cell cell, String comment) { + int columnIndex = cell.getColumnIndex(); + int rowIndex = cell.getRowIndex(); + Drawing drawing = cell.getSheet().createDrawingPatriarch(); + Comment cellComment = drawing.createCellComment(new XSSFClientAnchor(0, 0, 0, 0, + // 控制批注的位置 + columnIndex, rowIndex, columnIndex + 2, rowIndex + 2)); + cellComment.setString(new XSSFRichTextString(comment)); + cell.setCellComment(cellComment); + } + + private String buildComment(String patternOpen, String content) { + return patternOpen.concat(content).concat(PATTERN_CLOSE); + } + } + + /** + * Excel添加校验数据 + * XXX: 此处不能继承AbstractCellWriteHandler 会导致 mybatis typehandler 扫描时被加载 + * 导致老项目没有使用 easyexcel2.2 的项目启动失败。 + */ + private static class DataValidationWriteHandler { + private Map cellMetaMap; + + private DataValidationWriteHandler(DataSheetClient.Meta meta) { + this.cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), DataSheetClient.CellMeta::getName); + } + + public void afterCellDispose(Cell cell, boolean isHead) { + // 只在写入表头数据时才写入校验数据 + if (!isHead) { + return; + } + + DataSheetClient.CellMeta cellMeta = cellMetaMap.get(cell.getStringCellValue()); + // 目前仅支持将可选值作为下拉选项的校验数据 + if (cellMeta == null || CollectionUtils.isEmpty(cellMeta.getOptions())) { + return; + } + String[] options = cellMeta.getOptions().stream().map(String::valueOf).toArray(String[]::new); + + Sheet currentSheet = cell.getSheet(); + Workbook workbook = currentSheet.getWorkbook(); + // 直接将校验数据作为列表项的方式有数据大小限制(不超过256个字符), 因此需要将校验数据放在一个新增的隐藏表单中 + // 以表头的名字来命名新增的表单 + String sheetName = cell.getStringCellValue(); + Sheet validationSheet = workbook.createSheet(sheetName); + workbook.setSheetHidden(workbook.getSheetIndex(validationSheet), true); + + // 将可选项放在新增的隐藏表单里 + for (int i = 0; i < options.length; i++) { + Row rowInValidationSheet = validationSheet.createRow(i); + Cell cellInValidationSheet = rowInValidationSheet.createCell(0); + cellInValidationSheet.setCellValue(options[i]); + } + + // 设置列表校验数据的公式 [表名!开始列开始行:结束列结束行] + String listFormula = sheetName + "!$A$1:$A$" + options.length; + DataValidationHelper dataValidationHelper = currentSheet.getDataValidationHelper(); + DataValidationConstraint constraint = dataValidationHelper.createFormulaListConstraint(listFormula); + + // 从下一行开始10000行数据都增加校验数据 + CellRangeAddressList addressList = new CellRangeAddressList(cell.getRowIndex() + 1, + cell.getRowIndex() + 10001, cell.getColumnIndex(), cell.getColumnIndex()); + DataValidation dataValidation = dataValidationHelper.createValidation(constraint, addressList); + dataValidation.setSuppressDropDownArrow(true); + dataValidation.setShowErrorBox(true); + + currentSheet.addValidationData(dataValidation); + } + } + + private class RowColumnCounter { + @Getter + private long rowCount = 0; + @Getter + private long columnCount = 0; + @Getter + private long dataRowCount = 0; + + protected void countTopHints(int topHintsRowCount) { + rowCount += topHintsRowCount; + columnCount = Math.max(columnCount, 1); + } + + protected void countHead(int headColumnCount) { + rowCount += 1; + columnCount = Math.max(columnCount, headColumnCount); + } + + protected void countRow(int columnCount) { + rowCount += 1; + this.columnCount = Math.max(this.columnCount, columnCount); + dataRowCount += 1; + } + } + + private interface Exporter { + void writeTopHints(List topHints); + + void writeHead(List head); + + void writeMultiLineHeads(List> multiLineHeads); + + void writeRow(List row); + + void finish(); + + long getRowCount(); + + long getColumnCount(); + + long getDataRowCount(); + } +} diff --git a/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetImporter.java b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetImporter.java new file mode 100644 index 0000000..74db59f --- /dev/null +++ b/excel-support-lib/src/main/java/cn/axzo/foundation/excel/support/impl/DataSheetImporter.java @@ -0,0 +1,595 @@ +package cn.axzo.foundation.excel.support.impl; + +import cn.axzo.foundation.excel.support.DataSheetClient; +import cn.axzo.foundation.exception.BusinessException; +import cn.axzo.foundation.util.Regex; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.enums.CellExtraTypeEnum; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.excel.metadata.CellExtra; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; +import com.google.common.collect.*; +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static cn.axzo.foundation.excel.support.DataSheetClient.CellMeta.*; +import static cn.axzo.foundation.excel.support.DataSheetClient.Meta.*; +import static cn.axzo.foundation.excel.support.DataSheetClient.ResultCode.IMPORT_CELL_CONVERT_FAILED; + + +/** + * 使用ali的EasyExcel来读入excel/csv文件 + * + * @author yuanyi + */ +@Slf4j +public class DataSheetImporter extends DataSheetClient.ImporterBuilder { + + private static final String LINE_KEY_ERRORS = "errors"; + private static final String LINE_KEY_ROW_INDEX = "rowIndex"; + + /** + * 完成后处理事件. + */ + @Setter + private Consumer onCompleted; + + @Override + public DataSheetClient.Importer build() { + Preconditions.checkArgument(this.scene() != null, "scene不能为空"); + Preconditions.checkArgument(this.format() != null, "format不能为空"); + + // TODO: 当支持更多format的时候，需要生成对应的Importer实例 + return ExcelImporter.builder() + .scene(scene()) + .tableName(tableName()) + .meta(meta()) + .ignoreHeaderMeta(ignoreHeaderMeta()) + .debugEnabled(debugEnabled()) + .allowMaxLineCount(allowMaxLineCount()) + .headerConverter(headerConverter()) + .includeLineErrors(includeLineErrors()) + .ignoreUnknownColumn(ignoreUnknownColumn()) + .autoTrim(autoTrim()) + .onCompleted(onCompleted) + .operator(operator()) + .build(); + } + + @Builder + private static class ExcelImporter implements DataSheetClient.Importer { + + private static final Map> rangeBoundTypes = ImmutableMap.of( + RANGE_TYPE_OPEN_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN), + RANGE_TYPE_OPEN_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED), + RANGE_TYPE_CLOSED_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN), + RANGE_TYPE_CLOSED_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED)); + + private String scene; + private String tableName; + private DataSheetClient.Meta meta; + private Boolean ignoreHeaderMeta; + private boolean debugEnabled; + private Integer allowMaxLineCount; + private boolean includeLineErrors; + private boolean ignoreUnknownColumn; + private Boolean autoTrim; + private Function headerConverter; + private Consumer onCompleted; + private String operator; + + @Override + public DataSheetClient.ImportResp readAll(InputStream inputStream) { + Stopwatch stopwatch = Stopwatch.createStarted(); + + NoModelDataListener dataListener = new NoModelDataListener(headerConverter, autoTrim); + EasyExcel.read(inputStream, dataListener) + .autoTrim(autoTrim) + .extraRead(CellExtraTypeEnum.COMMENT).sheet(tableName).doRead(); + if (allowMaxLineCount != null && dataListener.lines.size() > allowMaxLineCount) { + throw DataSheetClient.ResultCode.IMPORT_LINES_REACHED_LIMIT + .toException("导入的数据超过了最大行数" + allowMaxLineCount); + } + + if (BooleanUtils.isNotTrue(this.ignoreHeaderMeta)) { + // 聚合来自入参和文件批注的meta信息；如果都存在，使用入参的meta覆盖文件批注的meta + this.meta = mergeMeta(this.meta, parseMetaFromData(dataListener)); + } + filterHeadMap(dataListener); + filterLines(dataListener); + + validateHeaders(dataListener); + validateMeta(dataListener); + + List headers = parseHeaders(dataListener); + List lines = parseLines(dataListener); + if (!includeLineErrors) { + // 没有设置includeLineErrors时，抛第一个发现的异常 + Optional errorLine = lines.stream() + .filter(line -> line.containsKey(LINE_KEY_ERRORS)) + .findFirst(); + if (errorLine.isPresent()) { + JSONObject error = errorLine.get().getJSONArray(LINE_KEY_ERRORS).getJSONObject(0); + Integer rowIndex = lines.indexOf(errorLine.get()); + Integer columnIndex = error.getInteger("columnIndex"); + + String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1, + error.getString("columnName"), error.getString("errorMsg")); + throw new DataSheetClient.DataSheetException(error.getString("subErrorCode"), errMsg, rowIndex, columnIndex); + } + } + + DataSheetClient.ImportResp resp = DataSheetClient.ImportResp.builder() + .scene(scene) + .templateCode(Optional.ofNullable(meta).map(DataSheetClient.Meta::getTemplateCode).orElse(null)) + .version(Optional.ofNullable(meta).map(DataSheetClient.Meta::getVersion).orElse(null)) + .headers(headers) + .lines(lines) + .headerRowCount(1) + .rowCount(lines.size()) + .columnCount(headers.size()) + .meta(meta) + .elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS)) + .operator(operator) + .build(); + + if (null != onCompleted) { + onCompleted.accept(resp); + } + return resp; + } + + private void filterHeadMap(NoModelDataListener dataListener) { + if (meta == null) { + return; + } + + Map headMap = dataListener.getHeadMap(); + + Set ignoreColumnNames = meta.getIgnoreColumnNames(); + // 收集需要过滤的列号 + Set ignoreColumnIndexes = headMap.entrySet().stream() + .filter(entry -> ignoreColumnNames.contains(entry.getValue())) + .map(Map.Entry::getKey).collect(Collectors.toSet()); + // 加上指定过滤的列号 + ignoreColumnIndexes.addAll(meta.getIgnoreColumnIndexes()); + + ignoreColumnIndexes.forEach(headMap::remove); + } + + private void filterLines(NoModelDataListener dataListener) { + if (meta == null) { + return; + } + Set toRemoveLines = ImmutableSet.copyOf(meta.getIgnoreRowIndexes()); + List> lines = dataListener.getLines(); + dataListener.setLines(IntStream.range(0, lines.size()) + // 这里要加1，是因为removeIndex是整个文档的行数来计算，包括了header + // 但是lines的数据，已经排除了header + .filter(index -> !toRemoveLines.contains(index + 1)) + .mapToObj(lines::get) + .collect(Collectors.toList())); + } + + private DataSheetClient.Meta parseMetaFromData(NoModelDataListener dataListener) { + Map headMap = dataListener.getHeadMap(); + List headComments = dataListener.getCellComments().stream() + // 获取第一行Head的批注信息 + .filter(cellExtra -> cellExtra.getRowIndex() == 0) + // 过滤带有Cell参数配置的批注信息 + .filter(cellExtra -> !Strings.isNullOrEmpty(StringUtils.substringBetween(cellExtra.getText(), + PATTERN_CELL_OPEN, PATTERN_CLOSE))) + // 排序，方便找到第一列，获取templateCode与version + .sorted(Comparator.comparing(CellExtra::getColumnIndex)) + .collect(Collectors.toList()); + // 没有批注信息，直接返回null；会以String来解析值 + if (headComments.isEmpty()) { + return null; + } + + // 从文件中解析meta信息 + JSONObject codeAndVersion = parseTemplateCodeAndVersion(headComments); + List cellMetas = headComments.stream() + .map(cellExtra -> parseCellMeta(cellExtra.getText(), headMap.get(cellExtra.getColumnIndex()))) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (cellMetas.isEmpty()) { + return null; + } + + return DataSheetClient.Meta.builder() + .templateCode(codeAndVersion.getString("templateCode")) + .version(codeAndVersion.getString("version")) + .cellMetas(cellMetas) + .ignoreColumnIndexes(parseIgnoreColumns(dataListener)) + .ignoreRowIndexes(parseIgnoreRows(dataListener)) + .build(); + } + + private DataSheetClient.Meta mergeMeta(DataSheetClient.Meta metaFromParam, DataSheetClient.Meta metaFromHeader) { + if (metaFromHeader == null || metaFromParam == null) { + return metaFromHeader == null ? metaFromParam : metaFromHeader; + } + + // 在两者都存在的时候，以文件的meta为准；仅仅将入参的CellMetas覆盖文件的CellMetas + if (metaFromHeader.getCellMetas() == null) { + metaFromHeader.setCellMetas(metaFromParam.getCellMetas()); + return metaFromHeader; + } + + // 使用入参中定义的列信息，覆盖文件的cellMetas + Map cellMetaMap = + Maps.uniqueIndex(metaFromParam.getCellMetas(), DataSheetClient.CellMeta::getKey); + List cellMetas = metaFromHeader.getCellMetas().stream() + .map(cellMeta -> cellMetaMap.getOrDefault(cellMeta.getKey(), cellMeta)) + .collect(Collectors.toList()); + metaFromHeader.setCellMetas(cellMetas); + return metaFromHeader; + } + + private List parseIgnoreRows(NoModelDataListener dataListener) { + return dataListener.getCellComments().stream() + .map(cellExtra -> { + String ignoreKey = StringUtils.substringBetween(cellExtra.getText(), + PATTERN_IGNORE_OPEN, PATTERN_CLOSE); + if (IGNORE_ROW_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) { + return cellExtra.getRowIndex(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private List parseIgnoreColumns(NoModelDataListener dataListener) { + return dataListener.getCellComments().stream() + .map(cellExtra -> { + String ignoreKey = StringUtils.substringBetween(cellExtra.getText(), + PATTERN_IGNORE_OPEN, PATTERN_CLOSE); + if (IGNORE_COLUMN_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) { + return cellExtra.getColumnIndex(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + public JSONObject parseTemplateCodeAndVersion(List headComments) { + // 从整个头部的批注，获取模版code和版本信息 + Optional parsedText = headComments.stream() + .map(comment -> StringUtils.substringBetween(comment.getText(), PATTERN_VERSION_OPEN, PATTERN_CLOSE)) + .filter(str -> !Strings.isNullOrEmpty(str)) + .findFirst(); + if (!parsedText.isPresent()) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_MISSING_VERSION.toException(); + } + // 格式为"CODE_VERSION" + List values = Splitter.on("_").splitToList(parsedText.get()); + if (values.size() != 2) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_VERSION_FORMAT_ERROR.toException(); + } + return new JSONObject() + .fluentPut("templateCode", values.get(0)) + .fluentPut("version", values.get(1)); + } + + private DataSheetClient.CellMeta parseCellMeta(String text, String name) { + String value = StringUtils.substringBetween(text, PATTERN_CELL_OPEN, PATTERN_CLOSE); + if (Strings.isNullOrEmpty(value)) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_MISSING_TYPE + .toException("没有找到列[{}]的类型信息", name); + } + List values = Splitter.on("_").splitToList(value); + // 格式： key_type_mandatory_params... + if (values.size() < 3) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_FORMAT_ERROR + .toException("列[{}]类型的批注格式不对[{}]", name, text); + } + + DataSheetClient.CellMeta.Type type = DataSheetClient.CellMeta.Type.from(values.get(1)); + JSONObject params = new JSONObject(); + if (type == DataSheetClient.CellMeta.Type.RANGE) { + if (values.size() != 4 || Strings.isNullOrEmpty(values.get(3))) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR + .toException("列[{}]范围类型的批注格式不对[{}]", name, text); + } + List boundType = rangeBoundTypes.get(values.get(3)); + if (boundType == null) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR + .toException(String.format("列[{}]范围类型的值不对", name)); + } + params.put(EXT_KEY_RANGE_LOWER_TYPE, boundType.get(0)); + params.put(EXT_KEY_RANGE_UPPER_TYPE, boundType.get(1)); + } + + JSONObject ext = new JSONObject(); + value = StringUtils.substringBetween(text, PATTERN_EXTRA_OPEN, PATTERN_CLOSE); + if (!Strings.isNullOrEmpty(value)) { + ext.putAll(Splitter.on("&").withKeyValueSeparator("=").split(value)); + } + + return DataSheetClient.CellMeta.builder() + .key(values.get(0)) + .name(name) + .type(type) + .mandatory("1".equals(values.get(2))) + .params(params) + .ext(ext) + .build(); + } + + private List parseHeaders(NoModelDataListener dataListener) { + return dataListener.getHeadMap().keySet().stream() + .sorted() + .map(key -> Strings.nullToEmpty(dataListener.getHeadMap().get(key))) + .collect(Collectors.toList()); + } + + private List parseLines(NoModelDataListener dataListener) { + // 如果没有找到meta信息，按照key=header(很可能是中文), 类型就为String + if (this.meta == null) { + Map headerMap = dataListener.getHeadMap(); + return dataListener.getLines().stream() + .map(line -> new JSONObject().fluentPutAll(line.entrySet().stream() + .map(entry -> Pair.of(headerMap.get(entry.getKey()), Strings.nullToEmpty(entry.getValue()))) + .collect(Collectors.toMap(Pair::getKey, Pair::getValue)))) + .collect(Collectors.toList()); + } + + // 根据meta来校验cell的类型 + Map headerMap = dataListener.getHeadMap(); + Map cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), + DataSheetClient.CellMeta::getName); + List> lines = dataListener.getLines(); + return IntStream.range(0, lines.size()) + .mapToObj(lineIndex -> { + Map line = lines.get(lineIndex); + // 收集每一行每一列的转换结果 + List convertRespList = headerMap.entrySet().stream() + .filter(e -> { + //XXX 当支持忽略列的时候, 只处理定义了cellMeta的数据 + if (ignoreUnknownColumn) { + return cellMetaMap.containsKey(e.getValue()); + } + + return true; + }) + .map(entry -> { + Integer columnIndex = entry.getKey(); + String header = entry.getValue(); + DataSheetClient.CellMeta cellMeta = cellMetaMap.get(header); + return convertType(cellMeta, line.get(columnIndex), lineIndex, columnIndex); + }) + .collect(Collectors.toList()); + + JSONObject row = new JSONObject() + .fluentPutAll(convertRespList.stream() + .filter(ColumnConvertResp::getSuccess) + // convertValue可能为null, 有非必需的字段 + .collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll)); + convertRespList.stream() + .filter(r -> BooleanUtils.isTrue(r.getSuccess())) + .filter(r -> r.getCellMeta().getCellValidator() != null) + .forEach(r -> { + try { + r.getCellMeta().getCellValidator().accept(r.getConvertedValue(), row); + } catch (BusinessException e) { + log.warn("failed to validate cell, resp={}", r, e); + String subErrorCode = null; + if (e instanceof DataSheetClient.DataSheetException) { + subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode(); + } + r.setSuccess(false); + r.setErrorCode(e.getErrorCode()); + r.setErrorMsg(e.getErrorMsg()); + r.setSubErrorCode(subErrorCode); + } + }); + + Map> convertRespMap = convertRespList.stream() + .collect(Collectors.groupingBy(ColumnConvertResp::getSuccess)); + + JSONObject convertedLine = new JSONObject() + .fluentPutAll(convertRespMap.getOrDefault(Boolean.TRUE, ImmutableList.of()).stream() + // convertValue可能为null, 有非必需的字段 + .collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll)); + if (convertRespMap.get(Boolean.FALSE) != null) { + // 转换失败的，将失败信息放到errors字段中 + convertedLine.put(LINE_KEY_ERRORS, convertRespMap.get(Boolean.FALSE).stream() + .map(ColumnConvertResp::getError) + .collect(Collectors.toList())); + } + convertedLine.put(LINE_KEY_ROW_INDEX, lineIndex); + return convertedLine; + }) + .collect(Collectors.toList()); + } + + private ColumnConvertResp convertType(DataSheetClient.CellMeta cellMeta, String rawValue, int rowIndex, int columnIndex) { + try { + return ColumnConvertResp.builder() + .success(true).cellMeta(cellMeta) + .rawValue(rawValue).convertedValue(cellMeta.convertType(rawValue)) + .build(); + } catch (BusinessException e) { + log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}", + cellMeta, rawValue, rowIndex, columnIndex, e); + String subErrorCode = null; + if (e instanceof DataSheetClient.DataSheetException) { + subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode(); + } + return ColumnConvertResp.builder() + .success(false).cellMeta(cellMeta).rawValue(rawValue) + .columnIndex(columnIndex) + .errorCode(e.getErrorCode()) + .errorMsg(e.getErrorMsg()) + .subErrorCode(subErrorCode) + .build(); + } catch (Exception e) { + log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}", + cellMeta, rawValue, rowIndex, columnIndex, e); + String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1, + cellMeta.getName(), Optional.ofNullable(e.getMessage()) + .orElse(IMPORT_CELL_CONVERT_FAILED.getMessage())); + throw new DataSheetClient.DataSheetException(IMPORT_CELL_CONVERT_FAILED.getSubBizCode(), errMsg, rowIndex, columnIndex); + } + } + + private void validateHeaders(NoModelDataListener dataListener) { + Map headMap = dataListener.getHeadMap(); + Set headerNames = ImmutableSet.copyOf(headMap.values()); + if (headerNames.size() != headMap.size()) { + List columnNames = headMap.values().stream() + .collect(Collectors.groupingBy(Function.identity())) + .entrySet().stream() + .filter(grouped -> grouped.getValue().size() > 1) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + throw DataSheetClient.ResultCode.IMPORT_COLUMN_DUPLICATED_NAME.toException(columnNames); + } + } + + private void validateMeta(NoModelDataListener dataListener) { + if (this.meta == null) { + return; + } + + if (meta.getCellMetas().size() != dataListener.getHeadMap().size()) { + if (dataListener.getHeadMap().size() > meta.getCellMetas().size()) { + if (!ignoreUnknownColumn) { + throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException(); + } + } + + long mandatoryColumnSize = meta.getCellMetas().stream() + .filter(cm -> BooleanUtils.isTrue(cm.getMandatory())) + .count(); + if (dataListener.getHeadMap().size() < mandatoryColumnSize) { + throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException(); + } + } + + Set headerNames = ImmutableSet.copyOf(dataListener.getHeadMap().values()); + Set cellNames = meta.getCellMetas().stream() + .map(DataSheetClient.CellMeta::getName) + .collect(Collectors.toSet()); + if (!ignoreUnknownColumn && !headerNames.equals(cellNames)) { + Set missingNames = Sets.difference(cellNames, headerNames); + Set redundantNames = Sets.difference(headerNames, cellNames); + List columnNames = Stream.of(missingNames, redundantNames) + .flatMap(Set::stream) + .collect(Collectors.toList()); + throw DataSheetClient.ResultCode.IMPORT_COLUMN_NAME_NOT_MATCHED.toException(columnNames); + } + meta.getCellMetas().forEach(DataSheetClient.CellMeta::validate); + } + + @Slf4j + @Data + private static class NoModelDataListener extends AnalysisEventListener> { + private Map headMap = Maps.newHashMap(); + private List> lines = Lists.newArrayList(); + private List cellComments = Lists.newArrayList(); + + private Function headerConverter; + private Boolean autoTrim; + + public NoModelDataListener(Function headerConverter, Boolean autoTrim) { + this.headerConverter = headerConverter; + this.autoTrim = autoTrim; + } + + @Override + public void invoke(Map data, AnalysisContext context) { + lines.add(strip(data)); + } + + @Override + public void invokeHeadMap(Map headMap, AnalysisContext context) { + Map convertedMap = headMap; + if (headerConverter != null) { + convertedMap = headMap.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> headerConverter.apply(e.getValue()))); + } + this.headMap.putAll(strip(convertedMap)); + } + + private Map strip(Map data) { + if (BooleanUtils.isFalse(autoTrim)) { + return data; + } + return data.entrySet().stream() + .map(entry -> Maps.immutableEntry(entry.getKey(), StringUtils.strip(entry.getValue(), Regex.WHITESPACE_CHARS))) + // value有可能为null，不能直接用Collectors.toMap + .collect(Maps::newHashMap, (m, v) -> m.put(v.getKey(), v.getValue()), HashMap::putAll); + } + + /** + * extra是在整个文件被解析后才会调用 + * + * @param extra + * @param context + */ + @Override + public void extra(CellExtra extra, AnalysisContext context) { + if (extra.getType() == CellExtraTypeEnum.COMMENT) { + cellComments.add(extra); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + + } + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + private static class ColumnConvertResp { + private Boolean success; + private DataSheetClient.CellMeta cellMeta; + private String rawValue; + private Object convertedValue; + + private Integer columnIndex; + private String errorCode; + private String errorMsg; + private String subErrorCode; + + public String getKey() { + return cellMeta.getKey(); + } + + public JSONObject getError() { + return new JSONObject() + .fluentPut("columnIndex", columnIndex) + .fluentPut("columnKey", cellMeta.getKey()) + .fluentPut("columnName", cellMeta.getName()) + .fluentPut("rawValue", rawValue) + .fluentPut("errorCode", errorCode) + .fluentPut("errorMsg", errorMsg) + .fluentPut("subErrorCode", subErrorCode); + } + } + } +} diff --git a/pom.xml b/pom.xml index 9df7ec1..11260fc 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ gateway-support-lib event-support-lib redis-support-lib + excel-support-lib From 6231b9ebb1b22953f0db3263d1ca35e25b6e26f6 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 5 Jul 2024 17:05:14 +0800 Subject: [PATCH 24/41] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../foundation/web/support/alert/AlertClientImplTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web-support-lib/src/test/java/cn/axzo/foundation/web/support/alert/AlertClientImplTest.java b/web-support-lib/src/test/java/cn/axzo/foundation/web/support/alert/AlertClientImplTest.java index fc2ca7d..4847e8c 100644 --- a/web-support-lib/src/test/java/cn/axzo/foundation/web/support/alert/AlertClientImplTest.java +++ b/web-support-lib/src/test/java/cn/axzo/foundation/web/support/alert/AlertClientImplTest.java @@ -12,9 +12,9 @@ class AlertClientImplTest { .consumer(EmailAlertConsumer.builder() .host("smtp.qiye.aliyun.com") .port(465) - .username("zengxiaobo@axzo.cn") - .password("Zxb19861109") - .from("zengxiaobo@axzo.cn") + .username("notice@axzo.cn") + .password("HIKwTkBgSrHRAyBF") + .from("notice@axzo.cn") .to("wangsiqian@axzo.cn") .build()) .executor(new ScheduledThreadPoolExecutor(1)) From b6bed4d225e32ea33b61d38c4fa25f431f31593a Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 5 Jul 2024 17:26:53 +0800 Subject: [PATCH 25/41] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9=E5=91=8A?= =?UTF-8?q?=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../axzo/foundation/web/support/alert/AlertClient.java | 9 ++++++++- .../foundation/web/support/alert/AlertClientImpl.java | 6 ++++++ .../web/support/alert/AlertClientImplTest.java | 6 +----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClient.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClient.java index 61db730..434d207 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClient.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClient.java @@ -14,6 +14,14 @@ public interface AlertClient { void post(Alert alert); + default void post(String key, Throwable ex, String message, Object... objects) { + post(new Alert(key, ex, message, objects)); + } + + default void post(String key, Throwable ex, String message) { + post(new Alert(key, ex, message, null)); + } + @Data class Alert { private static final int MAX_MESSAGE_LENGTH = 8_000; @@ -25,7 +33,6 @@ public interface AlertClient { String message; String stack; - @Builder public Alert(String key, Throwable ex, String message, Object... objects) { this.key = key; this.message = message; diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClientImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClientImpl.java index f08fbed..0a97e3e 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClientImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClientImpl.java @@ -26,6 +26,12 @@ public class AlertClientImpl implements AlertClient { ListMultimap alertsMap = Multimaps.synchronizedListMultimap(ArrayListMultimap.create()); + /** + * @param consumer + * @param executor + * @param period 默认间隔多少分钟发告警邮件 + * @param consumeImmediatelyPreSecond 如果每秒出现多少次异常告警阈值 + */ @Builder public AlertClientImpl(Consumer>> consumer, ScheduledThreadPoolExecutor executor, diff --git a/web-support-lib/src/test/java/cn/axzo/foundation/web/support/alert/AlertClientImplTest.java b/web-support-lib/src/test/java/cn/axzo/foundation/web/support/alert/AlertClientImplTest.java index 4847e8c..ee95f2d 100644 --- a/web-support-lib/src/test/java/cn/axzo/foundation/web/support/alert/AlertClientImplTest.java +++ b/web-support-lib/src/test/java/cn/axzo/foundation/web/support/alert/AlertClientImplTest.java @@ -21,11 +21,7 @@ class AlertClientImplTest { .build(); for (int i = 0; i < 15; i++) { - alertClient.post(AlertClient.Alert.builder() - .ex(new RuntimeException()) - .key("keykeykey") - .message("messagemessagemessagemessage") - .build()); + alertClient.post("key", new RuntimeException(), "message"); } } From 7d531ac8769a8d3e6f3de5f1bcbbfb35e0a03592 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 5 Jul 2024 17:38:27 +0800 Subject: [PATCH 26/41] =?UTF-8?q?feat:=20=E5=8E=BB=E6=8E=89=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E7=9A=84stacks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/cn/axzo/foundation/result/ApiResult.java | 9 --------- .../foundation/web/support/alert/AlertClientImpl.java | 4 ++++ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java index 9696d5c..9e175ad 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java +++ b/common-lib/src/main/java/cn/axzo/foundation/result/ApiResult.java @@ -1,6 +1,5 @@ package cn.axzo.foundation.result; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Strings; import lombok.AllArgsConstructor; @@ -27,8 +26,6 @@ public class ApiResult { protected T data; - protected JSONObject stacks; - public static ApiResult success() { return success(null); } @@ -74,12 +71,6 @@ public class ApiResult { return SUCCESS_CODE.equals(getCode()); } - - public ApiResult setStacks(JSONObject stacks) { - this.stacks = stacks; - return this; - } - /** * 根据appId 获取标准的code * 如果code > 100000 则认为可能已经带了appId diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClientImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClientImpl.java index 0a97e3e..cbd0902 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClientImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/AlertClientImpl.java @@ -6,6 +6,7 @@ import com.google.common.collect.Multimaps; import com.google.common.util.concurrent.RateLimiter; import lombok.Builder; import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; import java.time.LocalDateTime; import java.util.Collection; @@ -74,6 +75,9 @@ public class AlertClientImpl implements AlertClient { private void consume() { Map> map = alertsMap.asMap(); + if (CollectionUtils.isEmpty(map)) { + return; + } try { consumer.accept(map); } catch (Exception ex) { From 65ce48ddb80ee7c06613e5f017b4cf70e6c66761 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Mon, 8 Jul 2024 09:33:37 +0800 Subject: [PATCH 27/41] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E4=BA=8B?= =?UTF-8?q?=E4=BB=B6=E6=B6=88=E8=B4=B9=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../consumer/EventHandlerRepository.java | 34 +++++++++++-------- .../consumer/RetryableEventConsumer.java | 1 - 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/EventHandlerRepository.java b/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/EventHandlerRepository.java index 62a3964..b87f7bb 100644 --- a/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/EventHandlerRepository.java +++ b/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/EventHandlerRepository.java @@ -6,7 +6,9 @@ import cn.axzo.foundation.util.FastjsonUtils; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; -import com.google.common.base.*; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; @@ -16,11 +18,11 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.util.AntPathMatcher; import org.springframework.util.CollectionUtils; -import java.util.Objects; -import java.util.Optional; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -29,8 +31,7 @@ import java.util.stream.Collectors; */ @Slf4j public class EventHandlerRepository { - final protected ListMultimap handlers = ArrayListMultimap.create(); - private final Consumer DEFAULT_EXCEPTION_HANDLER = EventHandledWrapper::doPrintException; + protected final ListMultimap handlers = ArrayListMultimap.create(); private AntPathMatcher antPathMatcher; /** @@ -41,18 +42,22 @@ public class EventHandlerRepository { private final Boolean logEnabled; private final Predicate logFilter; + private Consumer globalExceptionHandler; + @Builder public EventHandlerRepository(boolean supportPattern, Boolean logEnabled, Predicate logFilter, Long logElapsedThreshold, - Long maxAllowElapsedMillis) { + Long maxAllowElapsedMillis, + Consumer globalExceptionHandler) { this.logEnabled = Optional.ofNullable(logEnabled).orElse(Boolean.TRUE); this.logFilter = logFilter; this.logElapsedThreshold = Optional.ofNullable(logElapsedThreshold).orElse(3L); this.maxAllowElapsedMillis = Optional.ofNullable(maxAllowElapsedMillis).orElse(10_000L); + this.globalExceptionHandler = Optional.ofNullable(globalExceptionHandler).orElse(EventHandledWrapper::doPrintException); if (supportPattern) { antPathMatcher = new AntPathMatcher(Event.EventCode.SEPARATOR); antPathMatcher.setCachePatterns(true); @@ -65,7 +70,7 @@ public class EventHandlerRepository { //如果传入了key则检查部重复 if (StringUtils.isNoneBlank(handlerHolder.getHandlerKey())) { Set existsHandlerKeys = handlers.values().stream() - .map(e -> e.getHandlerKey()) + .map(EventHandlerContext::getHandlerKey) .filter(StringUtils::isNoneBlank) .collect(Collectors.toSet()); Preconditions.checkArgument(!existsHandlerKeys.contains(handlerHolder.getHandlerKey())); @@ -100,7 +105,7 @@ public class EventHandlerRepository { && context.getHeaders().containsKey(DiffablePayload.DIFF_META_HEADER)) { byte[] diffMetaHeader = context.getHeaders().get(DiffablePayload.DIFF_META_HEADER); List diffMetas = JSON.parseObject( - new String(diffMetaHeader, Charsets.UTF_8), new TypeReference>() { + new String(diffMetaHeader, StandardCharsets.UTF_8), new TypeReference>() { }); differentiator = DiffablePayload.DiffMeta.toDifferentiator(diffMetas); } @@ -135,7 +140,7 @@ public class EventHandlerRepository { } if (elapsed > maxAllowElapsedMillis) { - String msg = String.format("[%s] take too long %d millis for %s to handle %s\nevent=%s", + String msg = String.format("[%s] take too long %d millis for %s to handle %s event=%s", context.getTraceId(), elapsed, handler.getName(), payloadDiffLog, eventLogText); log.warn(msg); } @@ -163,7 +168,7 @@ public class EventHandlerRepository { public boolean batch(List events, EventConsumer.Context context) { Stopwatch stopwatch = Stopwatch.createUnstarted(); List eventHandlers = getEventHandlers(context.getEventCode(), context); - eventHandlers.stream().forEach(handler -> { + eventHandlers.forEach(handler -> { try { stopwatch.start(); String clazzName = handler.getClass().getCanonicalName(); @@ -210,10 +215,11 @@ public class EventHandlerRepository { .handlerKey(context.getHandlerKey()) .exception(exception) .build(); + if (globalExceptionHandler != null) { + globalExceptionHandler.accept(wrapper); + } if (context.getExceptionHandler() != null) { context.getExceptionHandler().accept(wrapper); - } else { - DEFAULT_EXCEPTION_HANDLER.accept(wrapper); } } @@ -243,9 +249,9 @@ public class EventHandlerRepository { } // consumer显式声明了日志开关，则直接使用 if (logEnabled != null) { - return logEnabled && (logFilter == null || logFilter.apply(event)); + return logEnabled && (logFilter == null || logFilter.test(event)); } - return logFilter == null || logFilter.apply(event); + return logFilter == null || logFilter.test(event); } private boolean isSupportPattern() { diff --git a/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/RetryableEventConsumer.java b/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/RetryableEventConsumer.java index ded074f..79dac14 100644 --- a/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/RetryableEventConsumer.java +++ b/event-support-lib/src/main/java/cn/axzo/foundation/event/support/consumer/RetryableEventConsumer.java @@ -39,7 +39,6 @@ public class RetryableEventConsumer implements EventConsumer { Preconditions.checkNotNull(eventConsumer); this.eventConsumer = eventConsumer; exceptionHandler = context -> { - context.doPrintException(); if (Strings.isNullOrEmpty(context.getHandlerKey())) { return; } From cd6a5e4e729c0caba9528cb952f62424703829b4 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Wed, 10 Jul 2024 19:58:45 +0800 Subject: [PATCH 28/41] =?UTF-8?q?feat:=20=E4=BF=AE=E5=A4=8D=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E7=99=BB=E9=99=86=E4=BF=A1=E6=81=AF=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../foundation/web/support/context/AxContextInterceptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java index e495dab..067bd52 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/context/AxContextInterceptor.java @@ -96,7 +96,7 @@ public class AxContextInterceptor implements HandlerInterceptor { try { result = HTTP_CLIENT.get(supplierUrl, RequestParams.FormParams.builder() .headers(ImmutableMap.of("Authorization", authorization, - "terminal", request.getHeader("terminal"))) + "terminal", Strings.nullToEmpty(request.getHeader("terminal")))) .logEnable(true) .build()); } catch (Exception ex) { From f25f3b9a792b8234bdd1efa48b60fcbebe8be7d0 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 11 Jul 2024 09:56:27 +0800 Subject: [PATCH 29/41] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E5=81=A5?= =?UTF-8?q?=E5=BA=B7=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../axzo/foundation/web/support/config/DefaultWebMvcConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java index 1c767d9..6f7bd94 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java @@ -236,6 +236,8 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement //添加一个空的根实现. 避免爬虫访问到根路径后一直打印PageNotFound. 项目中可以覆盖 / 根路径的实现 registry.addStatusController("/", HttpStatus.NO_CONTENT); registry.addStatusController("/favicon.ico", HttpStatus.NO_CONTENT); + //健康检查 + registry.addStatusController("/checkDeath", HttpStatus.OK); super.addViewControllers(registry); } From 4654f80222164d0508257267e0ce77bd76e01560 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 11 Jul 2024 10:42:25 +0800 Subject: [PATCH 30/41] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0rb=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ .reviewboardrc | 1 + 2 files changed, 3 insertions(+) create mode 100644 .reviewboardrc diff --git a/.gitignore b/.gitignore index 23c23c2..ae22693 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,5 @@ build/ /web-support-lib/.flattened-pom.xml /event-support-lib/.flattened-pom.xml /gateway-support-lib/.flattened-pom.xml +/excel-support-lib/.flattened-pom.xml +/redis-support-lib/.flattened-pom.xml diff --git a/.reviewboardrc b/.reviewboardrc new file mode 100644 index 0000000..41e053c --- /dev/null +++ b/.reviewboardrc @@ -0,0 +1 @@ +REPOSITORY = 'axzo-lib-box' \ No newline at end of file From af1ad3cda27f3a5706e99fb605afeca527be08c9 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 11 Jul 2024 15:25:29 +0800 Subject: [PATCH 31/41] =?UTF-8?q?feat:=20=E9=87=8D=E6=9E=84filter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plugin/impl/RequestFilterHook.java | 477 +----------------- .../impl/filters/ConvertContentFilter.java | 30 ++ .../filters/ConvertListAsObjectFilter.java | 48 ++ .../impl/filters/ConvertListAsPageFilter.java | 46 ++ .../impl/filters/ConvertPageAsListFilter.java | 31 ++ .../impl/filters/CropContentFilter.java | 133 +++++ .../plugin/impl/filters/KeyNoQueryFilter.java | 42 ++ .../impl/filters/ListOrPageRecordsFilter.java | 49 ++ .../impl/filters/MoveInputFieldFilter.java | 82 +++ .../impl/filters/RequestParamCheckFilter.java | 95 ++++ 10 files changed, 577 insertions(+), 456 deletions(-) create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertContentFilter.java create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsObjectFilter.java create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsPageFilter.java create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertPageAsListFilter.java create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/CropContentFilter.java create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/KeyNoQueryFilter.java create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ListOrPageRecordsFilter.java create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/MoveInputFieldFilter.java create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/RequestParamCheckFilter.java diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java index 12d6c1f..1a992bd 100644 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java @@ -4,36 +4,29 @@ import cn.axzo.foundation.gateway.support.entity.GateResponse; import cn.axzo.foundation.gateway.support.entity.ProxyContext; import cn.axzo.foundation.gateway.support.entity.RequestContext; import cn.axzo.foundation.gateway.support.plugin.ProxyHook; -import cn.axzo.foundation.page.IPageReq; -import cn.axzo.foundation.page.PageResp; -import cn.axzo.foundation.result.ResultCode; -import cn.axzo.foundation.util.DataAssembleHelper; +import cn.axzo.foundation.gateway.support.plugin.impl.filters.*; import cn.axzo.foundation.util.FastjsonUtils; import cn.axzo.foundation.web.support.rpc.RequestParams; import cn.axzo.foundation.web.support.rpc.RpcClient; import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.JSONPath; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableMap; import lombok.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.function.Function; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.IntStream; /** * 对请求外部扩展插件。 使用如下： @@ -62,9 +55,6 @@ import java.util.stream.IntStream; * */ @Slf4j -@Builder -@NoArgsConstructor -@AllArgsConstructor public class RequestFilterHook implements ProxyHook { public static final String FILTER_BEAN = "bean"; public static final String BEAN_SEPARATOR = "|"; @@ -74,9 +64,24 @@ public class RequestFilterHook implements ProxyHook { * key: bean 名称 * value: transformer 对象。 */ - @NonNull private Function filterBeanResolver; + private static final Map DEFAULT_FILTERS = ImmutableMap.builder() + .put("convertListAsObjectFilter", new ConvertListAsObjectFilter()) + .put("convertListAsPageFilter", new ConvertListAsPageFilter()) + .put("convertPageAsListFilter", new ConvertPageAsListFilter()) + .put("cropContentFilter", new CropContentFilter()) + .put("requestParamCheckFilter", new RequestParamCheckFilter()) + .put("moveInputFieldFilter", new MoveInputFieldFilter()) + .put("keyNoQueryFilter", new KeyNoQueryFilter()) + .build(); + + + public RequestFilterHook(@NonNull Function filterBeanResolver) { + //优先从外部传入, 没有时则从default获取 + this.filterBeanResolver = s -> Optional.ofNullable(filterBeanResolver.apply(s)).orElse(DEFAULT_FILTERS.get(s)); + } + @Override public RequestParams preRequest(RequestContext reqContext, ProxyContext proxyContext, RpcClient rpcClient, String postURL, RequestParams postParams) { if (proxyContext.getParameterFilter() == null || proxyContext.getParameterFilter().getInFilterExtends().isEmpty()) { @@ -183,446 +188,6 @@ public class RequestFilterHook implements ProxyHook { } } - /** - * 后台页面中通过关键字段查询时, 需要移除其他字段的查询场景. 添加通用的请求过滤器来处理此类需求. - * 使用时, 需要单独注入. - */ - public final static class KeyNoQueryFilter implements RequestFilter { - @Override - public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) { - if (params == null) { - return params; - } - - Set keyNoFields = Splitter.on(",").omitEmptyStrings().trimResults() - .splitToStream(Strings.nullToEmpty(config.getString("keyNoFields"))) - .collect(Collectors.toSet()); - Set removeFields = Splitter.on(",").omitEmptyStrings().trimResults() - .splitToStream(Strings.nullToEmpty(config.getString("removeFields"))) - .collect(Collectors.toSet()); - if (keyNoFields.isEmpty() || removeFields.isEmpty()) { - return params; - } - - JSONObject paramsJson = (JSONObject) params; - if (paramsJson.keySet().stream().noneMatch(keyNoFields::contains)) { - return paramsJson; - } - - removeFields.forEach(paramsJson::remove); - return paramsJson; - } - } - - public abstract static class ListOrPageRecordsFilter implements RequestFilter { - @Override - public JSON filterOut(RequestContext reqContext, JSON response) { - return filterOut(reqContext, response, new JSONObject()); - } - - @Override - public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { - if (!(response instanceof JSONObject)) { - return response; - } - JSONObject jsonResp = (JSONObject) response; - Object content = jsonResp.get("data"); - if (!(content instanceof JSONObject) && !(content instanceof JSONArray)) { - return response; - } - - if (content instanceof JSONArray) { - List records = ((JSONArray) content).toJavaList(JSONObject.class); - if (CollectionUtils.isEmpty(records)) { - return response; - } - JSONArray filtered = new JSONArray().fluentAddAll(filterRecords(reqContext, records, config)); - return jsonResp.fluentPut("data", filtered); - } - - JSONObject page = (JSONObject) content; - JSONArray records = page.getJSONArray("data"); - if (CollectionUtils.isEmpty(records)) { - return response; - } - List filtered = filterRecords(reqContext, records.toJavaList(JSONObject.class), config); - page.put("data", new JSONArray().fluentAddAll(filtered)); - return response; - } - - public abstract List filterRecords(RequestContext reqContext, List records, JSONObject config); - } - - public abstract static class ConvertContentFilter implements RequestFilter { - @Override - public JSON filterOut(RequestContext reqContext, JSON response) { - return filterOut(reqContext, response, new JSONObject()); - } - - @Override - public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { - if (!(response instanceof JSONObject)) { - return response; - } - JSONObject jsonResp = (JSONObject) response; - Object content = jsonResp.get("data"); - if (content == null) { - return response; - } - Preconditions.checkArgument(content instanceof JSON, "ConvertContentFilter不支持原始response.content非json的情况"); - return jsonResp.fluentPut("data", filterContent(reqContext, (JSON) content, config)); - } - - public abstract JSON filterContent(RequestContext reqContext, JSON content, JSONObject config); - } - - public static class ConvertPageAsListFilter implements RequestFilter { - @Override - public JSON filterOut(RequestContext reqContext, JSON response) { - return filterOut(reqContext, response, new JSONObject()); - } - - @Override - public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { - if (!(response instanceof JSONObject)) { - return response; - } - JSONObject jsonResp = (JSONObject) response; - Object content = jsonResp.get("data"); - if (!(content instanceof JSONObject)) { - return response; - } - - JSONArray records = ((JSONObject) content).getJSONArray("data"); - return jsonResp.fluentPut("data", Optional.ofNullable(records).orElseGet(JSONArray::new)); - } - } - - public static class ConvertListAsObjectFilter implements RequestFilter { - @Override - public JSON filterOut(RequestContext reqContext, JSON response) { - return filterOut(reqContext, response, new JSONObject()); - } - - @Override - public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { - if (!(response instanceof JSONObject)) { - return response; - } - JSONObject jsonResp = (JSONObject) response; - Object content = jsonResp.get("data"); - - JSONArray arrayContent = null; - if (content instanceof JSONArray) { - arrayContent = (JSONArray) content; - } else if (content instanceof JSONObject) { - JSONArray records = ((JSONObject) content).getJSONArray("data"); - arrayContent = records == null ? new JSONArray() : records; - } else { - return response; - } - JSONObject obj = IntStream.range(0, arrayContent.size()) - .mapToObj(arrayContent::getJSONObject) - .findFirst() - .orElseGet(JSONObject::new); - return jsonResp.fluentPut("data", filterObject(reqContext, obj, config)); - } - - public JSONObject filterObject(RequestContext reqContext, JSONObject obj) { - return obj; - } - - public JSONObject filterObject(RequestContext reqContext, JSONObject obj, JSONObject config) { - return filterObject(reqContext, obj); - } - } - - public static class ConvertListAsPageFilter implements RequestFilter { - @Override - public JSON filterOut(RequestContext reqContext, JSON response) { - return filterOut(reqContext, response, new JSONObject()); - } - - @Override - public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { - if (!(response instanceof JSONObject)) { - return response; - } - JSONObject jsonResp = (JSONObject) response; - Object content = jsonResp.get("data"); - if (!(content instanceof JSONArray)) { - return response; - } - - List records = ((JSONArray) content).toJavaList(JSONObject.class); - PageResp page = PageResp.builder().total(records.size()).build(); - - IPageReq pageParam = JSONObject.parseObject(reqContext.getRequestBody(), IPageReq.class); - Optional.ofNullable(pageParam).map(IPageReq::getPage).ifPresent(page::setCurrent); - Optional.ofNullable(pageParam).map(IPageReq::getPageSize).ifPresent(page::setSize); - - page.setData(records.stream() - .skip((page.getCurrent() - 1) * page.getSize()) - .limit(page.getSize()) - .collect(Collectors.toList())); - - return jsonResp.fluentPut("data", page); - } - } - - public static class RequestParamCheckFilter implements RequestFilter { - @Override - public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) { - boolean isConfigRulePresent = config != null && !config.isEmpty() && config.containsKey("rules"); - Preconditions.checkArgument(isConfigRulePresent, "RequestParamCheckFilter必须指定检查规则"); - Preconditions.checkArgument(JSONObject.class.isAssignableFrom(params.getClass()), - "RequestParamCheckFilter只能检查JSONObject类型参数"); - //spring list默认读取为 map(key=0...N). 将values转换为Rule - List rules = config.getJSONObject("rules").entrySet().stream() - .map(e -> ((JSONObject) e.getValue()).toJavaObject(Rule.class)) - .collect(Collectors.toList()); - rules.stream().forEach(p -> - Preconditions.checkArgument(!StringUtils.isAllBlank(p.getField(), p.getJsonPath()), - "RequestParamCheckFilter规则错误field, jsonPath不能都为空")); - Operator operator = Optional.ofNullable(config.getString("operator")) - .map(e -> Operator.valueOf(e)).orElse(Operator.AND); - - List> errors = rules.stream() - .map(rule -> rule.check((JSONObject) params)) - .collect(Collectors.toList()); - //如果operator=AND. & 任意一个检查错误 则告警 - if (operator == Operator.AND && errors.stream().anyMatch(Optional::isPresent)) { - throw ResultCode.INVALID_PARAMS.toException(errors.stream().filter(Optional::isPresent).findFirst().get().get()); - } - //如果operator=OR. & 所有检查都是错误 则告警 - if (operator == Operator.OR && errors.stream().allMatch(Optional::isPresent)) { - throw ResultCode.INVALID_PARAMS.toException(errors.stream().filter(Optional::isPresent).findFirst().get().get()); - } - - return params; - } - - @Data - private static class Rule { - String jsonPath; - String field; - boolean required; - String regex; - private transient Pattern pattern; - - /** 检查值并返回错误信息, 正确则返回empty */ - protected Optional check(JSONObject param) { - Object value = getValue(param); - if (required && value == null) { - return Optional.of(field + "不能为空"); - } - Optional pattern = tryGetPattern(); - if (pattern.isPresent() && value != null && !pattern.get().matcher(value.toString()).find()) { - return Optional.of(field + "参数校验失败:" + regex); - } - return Optional.empty(); - } - - /** 优先根据field获取value. 否则根据jsonPath */ - private Object getValue(JSONObject param) { - if (!Strings.isNullOrEmpty(field)) { - return param.get(field); - } - return JSONPath.eval(param, jsonPath); - } - - private Optional tryGetPattern() { - if (Strings.isNullOrEmpty(regex)) { - return Optional.empty(); - } - if (pattern == null) { - pattern = Pattern.compile(regex); - } - return Optional.of(pattern); - } - } - - public enum Operator { - AND, - OR; - } - } - - /** - * 裁剪返回的content内容 - * 仅支持: - * 1.content是JSONObject - * 2.content是JSONArray，且其中是JSONObject - * 3.content是分页返回对象，即{"data":[{}]} - */ - public static class CropContentFilter extends ConvertContentFilter implements RequestFilter { - @Override - public JSON filterOut(RequestContext reqContext, JSON response) { - throw new UnsupportedOperationException("使用CropContentFilter必须指定裁剪配置"); - } - - @Override - public JSON filterContent(RequestContext reqContext, JSON content, JSONObject config) { - // see class: CropConfig - if (CollectionUtils.isEmpty(config)) { - throw new UnsupportedOperationException("使用CropContentFilter必须指定裁剪配置"); - } - return cropContent(content, config); - } - - private JSON cropContent(JSON content, JSONObject config) { - // content是json数组 - if (content instanceof JSONArray) { - return cropJSONArrayContent((JSONArray) content, config); - } - // content是json对象 - if (content instanceof JSONObject) { - JSONObject contentJSONObject = (JSONObject) content; - // 可能是分页content，此时支持裁剪分页列表中的json对象 - if (contentJSONObject.containsKey("data")) { - contentJSONObject.put("data", - cropJSONArrayContent(contentJSONObject.getJSONArray("data"), config)); - } - return doCrop(contentJSONObject, config); - } - return content; - } - - private JSONArray cropJSONArrayContent(JSONArray content, JSONObject config) { - // 只考虑JSONArray中是JSONObject对象的情况 - List contentList = content.stream() - .filter(obj -> obj instanceof JSONObject) - .map(obj -> JSONObject.parseObject(JSON.toJSONString(obj))) - .collect(Collectors.toList()); - if (contentList.isEmpty()) { - return content; - } - return JSON.parseArray(JSON.toJSONString(contentList.stream() - .map(c -> doCrop(c, config)) - .collect(Collectors.toList()))); - } - - /** - * 裁剪data中的字段 - * - * @param data - * @param config {@link CropConfig} - * @return - */ - private static JSONObject doCrop(JSONObject data, JSONObject config) { - if (CollectionUtils.isEmpty(config) || CollectionUtils.isEmpty(data)) { - return data; - } - CropConfig cropConfig = config.toJavaObject(CropConfig.class); - // 优先用includeKeys - if (!CollectionUtils.isEmpty(cropConfig.getIncludeKeys())) { - return DataAssembleHelper.filterBean(data, cropConfig.getIncludeKeys(), true); - } - if (!CollectionUtils.isEmpty(cropConfig.getExcludeKeys())) { - return DataAssembleHelper.filterBean(data, cropConfig.getExcludeKeys(), false); - } - return data; - } - - /** - * 裁剪content时的配置 - * - * @see CropContentFilter - */ - @Data - @Builder - @NoArgsConstructor - @AllArgsConstructor - private static class CropConfig { - /** - * 希望只包含这些key，只支持第一层key - * 如果有值，将忽略excludeKeys - */ - private String includeKeys; - /** - * 希望排除的key，只支持第一层key - */ - private String excludeKeys; - - public Set getIncludeKeys() { - if (Strings.isNullOrEmpty(includeKeys)) { - return Collections.emptySet(); - } - return ImmutableSet.copyOf(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(includeKeys)); - } - - public Set getExcludeKeys() { - if (Strings.isNullOrEmpty(excludeKeys)) { - return Collections.emptySet(); - } - return ImmutableSet.copyOf(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(excludeKeys)); - } - } - } - - /** - * 移动请求输入参数中字段到指定字段下. - */ - public final static class MoveInputFieldFilter implements RequestFilter { - @Override - public JSON filterIn(RequestContext reqContext, JSON params, JSONObject configJSON) { - if (params == null) { - return params; - } - - Config config = configJSON.toJavaObject(Config.class); - if (!config.isValid()) { - throw new IllegalArgumentException("不正确的配置参数"); - } - - Set sourceFields = config.getSourceFields(); - JSONObject sourceValue = DataAssembleHelper.filterBean(params, sourceFields); - JSONObject res = DataAssembleHelper.filterBean(params, sourceFields, false); - Preconditions.checkArgument(!res.containsKey(config.getTargetField()), "目标字段已经存在, 不能被覆盖"); - - res.put(config.getTargetField(), config.getTargetFieldType() == Config.FieldType.PROPERTY - // 如果是把字段move到listField作为它的第一个元素 - ? sourceValue : Lists.newArrayList(sourceValue)); - return res; - } - - @Data - @Builder - @NoArgsConstructor - @AllArgsConstructor - static class Config { - private String targetField; - private String sourceFields; - private FieldType targetFieldType; - - public Set getSourceFields() { - return Splitter.on(",").omitEmptyStrings().trimResults() - .splitToStream(Strings.nullToEmpty(sourceFields)) - .collect(Collectors.toSet()); - } - - boolean isValid() { - return !getSourceFields().isEmpty() && !StringUtils.isBlank(targetField); - } - - public FieldType getTargetFieldType() { - if (targetFieldType == null) { - return FieldType.PROPERTY; - } - return targetFieldType; - } - - enum FieldType { - PROPERTY, - /** - * 将Field看做一个列表, 将 moveields 放在toField下作为第一个元素. - */ - LIST; - } - - } - } - @Data @Builder @NoArgsConstructor diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertContentFilter.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertContentFilter.java new file mode 100644 index 0000000..33d974f --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertContentFilter.java @@ -0,0 +1,30 @@ +package cn.axzo.foundation.gateway.support.plugin.impl.filters; + +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Preconditions; + +public abstract class ConvertContentFilter implements RequestFilterHook.RequestFilter { + @Override + public JSON filterOut(RequestContext reqContext, JSON response) { + return filterOut(reqContext, response, new JSONObject()); + } + + @Override + public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { + if (!(response instanceof JSONObject)) { + return response; + } + JSONObject jsonResp = (JSONObject) response; + Object content = jsonResp.get("data"); + if (content == null) { + return response; + } + Preconditions.checkArgument(content instanceof JSON, "ConvertContentFilter不支持原始response.content非json的情况"); + return jsonResp.fluentPut("data", filterContent(reqContext, (JSON) content, config)); + } + + public abstract JSON filterContent(RequestContext reqContext, JSON content, JSONObject config); +} \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsObjectFilter.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsObjectFilter.java new file mode 100644 index 0000000..34c3796 --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsObjectFilter.java @@ -0,0 +1,48 @@ +package cn.axzo.foundation.gateway.support.plugin.impl.filters; + +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +import java.util.stream.IntStream; + +public class ConvertListAsObjectFilter implements RequestFilterHook.RequestFilter { + @Override + public JSON filterOut(RequestContext reqContext, JSON response) { + return filterOut(reqContext, response, new JSONObject()); + } + + @Override + public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { + if (!(response instanceof JSONObject)) { + return response; + } + JSONObject jsonResp = (JSONObject) response; + Object content = jsonResp.get("data"); + + JSONArray arrayContent = null; + if (content instanceof JSONArray) { + arrayContent = (JSONArray) content; + } else if (content instanceof JSONObject) { + JSONArray records = ((JSONObject) content).getJSONArray("data"); + arrayContent = records == null ? new JSONArray() : records; + } else { + return response; + } + JSONObject obj = IntStream.range(0, arrayContent.size()) + .mapToObj(arrayContent::getJSONObject) + .findFirst() + .orElseGet(JSONObject::new); + return jsonResp.fluentPut("data", filterObject(reqContext, obj, config)); + } + + public JSONObject filterObject(RequestContext reqContext, JSONObject obj) { + return obj; + } + + public JSONObject filterObject(RequestContext reqContext, JSONObject obj, JSONObject config) { + return filterObject(reqContext, obj); + } +} \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsPageFilter.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsPageFilter.java new file mode 100644 index 0000000..25e62ad --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertListAsPageFilter.java @@ -0,0 +1,46 @@ +package cn.axzo.foundation.gateway.support.plugin.impl.filters; + +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook; +import cn.axzo.foundation.page.IPageReq; +import cn.axzo.foundation.page.PageResp; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class ConvertListAsPageFilter implements RequestFilterHook.RequestFilter { + @Override + public JSON filterOut(RequestContext reqContext, JSON response) { + return filterOut(reqContext, response, new JSONObject()); + } + + @Override + public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { + if (!(response instanceof JSONObject)) { + return response; + } + JSONObject jsonResp = (JSONObject) response; + Object content = jsonResp.get("data"); + if (!(content instanceof JSONArray)) { + return response; + } + + List records = ((JSONArray) content).toJavaList(JSONObject.class); + PageResp page = PageResp.builder().total(records.size()).build(); + + IPageReq pageParam = JSONObject.parseObject(reqContext.getRequestBody(), IPageReq.class); + Optional.ofNullable(pageParam).map(IPageReq::getPage).ifPresent(page::setCurrent); + Optional.ofNullable(pageParam).map(IPageReq::getPageSize).ifPresent(page::setSize); + + page.setData(records.stream() + .skip((page.getCurrent() - 1) * page.getSize()) + .limit(page.getSize()) + .collect(Collectors.toList())); + + return jsonResp.fluentPut("data", page); + } +} \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertPageAsListFilter.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertPageAsListFilter.java new file mode 100644 index 0000000..f1a0403 --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ConvertPageAsListFilter.java @@ -0,0 +1,31 @@ +package cn.axzo.foundation.gateway.support.plugin.impl.filters; + +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +import java.util.Optional; + +public class ConvertPageAsListFilter implements RequestFilterHook.RequestFilter { + @Override + public JSON filterOut(RequestContext reqContext, JSON response) { + return filterOut(reqContext, response, new JSONObject()); + } + + @Override + public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { + if (!(response instanceof JSONObject)) { + return response; + } + JSONObject jsonResp = (JSONObject) response; + Object content = jsonResp.get("data"); + if (!(content instanceof JSONObject)) { + return response; + } + + JSONArray records = ((JSONObject) content).getJSONArray("data"); + return jsonResp.fluentPut("data", Optional.ofNullable(records).orElseGet(JSONArray::new)); + } +} \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/CropContentFilter.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/CropContentFilter.java new file mode 100644 index 0000000..a5b708f --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/CropContentFilter.java @@ -0,0 +1,133 @@ +package cn.axzo.foundation.gateway.support.plugin.impl.filters; + +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook; +import cn.axzo.foundation.util.DataAssembleHelper; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.util.CollectionUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 裁剪返回的content内容 + * 仅支持: + * 1.content是JSONObject + * 2.content是JSONArray，且其中是JSONObject + * 3.content是分页返回对象，即{"data":[{}]} + */ +public class CropContentFilter extends ConvertContentFilter implements RequestFilterHook.RequestFilter { + @Override + public JSON filterOut(RequestContext reqContext, JSON response) { + throw new UnsupportedOperationException("使用CropContentFilter必须指定裁剪配置"); + } + + @Override + public JSON filterContent(RequestContext reqContext, JSON content, JSONObject config) { + // see class: CropConfig + if (CollectionUtils.isEmpty(config)) { + throw new UnsupportedOperationException("使用CropContentFilter必须指定裁剪配置"); + } + return cropContent(content, config); + } + + private JSON cropContent(JSON content, JSONObject config) { + // content是json数组 + if (content instanceof JSONArray) { + return cropJSONArrayContent((JSONArray) content, config); + } + // content是json对象 + if (content instanceof JSONObject) { + JSONObject contentJSONObject = (JSONObject) content; + // 可能是分页content，此时支持裁剪分页列表中的json对象 + if (contentJSONObject.containsKey("data")) { + contentJSONObject.put("data", + cropJSONArrayContent(contentJSONObject.getJSONArray("data"), config)); + } + return doCrop(contentJSONObject, config); + } + return content; + } + + private JSONArray cropJSONArrayContent(JSONArray content, JSONObject config) { + // 只考虑JSONArray中是JSONObject对象的情况 + List contentList = content.stream() + .filter(obj -> obj instanceof JSONObject) + .map(obj -> JSONObject.parseObject(JSON.toJSONString(obj))) + .collect(Collectors.toList()); + if (contentList.isEmpty()) { + return content; + } + return JSON.parseArray(JSON.toJSONString(contentList.stream() + .map(c -> doCrop(c, config)) + .collect(Collectors.toList()))); + } + + /** + * 裁剪data中的字段 + * + * @param data + * @param config {@link CropConfig} + * @return + */ + private static JSONObject doCrop(JSONObject data, JSONObject config) { + if (CollectionUtils.isEmpty(config) || CollectionUtils.isEmpty(data)) { + return data; + } + CropConfig cropConfig = config.toJavaObject(CropConfig.class); + // 优先用includeKeys + if (!CollectionUtils.isEmpty(cropConfig.getIncludeKeys())) { + return DataAssembleHelper.filterBean(data, cropConfig.getIncludeKeys(), true); + } + if (!CollectionUtils.isEmpty(cropConfig.getExcludeKeys())) { + return DataAssembleHelper.filterBean(data, cropConfig.getExcludeKeys(), false); + } + return data; + } + + /** + * 裁剪content时的配置 + * + * @see CropContentFilter + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + private static class CropConfig { + /** + * 希望只包含这些key，只支持第一层key + * 如果有值，将忽略excludeKeys + */ + private String includeKeys; + /** + * 希望排除的key，只支持第一层key + */ + private String excludeKeys; + + public Set getIncludeKeys() { + if (Strings.isNullOrEmpty(includeKeys)) { + return Collections.emptySet(); + } + return ImmutableSet.copyOf(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(includeKeys)); + } + + public Set getExcludeKeys() { + if (Strings.isNullOrEmpty(excludeKeys)) { + return Collections.emptySet(); + } + return ImmutableSet.copyOf(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(excludeKeys)); + } + } +} \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/KeyNoQueryFilter.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/KeyNoQueryFilter.java new file mode 100644 index 0000000..00087f2 --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/KeyNoQueryFilter.java @@ -0,0 +1,42 @@ +package cn.axzo.foundation.gateway.support.plugin.impl.filters; + +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; + +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 后台页面中通过关键字段查询时, 需要移除其他字段的查询场景. 添加通用的请求过滤器来处理此类需求. + * 使用时, 需要单独注入. + */ +public class KeyNoQueryFilter implements RequestFilterHook.RequestFilter { + @Override + public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) { + if (params == null) { + return params; + } + + Set keyNoFields = Splitter.on(",").omitEmptyStrings().trimResults() + .splitToStream(Strings.nullToEmpty(config.getString("keyNoFields"))) + .collect(Collectors.toSet()); + Set removeFields = Splitter.on(",").omitEmptyStrings().trimResults() + .splitToStream(Strings.nullToEmpty(config.getString("removeFields"))) + .collect(Collectors.toSet()); + if (keyNoFields.isEmpty() || removeFields.isEmpty()) { + return params; + } + + JSONObject paramsJson = (JSONObject) params; + if (paramsJson.keySet().stream().noneMatch(keyNoFields::contains)) { + return paramsJson; + } + + removeFields.forEach(paramsJson::remove); + return paramsJson; + } +} \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ListOrPageRecordsFilter.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ListOrPageRecordsFilter.java new file mode 100644 index 0000000..8428e38 --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/ListOrPageRecordsFilter.java @@ -0,0 +1,49 @@ +package cn.axzo.foundation.gateway.support.plugin.impl.filters; + +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.springframework.util.CollectionUtils; + +import java.util.List; + +public abstract class ListOrPageRecordsFilter implements RequestFilterHook.RequestFilter { + @Override + public JSON filterOut(RequestContext reqContext, JSON response) { + return filterOut(reqContext, response, new JSONObject()); + } + + @Override + public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { + if (!(response instanceof JSONObject)) { + return response; + } + JSONObject jsonResp = (JSONObject) response; + Object content = jsonResp.get("data"); + if (!(content instanceof JSONObject) && !(content instanceof JSONArray)) { + return response; + } + + if (content instanceof JSONArray) { + List records = ((JSONArray) content).toJavaList(JSONObject.class); + if (CollectionUtils.isEmpty(records)) { + return response; + } + JSONArray filtered = new JSONArray().fluentAddAll(filterRecords(reqContext, records, config)); + return jsonResp.fluentPut("data", filtered); + } + + JSONObject page = (JSONObject) content; + JSONArray records = page.getJSONArray("data"); + if (CollectionUtils.isEmpty(records)) { + return response; + } + List filtered = filterRecords(reqContext, records.toJavaList(JSONObject.class), config); + page.put("data", new JSONArray().fluentAddAll(filtered)); + return response; + } + + public abstract List filterRecords(RequestContext reqContext, List records, JSONObject config); +} \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/MoveInputFieldFilter.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/MoveInputFieldFilter.java new file mode 100644 index 0000000..ed4feb7 --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/MoveInputFieldFilter.java @@ -0,0 +1,82 @@ +package cn.axzo.foundation.gateway.support.plugin.impl.filters; + +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook; +import cn.axzo.foundation.util.DataAssembleHelper; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 移动请求输入参数中字段到指定字段下. + */ +public class MoveInputFieldFilter implements RequestFilterHook.RequestFilter { + @Override + public JSON filterIn(RequestContext reqContext, JSON params, JSONObject configJSON) { + if (params == null) { + return params; + } + + Config config = configJSON.toJavaObject(Config.class); + if (!config.isValid()) { + throw new IllegalArgumentException("不正确的配置参数"); + } + + Set sourceFields = config.getSourceFields(); + JSONObject sourceValue = DataAssembleHelper.filterBean(params, sourceFields); + JSONObject res = DataAssembleHelper.filterBean(params, sourceFields, false); + Preconditions.checkArgument(!res.containsKey(config.getTargetField()), "目标字段已经存在, 不能被覆盖"); + + res.put(config.getTargetField(), config.getTargetFieldType() == Config.FieldType.PROPERTY + // 如果是把字段move到listField作为它的第一个元素 + ? sourceValue : Lists.newArrayList(sourceValue)); + return res; + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + static class Config { + private String targetField; + private String sourceFields; + private Config.FieldType targetFieldType; + + public Set getSourceFields() { + return Splitter.on(",").omitEmptyStrings().trimResults() + .splitToStream(Strings.nullToEmpty(sourceFields)) + .collect(Collectors.toSet()); + } + + boolean isValid() { + return !getSourceFields().isEmpty() && !StringUtils.isBlank(targetField); + } + + public Config.FieldType getTargetFieldType() { + if (targetFieldType == null) { + return Config.FieldType.PROPERTY; + } + return targetFieldType; + } + + enum FieldType { + PROPERTY, + /** + * 将Field看做一个列表, 将 moveields 放在toField下作为第一个元素. + */ + LIST; + } + + } +} \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/RequestParamCheckFilter.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/RequestParamCheckFilter.java new file mode 100644 index 0000000..7d74f89 --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/filters/RequestParamCheckFilter.java @@ -0,0 +1,95 @@ +package cn.axzo.foundation.gateway.support.plugin.impl.filters; + +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.plugin.impl.RequestFilterHook; +import cn.axzo.foundation.result.ResultCode; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSONPath; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class RequestParamCheckFilter implements RequestFilterHook.RequestFilter { + @Override + public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) { + boolean isConfigRulePresent = config != null && !config.isEmpty() && config.containsKey("rules"); + Preconditions.checkArgument(isConfigRulePresent, "RequestParamCheckFilter必须指定检查规则"); + Preconditions.checkArgument(JSONObject.class.isAssignableFrom(params.getClass()), + "RequestParamCheckFilter只能检查JSONObject类型参数"); + //spring list默认读取为 map(key=0...N). 将values转换为Rule + List rules = config.getJSONObject("rules").entrySet().stream() + .map(e -> ((JSONObject) e.getValue()).toJavaObject(Rule.class)) + .collect(Collectors.toList()); + rules.stream().forEach(p -> + Preconditions.checkArgument(!StringUtils.isAllBlank(p.getField(), p.getJsonPath()), + "RequestParamCheckFilter规则错误field, jsonPath不能都为空")); + Operator operator = Optional.ofNullable(config.getString("operator")) + .map(e -> Operator.valueOf(e)).orElse(Operator.AND); + + List> errors = rules.stream() + .map(rule -> rule.check((JSONObject) params)) + .collect(Collectors.toList()); + //如果operator=AND. & 任意一个检查错误 则告警 + if (operator == Operator.AND && errors.stream().anyMatch(Optional::isPresent)) { + throw ResultCode.INVALID_PARAMS.toException(errors.stream().filter(Optional::isPresent).findFirst().get().get()); + } + //如果operator=OR. & 所有检查都是错误 则告警 + if (operator == Operator.OR && errors.stream().allMatch(Optional::isPresent)) { + throw ResultCode.INVALID_PARAMS.toException(errors.stream().filter(Optional::isPresent).findFirst().get().get()); + } + + return params; + } + + @Data + private static class Rule { + String jsonPath; + String field; + boolean required; + String regex; + private transient Pattern pattern; + + /** 检查值并返回错误信息, 正确则返回empty */ + protected Optional check(JSONObject param) { + Object value = getValue(param); + if (required && value == null) { + return Optional.of(field + "不能为空"); + } + Optional pattern = tryGetPattern(); + if (pattern.isPresent() && value != null && !pattern.get().matcher(value.toString()).find()) { + return Optional.of(field + "参数校验失败:" + regex); + } + return Optional.empty(); + } + + /** 优先根据field获取value. 否则根据jsonPath */ + private Object getValue(JSONObject param) { + if (!Strings.isNullOrEmpty(field)) { + return param.get(field); + } + return JSONPath.eval(param, jsonPath); + } + + private Optional tryGetPattern() { + if (Strings.isNullOrEmpty(regex)) { + return Optional.empty(); + } + if (pattern == null) { + pattern = Pattern.compile(regex); + } + return Optional.of(pattern); + } + } + + public enum Operator { + AND, + OR; + } +} \ No newline at end of file From 8545fb233a8cedc4af3ef0bd718a8720050fc8a9 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 11 Jul 2024 15:27:29 +0800 Subject: [PATCH 32/41] =?UTF-8?q?feat:=20=E8=AE=BE=E7=BD=AEbuilder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gateway/support/plugin/impl/RequestFilterHook.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java index 1a992bd..20aafdc 100644 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java @@ -76,7 +76,7 @@ public class RequestFilterHook implements ProxyHook { .put("keyNoQueryFilter", new KeyNoQueryFilter()) .build(); - + @Builder public RequestFilterHook(@NonNull Function filterBeanResolver) { //优先从外部传入, 没有时则从default获取 this.filterBeanResolver = s -> Optional.ofNullable(filterBeanResolver.apply(s)).orElse(DEFAULT_FILTERS.get(s)); From 537ccadd0a33c1073a3c5bbaf84195709385eec7 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 11 Jul 2024 16:36:50 +0800 Subject: [PATCH 33/41] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0live=20profile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/cn/axzo/foundation/enums/AppEnvEnum.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common-lib/src/main/java/cn/axzo/foundation/enums/AppEnvEnum.java b/common-lib/src/main/java/cn/axzo/foundation/enums/AppEnvEnum.java index 2725f35..823ab25 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/enums/AppEnvEnum.java +++ b/common-lib/src/main/java/cn/axzo/foundation/enums/AppEnvEnum.java @@ -7,5 +7,6 @@ public enum AppEnvEnum { test, pre, prd, - master + master, + live } From 944fcaf70adf120d0f2d83e82b704ef533a9eaf4 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 11 Jul 2024 16:59:13 +0800 Subject: [PATCH 34/41] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96rpc=E6=9E=84?= =?UTF-8?q?=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web/support/apps/AppCenterImpl.java | 6 ++--- .../web/support/rpc/OkHttpClientImpl.java | 9 +++---- .../web/support/rpc/RpcClientImpl.java | 25 ++++++++----------- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java index 87fc58b..cc4db24 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java @@ -3,7 +3,6 @@ package cn.axzo.foundation.web.support.apps; import cn.axzo.foundation.page.PageResp; import cn.axzo.foundation.util.PageUtils; import cn.axzo.foundation.web.support.TimerRefreshCache; -import cn.axzo.foundation.web.support.rpc.RequestProxy; import cn.axzo.foundation.web.support.rpc.RpcClient; import cn.axzo.foundation.web.support.rpc.RpcClientImpl; import com.alibaba.fastjson.JSONArray; @@ -39,11 +38,12 @@ public class AppCenterImpl implements AppCenter { @Builder public AppCenterImpl(ScheduledThreadPoolExecutor executor, String debugHost, - Map debugAppRoutes) { + Map debugAppRoutes, + RpcClient rpcClient) { Objects.requireNonNull(executor); - this.rpcClient = RpcClientImpl.builder().requestProxy(RequestProxy.SIMPLE_PROXY).build(); + this.rpcClient = Optional.ofNullable(rpcClient).orElseGet(() -> RpcClientImpl.builder().build()); this.appCache = new TimerRefreshCache<>("appCenterCache", INITIAL_DELAY_MILLIS, REFRESH_INTERVAL_MILLIS, executor, this::loadApps); this.debugHost = debugHost; diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/OkHttpClientImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/OkHttpClientImpl.java index d2abf60..389e228 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/OkHttpClientImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/OkHttpClientImpl.java @@ -22,12 +22,11 @@ import java.util.function.Function; public class OkHttpClientImpl implements HttpClient { private OkHttpClient okHttpClient; private Call.Factory callFactory; - private Config config; - private List interceptorList; - private Set logResponseHeaderNames; + private final List interceptorList; + private final Set logResponseHeaderNames; - private final static long MAX_RESPONSE_LOG_LENGTH = 10_240L; + private static final long MAX_RESPONSE_LOG_LENGTH = 10_240L; @Builder public OkHttpClientImpl(Config config, List interceptorList, @@ -169,7 +168,7 @@ public class OkHttpClientImpl implements HttpClient { .connectionPool(new ConnectionPool(config.getMaxIdleConnections(), config.getKeepAliveMinutes(), TimeUnit.MINUTES)) .dispatcher(dispatcher); - interceptorList.forEach(interceptor -> builder.addInterceptor(interceptor)); + interceptorList.forEach(builder::addInterceptor); if (null != clientBuilder) { clientBuilder.accept(builder); diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java index 80a4237..078e9fd 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/rpc/RpcClientImpl.java @@ -25,7 +25,7 @@ public class RpcClientImpl implements RpcClient { @Getter protected HttpClient httpClient; protected RequestProxy requestProxy; - private static final Set AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo"); + private static final Set AXZO_HEADERS = ImmutableSet.of("workspaceId", "ouId", "Authorization", "terminal", "userinfo", "ctxLogId", "traceId"); // XXX: http/2会把所有Header都转成小写, 历史定义的Header都是大写的，在http/2协议下会透传失败。 private static final TreeSet CASE_INSENSITIVE_AXZO_HEADERS = AXZO_HEADERS.stream() .collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER))); @@ -36,7 +36,7 @@ public class RpcClientImpl implements RpcClient { .map(request -> Collections.list(request.getHeaderNames()).stream() // 通过http2协议的请求，默认会把大写转成小写，如"_SESSION_OBJECT" -> "_session_object"，导致session无法透传，下一跳需要通过session验签会失败 .filter(p -> CASE_INSENSITIVE_AXZO_HEADERS.contains(p) || p.startsWith(AZXO_HEADER_PREFIX)) - .collect(Collectors.toMap(e -> e, e -> request.getHeader(e), (oldValue, newValue) -> newValue))) + .collect(Collectors.toMap(e -> e, request::getHeader, (oldValue, newValue) -> newValue))) .orElse(Collections.emptyMap()), //设置callerApp () -> AxContext.getRequest().map(e -> { @@ -47,12 +47,12 @@ public class RpcClientImpl implements RpcClient { @Builder public RpcClientImpl(RequestProxy requestProxy, - HttpClient.Config config, - Supplier> requestHeaderSupplier) { + Supplier> requestHeaderSupplier, + HttpClient httpClient) { this.requestProxy = Optional.ofNullable(requestProxy).orElse(RequestProxy.SIMPLE_PROXY); - this.httpClient = OkHttpClientImpl.builder() - .config(config) - .build(); + this.httpClient = Optional.ofNullable(httpClient).orElseGet(() -> OkHttpClientImpl.builder() + .config(HttpClient.Config.DEFAULT) + .build()); this.customHeaderSupplier = requestHeaderSupplier; } @@ -80,17 +80,12 @@ public class RpcClientImpl implements RpcClient { Objects.requireNonNull(requestParams); //XXX 附加默认的header, 以及自定义的header - DEFAULT_HEADER_SUPPLIERS.stream() - .forEach(headerSupplier -> headerSupplier.get().entrySet() - .forEach(headerEntry -> - requestParams.addHeaderIfAbsent(headerEntry.getKey(), headerEntry.getValue()) - )); + DEFAULT_HEADER_SUPPLIERS + .forEach(headerSupplier -> headerSupplier.get().forEach(requestParams::addHeaderIfAbsent)); if (customHeaderSupplier != null) { Map map = customHeaderSupplier.get(); - map.entrySet().forEach(e -> { - requestParams.addHeader(e.getKey(), e.getValue()); - }); + map.forEach(requestParams::addHeader); } return requestProxy.request(token -> { From 35bfdff8e45c86daefc13e98e56bac8f60fa9869 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Thu, 11 Jul 2024 17:29:49 +0800 Subject: [PATCH 35/41] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96rpc=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=20step2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/foundation/web/support/apps/AppCenterImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java index cc4db24..0c00147 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/apps/AppCenterImpl.java @@ -38,12 +38,11 @@ public class AppCenterImpl implements AppCenter { @Builder public AppCenterImpl(ScheduledThreadPoolExecutor executor, String debugHost, - Map debugAppRoutes, - RpcClient rpcClient) { + Map debugAppRoutes) { Objects.requireNonNull(executor); - this.rpcClient = Optional.ofNullable(rpcClient).orElseGet(() -> RpcClientImpl.builder().build()); + this.rpcClient = RpcClientImpl.builder().build(); this.appCache = new TimerRefreshCache<>("appCenterCache", INITIAL_DELAY_MILLIS, REFRESH_INTERVAL_MILLIS, executor, this::loadApps); this.debugHost = debugHost; From 06a8b49c5472553b780b0f12925b254cf15c8873 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 12 Jul 2024 11:50:22 +0800 Subject: [PATCH 36/41] =?UTF-8?q?feat:=20filter=E7=BA=A7=E5=88=AB=E9=99=8D?= =?UTF-8?q?=E7=BA=A7=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原因 需要支持降级处理 修改 修改filterHook, 支持降级 --- gateway-support-lib/pom.xml | 5 + .../plugin/impl/RequestFilterHook.java | 117 +++++++++++++++++- .../plugin/impl/RequestFilterHookTest.java | 57 +++++++++ 3 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 gateway-support-lib/src/test/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHookTest.java diff --git a/gateway-support-lib/pom.xml b/gateway-support-lib/pom.xml index 0465afa..d66586a 100644 --- a/gateway-support-lib/pom.xml +++ b/gateway-support-lib/pom.xml @@ -45,6 +45,11 @@ + + com.alibaba.csp + sentinel-core + 1.8.0 + \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java index 20aafdc..719175b 100644 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java @@ -1,5 +1,6 @@ package cn.axzo.foundation.gateway.support.plugin.impl; +import cn.axzo.foundation.exception.BusinessException; import cn.axzo.foundation.gateway.support.entity.GateResponse; import cn.axzo.foundation.gateway.support.entity.ProxyContext; import cn.axzo.foundation.gateway.support.entity.RequestContext; @@ -8,6 +9,13 @@ import cn.axzo.foundation.gateway.support.plugin.impl.filters.*; import cn.axzo.foundation.util.FastjsonUtils; import cn.axzo.foundation.web.support.rpc.RequestParams; import cn.axzo.foundation.web.support.rpc.RpcClient; +import com.alibaba.csp.sentinel.Entry; +import com.alibaba.csp.sentinel.EntryType; +import com.alibaba.csp.sentinel.SphU; +import com.alibaba.csp.sentinel.Tracer; +import com.alibaba.csp.sentinel.slots.block.BlockException; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.google.common.base.Preconditions; @@ -15,6 +23,7 @@ import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import lombok.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -74,6 +83,17 @@ public class RequestFilterHook implements ProxyHook { .put("requestParamCheckFilter", new RequestParamCheckFilter()) .put("moveInputFieldFilter", new MoveInputFieldFilter()) .put("keyNoQueryFilter", new KeyNoQueryFilter()) + .put("emptyFilter", new RequestFilter() { + @Override + public JSON filterIn(RequestContext reqContext, JSON params, JSONObject config) { + return RequestFilter.super.filterIn(reqContext, params, config); + } + + @Override + public JSON filterOut(RequestContext reqContext, JSON response, JSONObject config) { + return RequestFilter.super.filterOut(reqContext, response, config); + } + }) .build(); @Builder @@ -103,9 +123,32 @@ public class RequestFilterHook implements ProxyHook { for (FilterBean bean : beans) { RequestFilter requestFilter = filterBeanResolver.apply(bean.getName()); Preconditions.checkState(requestFilter != null, bean.getName() + " 没在系统注册"); - requestBody = requestFilter.filterIn(reqContext, requestBody, bean.getConfig()); - } + FallBackConfig fallBackConfig = bean.getFallBackConfig(); + Entry entry = null; + try { + if (fallBackConfig != null) { + String resourceName = bean.getResourceName(reqContext.getRequestURI()); + fallBackConfig.registerRule(resourceName); + entry = SphU.entry(resourceName, EntryType.IN); + } + requestBody = requestFilter.filterIn(reqContext, requestBody, bean.getConfig()); + } catch (BlockException ex) { + //降级处理 + requestBody = filterBeanResolver.apply(fallBackConfig.getFallBackFilter()) + .filterIn(reqContext, requestBody, fallBackConfig.getConfig()); + } catch (Exception ex) { + //非业务异常记录trace + if (!BusinessException.class.isAssignableFrom(ex.getClass())) { + Tracer.traceEntry(ex, entry); + } + throw ex; + } finally { + if (entry != null) { + entry.exit(); + } + } + } // 只支持 json 格式 return ((RequestParams.BodyParams) postParams).toBuilder().content(requestBody).build(); } @@ -133,7 +176,31 @@ public class RequestFilterHook implements ProxyHook { for (FilterBean bean : beans) { RequestFilter requestFilter = filterBeanResolver.apply(bean.getName()); Preconditions.checkState(requestFilter != null, bean.getName() + " 没在系统注册"); - responseBody = requestFilter.filterOut(reqContext, responseBody, bean.getConfig()); + FallBackConfig fallBackConfig = bean.getFallBackConfig(); + + Entry entry = null; + try { + if (fallBackConfig != null) { + String resourceName = bean.getResourceName(reqContext.getRequestURI()); + fallBackConfig.registerRule(resourceName); + entry = SphU.entry(resourceName, EntryType.OUT); + } + responseBody = requestFilter.filterOut(reqContext, responseBody, bean.getConfig()); + } catch (BlockException ex) { + //降级处理 + responseBody = filterBeanResolver.apply(fallBackConfig.getFallBackFilter()) + .filterIn(reqContext, responseBody, fallBackConfig.getConfig()); + } catch (Exception ex) { + //非业务异常记录trace + if (!BusinessException.class.isAssignableFrom(ex.getClass())) { + Tracer.traceEntry(ex, entry); + } + throw ex; + } finally { + if (entry != null) { + entry.exit(); + } + } } GateResponse res = response.toBuilder().build(); @@ -195,5 +262,49 @@ public class RequestFilterHook implements ProxyHook { private static class FilterBean { private String name; private JSONObject config; + + public FallBackConfig getFallBackConfig() { + return Optional.ofNullable(config) + .flatMap(e -> Optional.ofNullable(e.getJSONObject("fallBack")) + .map(fallBack -> fallBack.toJavaObject(FallBackConfig.class))) + .orElse(null); + } + + public String getResourceName(String requestUrl) { + return "RequestFilterHook:" + name + "@" + requestUrl; + } + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + private static class FallBackConfig { + String fallBackFilter = "emptyFilter"; + JSONObject config; + /** 策略, 支持FLOW_GRADE_QPS慢查询比例, DEGRADE_GRADE_EXCEPTION_RATIO异常比例, DEGRADE_GRADE_EXCEPTION_COUNT异常数 */ + Integer grade = 2; + /** 默认10s内请求大于5个, 同时错误超过一半则降级30s */ + Double count = 0.5D; + Integer timeWindow = 30; + Integer minRequestAmount = 5; + /** 慢比例有效 */ + Double slowRatioThreshold = 1.0; + Integer statIntervalMs = 10000; + + public void registerRule(String resourceName) { + if (DegradeRuleManager.hasConfig(resourceName)) { + return; + } + DegradeRule degradeRule = new DegradeRule(resourceName); + degradeRule.setGrade(grade); + degradeRule.setCount(count); + degradeRule.setTimeWindow(timeWindow); + degradeRule.setMinRequestAmount(minRequestAmount); + degradeRule.setSlowRatioThreshold(slowRatioThreshold); + degradeRule.setStatIntervalMs(statIntervalMs); + + DegradeRuleManager.setRulesForResource(resourceName, ImmutableSet.of(degradeRule)); + } } } diff --git a/gateway-support-lib/src/test/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHookTest.java b/gateway-support-lib/src/test/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHookTest.java new file mode 100644 index 0000000..64cb226 --- /dev/null +++ b/gateway-support-lib/src/test/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHookTest.java @@ -0,0 +1,57 @@ +package cn.axzo.foundation.gateway.support.plugin.impl; + + +import com.alibaba.csp.sentinel.Entry; +import com.alibaba.csp.sentinel.EntryType; +import com.alibaba.csp.sentinel.SphU; +import com.alibaba.csp.sentinel.Tracer; +import com.alibaba.csp.sentinel.slots.block.BlockException; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +class RequestFilterHookTest { + + + public static void main(String[] args) { + + String resourceName = "aaaa"; + + DegradeRule degradeRule = new DegradeRule(resourceName); + degradeRule.setGrade(2); + degradeRule.setCount(0.5); + degradeRule.setTimeWindow(30); + degradeRule.setMinRequestAmount(5); + degradeRule.setStatIntervalMs(10); + + DegradeRuleManager.setRulesForResource(resourceName, ImmutableSet.of(degradeRule)); + + List blocked = IntStream.range(0, 20).mapToObj(e -> { + Entry entry = null; + try { + entry = SphU.entry(resourceName, EntryType.IN); + if (e % 2 == 0) { + throw new RuntimeException(); + } + return e; + } catch (BlockException ex) { + return -1; + } catch (Exception ex) { + Tracer.traceEntry(ex, entry); + return -2; + } finally { + if (entry != null) { + entry.exit(); + } + } + }) + .collect(Collectors.toList()); + + System.out.printf(blocked.toString()); + } + +} \ No newline at end of file From 0dfd96c83c4325efdf5152d8028a375f34971330 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Fri, 12 Jul 2024 16:03:17 +0800 Subject: [PATCH 37/41] =?UTF-8?q?feat:=20=E8=B7=AF=E7=94=B1=E7=BA=A7?= =?UTF-8?q?=E5=88=AB=E7=9A=84=E6=9C=8D=E5=8A=A1=E9=99=8D=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原因 路由级别的服务降级 修改 修改SimpleProxy, 支持降级 --- .../gateway/support/BizGatewayImpl.java | 13 +-- .../gateway/support/entity/GlobalContext.java | 18 +++-- .../support/fallback/FallbackConfig.java | 50 ++++++++++++ .../plugin/impl/RequestFilterHook.java | 79 ++++--------------- .../support/proxy/GateResponseSupplier.java | 31 ++++++++ .../support/proxy/impl/SimpleProxy.java | 44 +++++++++-- .../web/support/alert/EmailAlertConsumer.java | 6 ++ 7 files changed, 154 insertions(+), 87 deletions(-) create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/fallback/FallbackConfig.java create mode 100644 gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/GateResponseSupplier.java diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/BizGatewayImpl.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/BizGatewayImpl.java index fdbf9d8..b05cebf 100755 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/BizGatewayImpl.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/BizGatewayImpl.java @@ -214,15 +214,6 @@ public class BizGatewayImpl implements BizGateway { ((StopWatchHook) hook).initialize(); } - private boolean isGrayStage(RequestContext requestContext) { - // TODO: correct gray stage - if (!CollectionUtils.isEmpty(requestContext.getHeaders()) - && StringUtils.equals(requestContext.getHeaders().get("gray_stage"), "true")) { - return true; - } - return false; - } - @Override public Proxy findProxy(RequestContext context) { initializeStopWatchHook(); @@ -377,9 +368,7 @@ public class BizGatewayImpl implements BizGateway { if (getRouteRules() == null) { return ImmutableMap.of(); } - return getRouteRules().stream().map(e -> { - return buildProxy(e); - }).collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); + return getRouteRules().stream().map(this::buildProxy).collect(Collectors.toMap(Pair::getKey, Pair::getValue)); } diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/entity/GlobalContext.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/entity/GlobalContext.java index f93aa24..94cddc7 100755 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/entity/GlobalContext.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/entity/GlobalContext.java @@ -2,6 +2,7 @@ package cn.axzo.foundation.gateway.support.entity; import cn.axzo.foundation.enums.AppEnvEnum; import cn.axzo.foundation.gateway.support.plugin.ProxyHookChain; +import cn.axzo.foundation.gateway.support.proxy.GateResponseSupplier; import cn.axzo.foundation.gateway.support.utils.RpcClientProvider; import cn.axzo.foundation.web.support.AppRuntime; import com.google.common.base.Strings; @@ -34,20 +35,23 @@ public class GlobalContext { @Getter private AppEnvEnum gateEnv; @Getter - final private RpcClientProvider rpcClientProvider; + private final RpcClientProvider rpcClientProvider; @Getter - final private BiConsumer alertConsumer; + private final BiConsumer alertConsumer; @Getter - final private Function> serviceSupplier; + private final Function> serviceSupplier; + /* 服务降级时默认返回 */ + @Getter + private final Function fallbackSupplier; @Getter /** 全局的代理Hook列表 */ - final private ProxyHookChain proxyHookChain; + private final ProxyHookChain proxyHookChain; @Getter - final private Long blockingMillis; + private final Long blockingMillis; @Getter - final private Map localProxies; + private final Map localProxies; @Getter - /** 网关代理配置最后更新时间 */ + /* 网关代理配置最后更新时间 */ private Long version; @Getter /** 需要debug的URI列表 */ diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/fallback/FallbackConfig.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/fallback/FallbackConfig.java new file mode 100644 index 0000000..34eb1b2 --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/fallback/FallbackConfig.java @@ -0,0 +1,50 @@ +package cn.axzo.foundation.gateway.support.fallback; + +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.ImmutableSet; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FallbackConfig { + String bean; + JSONObject config; + /** 策略, 支持FLOW_GRADE_QPS慢查询比例, DEGRADE_GRADE_EXCEPTION_RATIO异常比例, DEGRADE_GRADE_EXCEPTION_COUNT异常数 */ + Integer grade = 2; + /** 默认10s内请求大于5个, 同时错误超过一半则降级30s */ + Double count = 0.5D; + Integer timeWindow = 30; + Integer minRequestAmount = 5; + /** 慢比例有效 */ + Double slowRatioThreshold = 1.0; + Integer statIntervalMs = 10000; + + public static FallbackConfig fromConfig(JSONObject config) { + if (config == null || !config.containsKey("fallback")) { + return null; + } + return config.getJSONObject("fallback").toJavaObject(FallbackConfig.class); + } + + public void registerRule(String resourceName) { + if (DegradeRuleManager.hasConfig(resourceName)) { + return; + } + DegradeRule degradeRule = new DegradeRule(resourceName); + degradeRule.setGrade(grade); + degradeRule.setCount(count); + degradeRule.setTimeWindow(timeWindow); + degradeRule.setMinRequestAmount(minRequestAmount); + degradeRule.setSlowRatioThreshold(slowRatioThreshold); + degradeRule.setStatIntervalMs(statIntervalMs); + + DegradeRuleManager.setRulesForResource(resourceName, ImmutableSet.of(degradeRule)); + } +} \ No newline at end of file diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java index 719175b..16b86e1 100644 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/plugin/impl/RequestFilterHook.java @@ -1,9 +1,9 @@ package cn.axzo.foundation.gateway.support.plugin.impl; -import cn.axzo.foundation.exception.BusinessException; import cn.axzo.foundation.gateway.support.entity.GateResponse; import cn.axzo.foundation.gateway.support.entity.ProxyContext; import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.fallback.FallbackConfig; import cn.axzo.foundation.gateway.support.plugin.ProxyHook; import cn.axzo.foundation.gateway.support.plugin.impl.filters.*; import cn.axzo.foundation.util.FastjsonUtils; @@ -14,8 +14,6 @@ import com.alibaba.csp.sentinel.EntryType; import com.alibaba.csp.sentinel.SphU; import com.alibaba.csp.sentinel.Tracer; import com.alibaba.csp.sentinel.slots.block.BlockException; -import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; -import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.google.common.base.Preconditions; @@ -23,7 +21,6 @@ import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import lombok.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -123,25 +120,24 @@ public class RequestFilterHook implements ProxyHook { for (FilterBean bean : beans) { RequestFilter requestFilter = filterBeanResolver.apply(bean.getName()); Preconditions.checkState(requestFilter != null, bean.getName() + " 没在系统注册"); - FallBackConfig fallBackConfig = bean.getFallBackConfig(); + FallbackConfig fallbackConfig = FallbackConfig.fromConfig(bean.getConfig()); Entry entry = null; try { - if (fallBackConfig != null) { + if (fallbackConfig != null) { String resourceName = bean.getResourceName(reqContext.getRequestURI()); - fallBackConfig.registerRule(resourceName); + fallbackConfig.registerRule(resourceName); entry = SphU.entry(resourceName, EntryType.IN); } requestBody = requestFilter.filterIn(reqContext, requestBody, bean.getConfig()); } catch (BlockException ex) { + String filterBean = StringUtils.firstNonBlank(fallbackConfig.getBean(), "emptyFilter"); + log.warn("request fall back: {}, filter: {}, fallbackBean : {}", + reqContext.getRequestURI(), bean.getName(), filterBean, ex); //降级处理 - requestBody = filterBeanResolver.apply(fallBackConfig.getFallBackFilter()) - .filterIn(reqContext, requestBody, fallBackConfig.getConfig()); + requestBody = filterBeanResolver.apply(filterBean).filterIn(reqContext, requestBody, fallbackConfig.getConfig()); } catch (Exception ex) { - //非业务异常记录trace - if (!BusinessException.class.isAssignableFrom(ex.getClass())) { - Tracer.traceEntry(ex, entry); - } + Tracer.traceEntry(ex, entry); throw ex; } finally { if (entry != null) { @@ -176,25 +172,24 @@ public class RequestFilterHook implements ProxyHook { for (FilterBean bean : beans) { RequestFilter requestFilter = filterBeanResolver.apply(bean.getName()); Preconditions.checkState(requestFilter != null, bean.getName() + " 没在系统注册"); - FallBackConfig fallBackConfig = bean.getFallBackConfig(); + FallbackConfig fallbackConfig = FallbackConfig.fromConfig(bean.getConfig()); Entry entry = null; try { - if (fallBackConfig != null) { + if (fallbackConfig != null) { String resourceName = bean.getResourceName(reqContext.getRequestURI()); - fallBackConfig.registerRule(resourceName); + fallbackConfig.registerRule(resourceName); entry = SphU.entry(resourceName, EntryType.OUT); } responseBody = requestFilter.filterOut(reqContext, responseBody, bean.getConfig()); } catch (BlockException ex) { + String filterBean = StringUtils.firstNonBlank(fallbackConfig.getBean(), "emptyFilter"); + log.warn("response fall back: {}, filter: {}, fallbackBean : {}", + reqContext.getRequestURI(), bean.getName(), filterBean, ex); //降级处理 - responseBody = filterBeanResolver.apply(fallBackConfig.getFallBackFilter()) - .filterIn(reqContext, responseBody, fallBackConfig.getConfig()); + responseBody = filterBeanResolver.apply(filterBean).filterOut(reqContext, responseBody, fallbackConfig.getConfig()); } catch (Exception ex) { - //非业务异常记录trace - if (!BusinessException.class.isAssignableFrom(ex.getClass())) { - Tracer.traceEntry(ex, entry); - } + Tracer.traceEntry(ex, entry); throw ex; } finally { if (entry != null) { @@ -263,48 +258,8 @@ public class RequestFilterHook implements ProxyHook { private String name; private JSONObject config; - public FallBackConfig getFallBackConfig() { - return Optional.ofNullable(config) - .flatMap(e -> Optional.ofNullable(e.getJSONObject("fallBack")) - .map(fallBack -> fallBack.toJavaObject(FallBackConfig.class))) - .orElse(null); - } - public String getResourceName(String requestUrl) { return "RequestFilterHook:" + name + "@" + requestUrl; } } - - @Data - @Builder - @NoArgsConstructor - @AllArgsConstructor - private static class FallBackConfig { - String fallBackFilter = "emptyFilter"; - JSONObject config; - /** 策略, 支持FLOW_GRADE_QPS慢查询比例, DEGRADE_GRADE_EXCEPTION_RATIO异常比例, DEGRADE_GRADE_EXCEPTION_COUNT异常数 */ - Integer grade = 2; - /** 默认10s内请求大于5个, 同时错误超过一半则降级30s */ - Double count = 0.5D; - Integer timeWindow = 30; - Integer minRequestAmount = 5; - /** 慢比例有效 */ - Double slowRatioThreshold = 1.0; - Integer statIntervalMs = 10000; - - public void registerRule(String resourceName) { - if (DegradeRuleManager.hasConfig(resourceName)) { - return; - } - DegradeRule degradeRule = new DegradeRule(resourceName); - degradeRule.setGrade(grade); - degradeRule.setCount(count); - degradeRule.setTimeWindow(timeWindow); - degradeRule.setMinRequestAmount(minRequestAmount); - degradeRule.setSlowRatioThreshold(slowRatioThreshold); - degradeRule.setStatIntervalMs(statIntervalMs); - - DegradeRuleManager.setRulesForResource(resourceName, ImmutableSet.of(degradeRule)); - } - } } diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/GateResponseSupplier.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/GateResponseSupplier.java new file mode 100644 index 0000000..afdffe2 --- /dev/null +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/GateResponseSupplier.java @@ -0,0 +1,31 @@ +package cn.axzo.foundation.gateway.support.proxy; + +import cn.axzo.foundation.gateway.support.entity.GateResponse; +import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.result.ApiResult; +import cn.axzo.foundation.web.support.rpc.RequestParams; +import com.alibaba.fastjson.JSON; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.springframework.http.HttpStatus; + +import java.nio.charset.StandardCharsets; + +/** + * 返回GateResponse + * 目前在服务降级时会调用 + */ +public interface GateResponseSupplier { + + GateResponse getResponse(RequestContext requestContext, String resolvedUrl, RequestParams requestParams); + + GateResponseSupplier DEFAULT = (requestContext, resolvedUrl, requestParams) -> { + byte[] bytes = JSON.toJSONString(ApiResult.success()).getBytes(StandardCharsets.UTF_8); + return GateResponse.builder() + .status(HttpStatus.OK) + .content(bytes) + .headers(ImmutableMap.of("content-type", ImmutableList.of("application/json;charset=UTF-8") + , "content-length", ImmutableList.of(bytes.length + ""))) + .build(); + }; +} diff --git a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/impl/SimpleProxy.java b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/impl/SimpleProxy.java index 2c07cb1..75e3fd2 100755 --- a/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/impl/SimpleProxy.java +++ b/gateway-support-lib/src/main/java/cn/axzo/foundation/gateway/support/proxy/impl/SimpleProxy.java @@ -3,8 +3,16 @@ package cn.axzo.foundation.gateway.support.proxy.impl; import cn.axzo.foundation.gateway.support.entity.GateResponse; import cn.axzo.foundation.gateway.support.entity.ProxyContext; import cn.axzo.foundation.gateway.support.entity.RequestContext; +import cn.axzo.foundation.gateway.support.exception.ApiNotFoundException; +import cn.axzo.foundation.gateway.support.fallback.FallbackConfig; import cn.axzo.foundation.gateway.support.plugin.ProxyHookChain; +import cn.axzo.foundation.gateway.support.proxy.GateResponseSupplier; import cn.axzo.foundation.web.support.rpc.RequestParams; +import cn.axzo.foundation.web.support.rpc.RpcClient; +import com.alibaba.csp.sentinel.Entry; +import com.alibaba.csp.sentinel.SphU; +import com.alibaba.csp.sentinel.Tracer; +import com.alibaba.csp.sentinel.slots.block.BlockException; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONException; @@ -14,6 +22,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.net.UrlEscapers; +import lombok.extern.slf4j.Slf4j; import okhttp3.MediaType; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -23,12 +32,10 @@ import javax.servlet.http.HttpServletRequest; import java.util.*; import java.util.stream.Collectors; -import cn.axzo.foundation.gateway.support.exception.ApiNotFoundException; -import cn.axzo.foundation.web.support.rpc.RpcClient; - /** * 简单请求Proxy, 根据配置中的服务编码对应的域名, 在具体实现代理中进行URL及参数转换. */ +@Slf4j public abstract class SimpleProxy extends AbstractProxy { abstract String resolveServiceCode(RequestContext context); @@ -74,8 +81,33 @@ public abstract class SimpleProxy extends AbstractProxy { RpcClient rpcClient = getServiceResolver().getRpcClient(requestContext, hookedCode); RequestParams resolvedParams = getHookChain().preRequest(requestContext, getContext(), rpcClient, resolvedUrl, requestParams); - GateResponse response = requestContext.getRequestMethod().request( - rpcClient, resolvedUrl, resolvedParams); + FallbackConfig fallbackConfig = FallbackConfig.fromConfig(getContext().getProxyParam()); + + GateResponse response; + Entry entry = null; + try { + if (fallbackConfig != null) { + String resourceName = "SimpleProxy:" + requestContext.getServiceCode() + "@" + requestContext.getRequestURI(); + fallbackConfig.registerRule(resourceName); + entry = SphU.entry(resourceName); + } + response = requestContext.getRequestMethod().request( + rpcClient, resolvedUrl, resolvedParams); + } catch (BlockException ex) { + log.warn("proxy fall back: {}, fallbackBean : {}", requestContext.getRequestURI(), fallbackConfig.getBean(), ex); + //降级处理 + response = Optional.ofNullable(getContext().getGlobalContext().getFallbackSupplier()) + .flatMap(e -> Optional.ofNullable(e.apply(fallbackConfig.getBean()))) + .orElse(GateResponseSupplier.DEFAULT).getResponse(requestContext, resolvedUrl, requestParams); + } catch (Exception ex) { + Tracer.traceEntry(ex, entry); + throw ex; + } finally { + if (entry != null) { + entry.exit(); + } + } + GateResponse res = getHookChain().postResponse(requestContext, getContext(), response); filterResponse(requestContext, res); @@ -102,7 +134,7 @@ public abstract class SimpleProxy extends AbstractProxy { Optional subtypeOptional = Optional.ofNullable(context.getOriginalRequest()) .map(HttpServletRequest::getContentType) .map(MediaType::parse) - .map(e->e.subtype().toLowerCase()); + .map(e -> e.subtype().toLowerCase()); if (subtypeOptional.isPresent() && JSON_YML_SUBTYPES.contains(subtypeOptional.get())) { String requestBody = StringUtils.firstNonBlank(context.getRequestBody(), "{}"); Object body; diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/EmailAlertConsumer.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/EmailAlertConsumer.java index 09a9c87..9d986b1 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/EmailAlertConsumer.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/alert/EmailAlertConsumer.java @@ -1,6 +1,8 @@ package cn.axzo.foundation.web.support.alert; +import cn.axzo.foundation.enums.AppEnvEnum; import cn.axzo.foundation.web.support.AppRuntime; +import com.alibaba.fastjson.JSONObject; import lombok.Builder; import lombok.extern.slf4j.Slf4j; @@ -57,6 +59,10 @@ public class EmailAlertConsumer implements Consumer> alerts) { + if (appRuntime.getEnv() == AppEnvEnum.local) { + log.error("local alerts: {}", JSONObject.toJSONString(alerts)); + return; + } Message message = new MimeMessage(session); try { message.setFrom(new InternetAddress(from)); From 13f75f41b76c68a1c065b0b3ad6e2c73d4bb54b6 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Mon, 15 Jul 2024 09:55:29 +0800 Subject: [PATCH 38/41] =?UTF-8?q?feat:=20=E6=9E=84=E5=BB=BA=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原因 构建代码 修改 构建代码 --- common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java | 1 - 1 file changed, 1 deletion(-) diff --git a/common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java b/common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java index 825d5da..a6d5f74 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java +++ b/common-lib/src/main/java/cn/axzo/foundation/page/PageResp.java @@ -42,5 +42,4 @@ public class PageResp { } return pages; } - } From 44d73ab3607dac2e66c61e64b08ecb71045ff9db Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Mon, 15 Jul 2024 17:34:57 +0800 Subject: [PATCH 39/41] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0traceId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原因 增加traceId 修改 增加traceId --- .../cn/axzo/foundation/util/TraceUtils.java | 2 +- .../support/config/DefaultWebMvcConfig.java | 2 ++ .../interceptors/TraceInterceptor.java | 33 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/TraceInterceptor.java diff --git a/common-lib/src/main/java/cn/axzo/foundation/util/TraceUtils.java b/common-lib/src/main/java/cn/axzo/foundation/util/TraceUtils.java index 3a68d6a..f48c07c 100644 --- a/common-lib/src/main/java/cn/axzo/foundation/util/TraceUtils.java +++ b/common-lib/src/main/java/cn/axzo/foundation/util/TraceUtils.java @@ -6,7 +6,7 @@ import org.slf4j.MDC; @UtilityClass public class TraceUtils { - public static final String TRACE_ID = "axzo-trace-id"; + public static final String TRACE_ID = "traceId"; /** * 多设置一个key = TraceId, value为traceId的变量到MDC. 以兼容目前的logback-spring.xml的配置 */ diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java index 6f7bd94..34cee7b 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java @@ -7,6 +7,7 @@ import cn.axzo.foundation.web.support.exception.AbstractExceptionHandler; import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor; import cn.axzo.foundation.web.support.interceptors.PrettyPrintInterceptor; import cn.axzo.foundation.web.support.interceptors.PrintVerboseInterceptor; +import cn.axzo.foundation.web.support.interceptors.TraceInterceptor; import cn.axzo.foundation.web.support.resolvers.AxContextResolver; import cn.axzo.foundation.web.support.resolvers.CallerAppResolver; import com.alibaba.fastjson.JSONArray; @@ -187,6 +188,7 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement registry.addInterceptor(new CorsInterceptor(corsProcessorType)); registry.addInterceptor(new PrettyPrintInterceptor()); registry.addInterceptor(new CallerAppInterceptor(appRuntime)); + registry.addInterceptor(new TraceInterceptor()); super.addInterceptors(registry); diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/TraceInterceptor.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/TraceInterceptor.java new file mode 100644 index 0000000..ce9142e --- /dev/null +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/TraceInterceptor.java @@ -0,0 +1,33 @@ +package cn.axzo.foundation.web.support.interceptors; + +import cn.axzo.foundation.util.TraceUtils; +import com.google.common.base.Strings; +import org.springframework.web.servlet.HandlerInterceptor; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * 在request上下文中添加 traceId + */ +public class TraceInterceptor implements HandlerInterceptor { + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + String upstreamTraceId = request.getHeader(TraceUtils.TRACE_ID); + if (!Strings.isNullOrEmpty(upstreamTraceId)) { + // 优先使用上游的 traceId + TraceUtils.putTraceId(upstreamTraceId); + } + + String traceId = TraceUtils.getOrCreateTraceId(); + request.setAttribute(TraceUtils.TRACE_ID_IN_MDC, traceId); + return true; + } + + @Override + public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { + TraceUtils.removeTraceId(); + } +} + From 2c2b0dafb07fcef640fe30e32bd3e30159c9b3c5 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Mon, 15 Jul 2024 18:49:08 +0800 Subject: [PATCH 40/41] =?UTF-8?q?feat:=20=E5=81=A5=E5=BA=B7=E6=A3=80?= =?UTF-8?q?=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原因 健康检查 修改 健康检查 --- .../support/config/DefaultWebMvcConfig.java | 8 ++----- .../interceptors/CheckDeathInterceptor.java | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/CheckDeathInterceptor.java diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java index 34cee7b..df37e14 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java @@ -4,10 +4,7 @@ import cn.axzo.foundation.util.FastjsonUtils; import cn.axzo.foundation.web.support.AppRuntime; import cn.axzo.foundation.web.support.context.AxContextInterceptor; import cn.axzo.foundation.web.support.exception.AbstractExceptionHandler; -import cn.axzo.foundation.web.support.interceptors.CallerAppInterceptor; -import cn.axzo.foundation.web.support.interceptors.PrettyPrintInterceptor; -import cn.axzo.foundation.web.support.interceptors.PrintVerboseInterceptor; -import cn.axzo.foundation.web.support.interceptors.TraceInterceptor; +import cn.axzo.foundation.web.support.interceptors.*; import cn.axzo.foundation.web.support.resolvers.AxContextResolver; import cn.axzo.foundation.web.support.resolvers.CallerAppResolver; import com.alibaba.fastjson.JSONArray; @@ -189,6 +186,7 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement registry.addInterceptor(new PrettyPrintInterceptor()); registry.addInterceptor(new CallerAppInterceptor(appRuntime)); registry.addInterceptor(new TraceInterceptor()); + registry.addInterceptor(new CheckDeathInterceptor()); super.addInterceptors(registry); @@ -238,8 +236,6 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement //添加一个空的根实现. 避免爬虫访问到根路径后一直打印PageNotFound. 项目中可以覆盖 / 根路径的实现 registry.addStatusController("/", HttpStatus.NO_CONTENT); registry.addStatusController("/favicon.ico", HttpStatus.NO_CONTENT); - //健康检查 - registry.addStatusController("/checkDeath", HttpStatus.OK); super.addViewControllers(registry); } diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/CheckDeathInterceptor.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/CheckDeathInterceptor.java new file mode 100644 index 0000000..0c14df7 --- /dev/null +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/interceptors/CheckDeathInterceptor.java @@ -0,0 +1,23 @@ +package cn.axzo.foundation.web.support.interceptors; + +import org.springframework.http.HttpStatus; +import org.springframework.web.servlet.HandlerInterceptor; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public class CheckDeathInterceptor implements HandlerInterceptor { + + private static final String CHECK_DEATH_URL = "/checkDeath"; + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + if (CHECK_DEATH_URL.equals(request.getRequestURI())) { + response.setStatus(HttpStatus.OK.value()); + return false; + } + return true; + } + +} + From 0b656f25ae9c788b976bf27857275a8b6be28400 Mon Sep 17 00:00:00 2001 From: zengxiaobo Date: Tue, 16 Jul 2024 13:46:54 +0800 Subject: [PATCH 41/41] =?UTF-8?q?feat:=20=E5=81=A5=E5=BA=B7=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=20step=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原因 健康检查 修改 健康检查 --- .../foundation/web/support/config/DefaultWebMvcConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java index df37e14..93fee08 100644 --- a/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java +++ b/web-support-lib/src/main/java/cn/axzo/foundation/web/support/config/DefaultWebMvcConfig.java @@ -191,9 +191,7 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement super.addInterceptors(registry); if (null != handlerInterceptors) { - Arrays.stream(handlerInterceptors).forEach(e -> { - registry.addInterceptor(e); - }); + Arrays.stream(handlerInterceptors).forEach(registry::addInterceptor); } registry.addInterceptor(new AxContextInterceptor(appRuntime, debugHost)); } @@ -236,6 +234,8 @@ public class DefaultWebMvcConfig extends DelegatingWebMvcConfiguration implement //添加一个空的根实现. 避免爬虫访问到根路径后一直打印PageNotFound. 项目中可以覆盖 / 根路径的实现 registry.addStatusController("/", HttpStatus.NO_CONTENT); registry.addStatusController("/favicon.ico", HttpStatus.NO_CONTENT); + //这里必须添加一个映射配合CheckDeathInterceptor使用, 否则会出现NoMapping + registry.addStatusController("/checkDeath", HttpStatus.NO_CONTENT); super.addViewControllers(registry); }