From e09dad2f0b00144262fe9929b9b3e87821e3b1b2 Mon Sep 17 00:00:00 2001 From: wangjibo Date: Thu, 27 Jul 2023 14:34:29 +0800 Subject: [PATCH 01/17] =?UTF-8?q?=E7=A9=BA=E7=99=BD=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a09932e..882fbd0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # pokonyan -叮当猫,中台公共属性的组件集合 \ No newline at end of file +叮当猫,中台公共属性的组件集合 From 74171b683d45e9f1f8599654d41ec62a3ccd267d Mon Sep 17 00:00:00 2001 From: lilong Date: Fri, 15 Mar 2024 16:51:57 +0800 Subject: [PATCH 02/17] =?UTF-8?q?feat:=E5=A2=9E=E5=8A=A0dao-support?= =?UTF-8?q?=EF=BC=8CRateLimiter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 9 + .../cn/axzo/pokonyan/client/RateLimiter.java | 84 ++++ .../pokonyan/client/RateLimiterClient.java | 43 ++ .../client/impl/RateLimiterClientImpl.java | 52 +++ .../client/impl/RedisRateLimiterImpl.java | 108 +++++ .../pokonyan/dao/converter/PageConverter.java | 124 ++++++ .../pokonyan/dao/mysql/JsonImportHelper.java | 265 ++++++++++++ .../dao/mysql/MybatisPlusCacheHelper.java | 132 ++++++ .../dao/mysql/MybatisPlusConverterUtils.java | 61 +++ .../pokonyan/dao/mysql/MybatisPlusHelper.java | 73 ++++ .../mysql/MybatisPlusOperatorProcessor.java | 141 ++++++ .../pokonyan/dao/mysql/MysqlJsonHelper.java | 117 +++++ .../dao/mysql/QueryWrapperHelper.java | 50 +++ .../dao/mysql/type/BaseListTypeHandler.java | 51 +++ .../mysql/type/LinkedHashSetTypeHandler.java | 54 +++ .../dao/mysql/type/SetTypeHandler.java | 60 +++ .../cn/axzo/pokonyan/dao/page/IPageParam.java | 37 ++ .../pokonyan/dao/utils/RepairDataHelper.java | 255 +++++++++++ .../pokonyan/dao/wrapper/CriteriaField.java | 52 +++ .../pokonyan/dao/wrapper/CriteriaWrapper.java | 408 ++++++++++++++++++ .../axzo/pokonyan/dao/wrapper/Operator.java | 47 ++ .../dao/wrapper/OperatorProcessor.java | 19 + .../dao/wrapper/SimpleWrapperConverter.java | 158 +++++++ .../pokonyan/dao/wrapper/TriConsumer.java | 17 + 24 files changed, 2417 insertions(+) create mode 100644 src/main/java/cn/axzo/pokonyan/client/RateLimiter.java create mode 100644 src/main/java/cn/axzo/pokonyan/client/RateLimiterClient.java create mode 100644 src/main/java/cn/axzo/pokonyan/client/impl/RateLimiterClientImpl.java create mode 100644 src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/converter/PageConverter.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/JsonImportHelper.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusCacheHelper.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusConverterUtils.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusHelper.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusOperatorProcessor.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/MysqlJsonHelper.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/QueryWrapperHelper.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/type/BaseListTypeHandler.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/type/LinkedHashSetTypeHandler.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/mysql/type/SetTypeHandler.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/page/IPageParam.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/utils/RepairDataHelper.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaField.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaWrapper.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/wrapper/Operator.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/wrapper/OperatorProcessor.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/wrapper/SimpleWrapperConverter.java create mode 100644 src/main/java/cn/axzo/pokonyan/dao/wrapper/TriConsumer.java diff --git a/pom.xml b/pom.xml index d355fbd..de9f7fe 100644 --- a/pom.xml +++ b/pom.xml @@ -104,6 +104,15 @@ cn.axzo.framework axzo-common-domain + + mysql + mysql-connector-java + provided + + + org.redisson + redisson-spring-boot-starter + diff --git a/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java b/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java new file mode 100644 index 0000000..e440feb --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java @@ -0,0 +1,84 @@ +package cn.axzo.pokonyan.client; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import lombok.ToString; +import org.springframework.util.CollectionUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +public interface RateLimiter { + /** + * 尝试获得锁, 获取失败则返回Optional.empty() + * 如果获取锁成功. 则返回Optional. 同时计数器增加 + * Permit支持取消 + * + * @param value 业务标识 + * @return + */ + boolean tryAcquire(Object value); + + /** + * 获取窗口类型 + * + * @return + */ + WindowType getWindowType(); + + class Permit { + private List cancelRunners; + + @Builder + public Permit(List cancelRunners) { + Objects.requireNonNull(cancelRunners); + this.cancelRunners = cancelRunners; + } + + public void cancel() { + if (!cancelRunners.isEmpty()) { + cancelRunners.stream().forEach(e -> e.run()); + } + } + } + + @Getter + @AllArgsConstructor + enum WindowType { + /** + * 滑动窗口, 窗口范围: start = currentMillis - WindowDuration, end = currentMillis + */ + SLIDING("s"); + + //减少redisKey长度 + private final String shortName; + } + + /** + * 限流规则 + *
+     *     seconds: 窗口时长
+     *     permits: 允许发放的令牌数量
+     * 
+ */ + @Data + @Builder + @AllArgsConstructor + @ToString + class LimitRule { + long seconds; + int permits; + + public boolean isValid() { + return seconds > 0 && permits > 0; + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/client/RateLimiterClient.java b/src/main/java/cn/axzo/pokonyan/client/RateLimiterClient.java new file mode 100644 index 0000000..48c87b9 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/RateLimiterClient.java @@ -0,0 +1,43 @@ +package cn.axzo.pokonyan.client; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +public interface RateLimiterClient { + + /** + * 构建一个基于Redis的RateLimiter + * + * @return + */ + RateLimiter build(RateLimiterReq rateLimiterReq); + + /** + * 根据windowType与ruleExpression构建一个基于Redis的RateLimiter + * @param limiterKey + * @param windowType + * @param seconds + * @param permits + * @return + */ + default RateLimiter build(String limiterKey, RateLimiter.WindowType windowType, long seconds, int permits) { + return build(RateLimiterReq.builder() + .windowType(windowType) + .rule(RateLimiter.LimitRule.builder() + .seconds(seconds) + .permits(permits) + .build()) + .limiterKey(limiterKey) + .build()); + } + + @Data + @Builder + @AllArgsConstructor + class RateLimiterReq { + RateLimiter.WindowType windowType; + RateLimiter.LimitRule rule; + String limiterKey; + } +} diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/RateLimiterClientImpl.java b/src/main/java/cn/axzo/pokonyan/client/impl/RateLimiterClientImpl.java new file mode 100644 index 0000000..f7e0196 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/impl/RateLimiterClientImpl.java @@ -0,0 +1,52 @@ +package cn.axzo.pokonyan.client.impl; + +import cn.axzo.pokonyan.client.RateLimiter; +import cn.axzo.pokonyan.client.RateLimiterClient; +import lombok.AccessLevel; +import lombok.Data; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.redisson.api.RedissonClient; + +import java.util.Objects; + +@Slf4j +public class RateLimiterClientImpl implements RateLimiterClient { + + @Setter(AccessLevel.PROTECTED) + private RedissonClient redissonClient; + + @Override + public RateLimiter build(RateLimiterReq rateLimiterReq) { + return RedisRateLimiterImpl.builder() + .windowType(rateLimiterReq.getWindowType()) + .limitRule(rateLimiterReq.getRule()) + .limiterKey(rateLimiterReq.getLimiterKey()) + .redissonClient(redissonClient) + .build(); + } + + public static Builder builder() { + return new Builder(); + } + + @Data + public static class Builder { + + private RedissonClient redissonClient; + + public Builder redisTemplate(RedissonClient redissonClient) { + this.redissonClient = redissonClient; + return this; + } + + public RateLimiterClient build() { + //单元测试环境也构建同样的RateLimiterClient + Objects.requireNonNull(redissonClient); + + RateLimiterClientImpl client = new RateLimiterClientImpl(); + client.setRedissonClient(redissonClient); + return client; + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java b/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java new file mode 100644 index 0000000..5c1c675 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java @@ -0,0 +1,108 @@ +package cn.axzo.pokonyan.client.impl; + +import cn.axzo.pokonyan.client.RateLimiter; +import com.alibaba.fastjson.JSONObject; +import com.google.common.hash.Hashing; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.Charsets; +import org.redisson.api.RRateLimiter; +import org.redisson.api.RateIntervalUnit; +import org.redisson.api.RateType; +import org.redisson.api.RedissonClient; + +import java.util.Objects; + +@Slf4j +public class RedisRateLimiterImpl implements RateLimiter { + private RedissonClient redissonClient; + private RateLimiterWorker rateLimiterWorker; + /** + * 自定义的key, 避免redisKey冲突. 必填 + */ + private String limiterKey; + private LimitRule limitRule; + private WindowType windowType; + + @Builder + RedisRateLimiterImpl(RedissonClient redissonClient, + WindowType windowType, + String limiterKey, + LimitRule limitRule, + Integer maxWindowDurationHour) { + Objects.requireNonNull(redissonClient); + Objects.requireNonNull(windowType); + Objects.requireNonNull(limitRule); + Objects.requireNonNull(limiterKey); + if (!limitRule.isValid()) { + throw new RuntimeException(String.format("invalid rate expression, limitRule = %s", JSONObject.toJSONString(limitRule))); + } + + this.windowType = windowType; + this.redissonClient = redissonClient; + this.limitRule = limitRule; + this.limiterKey = limiterKey; + this.rateLimiterWorker = buildWorker(windowType); + } + + @Override + public boolean tryAcquire(Object value) { + return rateLimiterWorker.tryAcquire(value); + } + + @Override + public WindowType getWindowType() { + return windowType; + } + + private RateLimiterWorker buildWorker(WindowType windowType) { + if (windowType == WindowType.SLIDING) { + return new SlidingWindowRateLimiter(); + } + throw new RuntimeException(String.format("unsupported window type, window type = %s", windowType)); + } + + private String buildRedisKey(Object value) { + String hash = Hashing.murmur3_128().newHasher() + .putString(limiterKey, Charsets.UTF_8) + .putString(String.valueOf(value), Charsets.UTF_8) + .hash() + .toString(); + + return new StringBuilder("rl").append(getWindowType().getShortName()).append(hash).toString(); + } + + /** + * 滑动窗口限流, 每次获取令牌成功时间加入到zset. 后续获取令牌时每次检查zset中WindowDuration中已获取令牌数. 并判断是否可以继续获取令牌 + *
+     *     key = value
+     *     zset value = currentMillis. score = currentMillis
+     *     获取令牌时, 在计算zset中 score = [currentMillis-WindowDuration, currentMillis} 的element数量
+     * 
+ */ + class SlidingWindowRateLimiter implements RateLimiterWorker { + public boolean tryAcquire(Object value) { + String key = buildRedisKey(value); + long now = System.currentTimeMillis(); + + RRateLimiter rateLimiter = redissonClient.getRateLimiter(key); + + rateLimiter.availablePermits(); + if (!rateLimiter.isExists()) { + rateLimiter.trySetRate(RateType.OVERALL, limitRule.getPermits(), limitRule.getSeconds(), RateIntervalUnit.SECONDS); + } + + return rateLimiter.tryAcquire(1); + } + } + + interface RateLimiterWorker { + /** + * 尝试获取令牌 + * + * @param value + * @return 如果获取成功则返回true, 失败则为false + */ + boolean tryAcquire(Object value); + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/converter/PageConverter.java b/src/main/java/cn/axzo/pokonyan/dao/converter/PageConverter.java new file mode 100644 index 0000000..3770871 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/converter/PageConverter.java @@ -0,0 +1,124 @@ +package cn.axzo.pokonyan.dao.converter; + +import cn.axzo.pokonyan.dao.mysql.MybatisPlusConverterUtils; +import cn.axzo.pokonyan.dao.page.IPageParam; +import com.baomidou.mybatisplus.core.metadata.OrderItem; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +@UtilityClass +public class PageConverter { + + /** + * 将bfs page转换为MybatisPlus的IPage + * 支持根据entity上的字段来排序 + * + * @param page + * @param entityClz + * @param + * @return + */ + public static Page convertToMybatis(IPageParam page, Class entityClz) { + int pageSize = Math.min(Optional.ofNullable(page.getPageSize()).orElse(IPageParam.DEFAULT_PAGE_SIZE), IPageParam.MAX_PAGE_SIZE); + Integer current = Optional.ofNullable(page.getPageNumber()).orElse(IPageParam.DEFAULT_PAGE_NUMBER); + + Page myBatisPage + = new Page<>(current, pageSize); + Map fieldColumnMap = entityClz == null ? ImmutableMap.of() : MybatisPlusConverterUtils.getFieldMapping(entityClz); + + List orderItems = Optional.ofNullable(page.getSort()).orElse(ImmutableList.of()).stream() + .map(e -> { + String property = StringUtils.substringBefore(e, IPageParam.SORT_DELIMITER); + // 尝试把实体类上的字段转换为数据库column + if (fieldColumnMap.containsKey(property)) { + property = fieldColumnMap.get(property); + } + String direction = StringUtils.substringAfter(e, IPageParam.SORT_DELIMITER); + if (direction != null && IPageParam.SORT_DESC.equals(direction)) { + return OrderItem.desc(property); + } + return OrderItem.asc(property); + }) + .collect(Collectors.toList()); + + myBatisPage.setOrders(orderItems); + + return myBatisPage; + } + + /** + * 将所有的数据通过page接口写入到list. 并返回 + * function中需要参数为新的pageNum, 默认从第一页开始加载. 直到返回的记录行数小于 预期的行数 + */ + public static List drainAll(Function> function) { + return drainAll(function, null); + } + + /** + * 将所有的数据通过page接口写入到list. 并返回 + * function中需要参数为新的pageNum, 默认从第一页开始加载. 直到返回的记录行数小于 预期的行数 + * breaker可以自行决定何时中断,允许为空,为空表示会拉取所有 + */ + public static List drainAll(Function> function, Function, Boolean> breaker) { + List totalData = Lists.newArrayList(); + int pageNum = IPageParam.DEFAULT_PAGE_NUMBER; + while (true) { + Page result = function.apply(pageNum); + totalData.addAll(result.getRecords()); + + if (result.getRecords().size() < result.getSize()) { + break; + } + if (breaker != null && BooleanUtils.isTrue(breaker.apply(totalData))) { + break; + } + pageNum += 1; + } + return totalData; + } + + /** + * 将MybatisPlus的IPage转换为spring的Page, 用于返回 + * + * @param page + * @param + * @return + */ +// public static Page convertToBfs(IPage page) { +// List sorts = page.orders().stream() +// .map(e -> e.getColumn().concat(IPageParam.SORT_DELIMITER).concat(e.isAsc() ? IPageParam.SORT_ASC : IPageParam.SORT_DESC)) +// .collect(Collectors.toList()); +// Page result = Page.builder() +// .total(page.getTotal()) +// .current(page.getCurrent()) +// .size(page.getSize()) +// .build(); +// +// result.setTotal(page.getTotal()); +// result.setRecords(page.getRecords()); +// +// return result; +// } +// +// /** +// * 读取 bfs 的 page 请求,读取 mybatis 并将结果转换成 bfs 的 page +// * mybatisPage(page, p->xxxDao.selectPage(p, query)); +// * XXX 针对排序字段作出优化,通过传入entityClz用于确定排序sql中的真实字段名 +// */ +// public static Page mybatisPage(IPageParam page, Function pageLoader, Class entityClz) { +// final IPage p = PageConverter.convertToMybatis(page, entityClz); +// IPage iPage = pageLoader.apply(p); +// return PageConverter.convertToBfs(iPage); +// } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/JsonImportHelper.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/JsonImportHelper.java new file mode 100644 index 0000000..b4667a4 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/JsonImportHelper.java @@ -0,0 +1,265 @@ +package cn.axzo.pokonyan.dao.mysql; + +import cn.axzo.pokonyan.dao.mysql.type.SetTypeHandler; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.TableFieldInfo; +import com.baomidou.mybatisplus.core.metadata.TableInfo; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.util.CollectionUtils; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +/** + * 一个简单工具类可以从测试环境中通过intellij工具导入数据成json字符串, 然后运行这个脚本导入到线上环境. + * intellij中请使用JSON-groovy格式输出成json + * 注意.导出的数据的字段是数据库的列名需要转换. + *
+ * [
+ *      {
+ *          "id": 2,
+ *          "app_ids": "9999",
+ *          "name": "集成测试告警",
+ *          "subject": "集成测试告警",
+ *          "priority": 98,
+ *          "recipients": "",
+ *          "send_threshold": 1,
+ *          "send_interval": 1,
+ *          "description": "集成测试告警",
+ *          "ext": "",
+ *          "status": "ENABLED",
+ *          "create_time": "2019-10-25 07:10:43",
+ *          "update_time": "2019-11-23 19:51:50"
+ *      }
+ *     ....
+ * ]
+ * 
+ * sample + *
+ *         importHelper = JsonImportHelper.builder().baseMapper(alertRuleDao)
+ *                 .bizKeyNames(List.of("name"))
+ *                 .excludeFields(Set.of())
+ *                 .clz(AlertRule.class)
+ *                 .saveOrUpdate(null).build();
+ * 
+ * + * @param + * @param + */ +public class JsonImportHelper, T> { + + private static final Set DEFAULT_EXCLUDE_FIELDS = ImmutableSet.of("", "id", "rowid", "updatedTime", "createTime", "updateTime", "createTime", "modifyTime"); + private static final String DEFAULT_ID_FIELD_NAME = "id"; + private static final int MAX_IMPORT_COUNT = 100; + private boolean jsonSmart; + + private String idFieldName; + private HashSet excludeFields; + private M baseMapper; + /** + * 业务主键. 比如code, 唯一的名称...能唯一标识这条数据. + */ + private List bizKeyNames; + private Class clz; + private BiConsumer saveOrUpdate; + + /** + * @param baseMapper + * @param excludeFields 排除的字段. 比如更新日期,创建日期,id. 默认已经集成. + * @param bizKeyNames + * @param clz + * @param saveOrUpdate + */ + @Builder + public JsonImportHelper(M baseMapper, Set excludeFields, List bizKeyNames, Class clz, + BiConsumer saveOrUpdate, String idFieldName) { + Preconditions.checkArgument(!CollectionUtils.isEmpty(bizKeyNames)); + Preconditions.checkArgument(baseMapper != null); + Preconditions.checkArgument(clz != null); + + this.baseMapper = baseMapper; + this.excludeFields = new HashSet<>(DEFAULT_EXCLUDE_FIELDS); + if (!CollectionUtils.isEmpty(excludeFields)) { + this.excludeFields.addAll(excludeFields); + } + this.bizKeyNames = bizKeyNames; + this.clz = clz; + this.saveOrUpdate = saveOrUpdate; + this.jsonSmart = true; + this.idFieldName = Strings.isNullOrEmpty(idFieldName) ? DEFAULT_ID_FIELD_NAME : idFieldName; + } + + public void setJsonSmart(boolean jsonSmart) { + this.jsonSmart = jsonSmart; + } + + /** + * 执行导入数据操作.返回需要更新, 查询, 没有变化的数据. 如果limit 是0 只是查看数据结果, 不发生操作. + * 一次最多不超过200条数据 + * + * @param rawJson + * @param limit 指定需要更新的数据数量. 如果是0, 不会更新数据. 只返回结果 + * @return 返回更新数据列表. + */ + public ImportResp run(List rawJson, int limit) { + Preconditions.checkArgument(rawJson.size() <= MAX_IMPORT_COUNT); + List importRows = resolveRows(rawJson); + QueryWrapper query = new QueryWrapper<>(); + + query.last("limit 1000"); + TableInfo tableInfo = TableInfoHelper.getTableInfo(clz); + Map columnMap = tableInfo.getFieldList().stream().collect(Collectors.toMap(e -> e.getColumn(), e -> e.getProperty())); + // 主键没有columnMap中,需要显示声明. + columnMap.put(tableInfo.getKeyColumn(), tableInfo.getKeyProperty()); + // 没有直接使用selectList处理json类型列有问题 + // baseMapper.selectMaps(query) 返回数据是db 列名, 需要转换 + List dbRows = baseMapper.selectMaps(query).stream().map(e -> { + final Map p = e.entrySet().stream() + .collect(Collectors.toMap(entry -> columnMap.get(entry.getKey().toLowerCase()), entry -> entry.getValue())); + return new JSONObject(p); + }).collect(Collectors.toList()); + + ImportResp diffRes = diff(dbRows, importRows); + diffRes.insertRows.stream().limit(limit).forEach(e -> doSaveOrUpdate(e, true)); + diffRes.updateRows.stream().limit(limit).forEach(e -> doSaveOrUpdate(e, false)); + return diffRes; + } + + private void doSaveOrUpdate(T entity, boolean insert) { + if (saveOrUpdate != null) { + saveOrUpdate.accept(entity, insert); + } else { + if (insert) { + baseMapper.insert(entity); + } else { + baseMapper.updateById(entity); + } + } + } + + /** + * 解决数据库的列名到属性名的转换, 并过滤不存在和需要过滤的列 + * + * @param rows + * @return + */ + List resolveRows(List rows) { + TableInfo tableInfo = TableInfoHelper.getTableInfo(clz); + // 兼容客户端上传的数据是表的column名称(下划线), 或者是熟悉的名称(camel) + ImmutableMap columnMap = Maps.uniqueIndex(tableInfo.getFieldList(), TableFieldInfo::getColumn); + ImmutableMap propertyMap = Maps.uniqueIndex(tableInfo.getFieldList(), TableFieldInfo::getProperty); + Map columnPropertyMap = new HashMap<>(columnMap); + columnPropertyMap.putAll(propertyMap); + + return rows.stream().map(e -> { + Map collect = e.entrySet().stream().map(node -> { + TableFieldInfo column = columnPropertyMap.get(node.getKey()); + if (column == null) { + return Pair.of("", ""); + } + Object nodeValue = node.getValue(); + if (jsonSmart && nodeValue != null && (node.getValue() instanceof String)) { + String value = (String) (node.getValue()); + if (value.startsWith("{") && JSON.isValidObject(value)) { + nodeValue = JSONObject.parseObject(value); + } + if (value.startsWith("[") && JSON.isValidArray(value)) { + nodeValue = JSONObject.parseArray(value); + } + if (column.getTypeHandler() == SetTypeHandler.class) { + nodeValue = ImmutableSet.copyOf(Splitter.on(",").omitEmptyStrings().trimResults().splitToList(value)); + } + } + return Pair.of(column.getProperty(), nodeValue); + }) + .filter(x -> x.getValue() != null) + .filter(x -> !excludeFields.contains(x.getKey())) + .collect(Collectors.toMap(x -> x.getKey(), x -> x.getValue())); + return new JSONObject(collect); + }).collect(Collectors.toList()); + } + + ImportResp diff(List dbRows, List importRows) { + // 通过逻辑biz key来关联数据库和导入的数据, 得到需要需要创建和更新的数据. + ImmutableMap dbRowMap = Maps.uniqueIndex(dbRows, e -> { + return bizKeyNames.stream().map(key -> e.getString(key)).collect(Collectors.joining(",")); + }); + + ImmutableMap importRowMap = Maps.uniqueIndex(importRows, e -> { + return bizKeyNames.stream().map(key -> e.getString(key)).collect(Collectors.joining(",")); + }); + ImportResp res = new ImportResp(); + for (Map.Entry entry : importRowMap.entrySet()) { + JSONObject dbRow = dbRowMap.get(entry.getKey()); + JSONObject importRow = entry.getValue(); + if (dbRow != null) { + // 关联成功 + if (isSame(dbRow, importRow)) { + res.sameRows.add(JSONObject.toJavaObject(importRow, clz)); + } else { + // update the id by db row's + JSONObject updated = new JSONObject(importRow); + updated.put(this.idFieldName, dbRow.getOrDefault(this.idFieldName, null)); + res.updateRows.add(JSONObject.toJavaObject(updated, clz)); + } + } else { + // 没有关联上, 说明需要新增. + res.insertRows.add(JSONObject.toJavaObject(importRow, clz)); + } + } + return res; + } + + /** + * 将json对象串成一个字段串, 来比较2个json是否一致. + * + * @param src + * @param target + * @return + */ + boolean isSame(JSONObject src, JSONObject target) { + Function mixer = i -> { + return i.entrySet().stream().filter(e -> !excludeFields.contains(e.getKey())) + .sorted(Comparator.comparing(Map.Entry::getKey)) + .filter(e -> e.getValue() != null) + .filter(e -> !Strings.isNullOrEmpty(e.getValue().toString())) + .map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(",")); + }; + String srcText = mixer.apply(src); + String targetText = mixer.apply(target); + boolean res = srcText.equals(targetText); + return res; + } + + @NoArgsConstructor + @AllArgsConstructor + @Data + @Builder + public final static class ImportResp { + List updateRows = Lists.newArrayList(); + List insertRows = Lists.newArrayList(); + List sameRows = Lists.newArrayList(); + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusCacheHelper.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusCacheHelper.java new file mode 100644 index 0000000..c535c40 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusCacheHelper.java @@ -0,0 +1,132 @@ +package cn.axzo.pokonyan.dao.mysql; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.core.toolkit.support.SFunction; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NonNull; + +import java.io.Serializable; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + *通过basempper 读取 mysql 中数据,并缓存LoadingCache中 + * 提供了 2 中构建方式 + *
    + *
  • + *

    IdBuilder


    + * LoadingCache> cache = MybatisPlusCache.KeyBuilder.builder() + * .expire(Duration.ofMinutes(5)).maxSize(100L).baseMapper(mapper) + * .keyFunction(User::gtiId).build().toLoadingCache(); + *
  • + *
  • + *

    AllBuilder


    + * LoadingCache> cache = MybatisPlusCache.AllBuilder.builder() + * .expire(Duration.ofMinutes(5)).maxSize(100L).baseMapper(mapper) + * .queryBuilder(k -> new QueryWrapper()) + * .entityFilter(e -> false) + * .build() + * .toLoadingCache(); + *
  • + *
+ * + * + */ +public class MybatisPlusCacheHelper { + + @Builder + @AllArgsConstructor + public static class KeyBuilder { + @NonNull + private BaseMapper baseMapper; + @NonNull + private Long maxSize; + @NonNull + private Duration expire; + @NonNull + SFunction keyFunction; + + public LoadingCache> toLoadingCache() { + return CacheBuilder.newBuilder() + .expireAfterWrite(expire) + .maximumSize(maxSize) + .recordStats() + .build(new CacheLoader>() { + @Override + public Optional load(K key) throws Exception { + return Optional.ofNullable(baseMapper.selectOne(Wrappers.lambdaQuery().eq(keyFunction, key))); + } + + @Override + public Map> loadAll(Iterable keys) throws Exception { + LambdaQueryWrapper query = Wrappers.lambdaQuery().in(keyFunction, ImmutableList.copyOf(keys)); + Map rows = baseMapper.selectList(query).stream().collect(Collectors.toMap(keyFunction, e -> e)); + Map> res = Maps.newHashMapWithExpectedSize(rows.size()); + for (K key : keys) { + res.put(key, Optional.ofNullable(rows.get(key))); + } + return res; + } + }); + } + } + + @Builder + @AllArgsConstructor + public static class AllBuilder { + @NonNull + private BaseMapper baseMapper; + @NonNull + private Long maxSize; + @NonNull + private Duration expire; + /** + * 查询 QueryWrapper 构建器 + */ + Function> queryBuilder; + + /** + * 实体过滤器 + */ + Predicate entityFilter; + + public LoadingCache> toLoadingCache(@NonNull Function entityConverter) { + return CacheBuilder.newBuilder() + .expireAfterWrite(expire) + .maximumSize(maxSize) + .recordStats() + .build(new CacheLoader>() { + @Override + public List load(K key) throws Exception { + Wrapper query = null; + if (queryBuilder != null) { + query = queryBuilder.apply(key); + } + List res = baseMapper.selectList(query); + if (entityFilter != null) { + res = res.stream().filter(entityFilter).collect(Collectors.toList()); + } + return res.stream().map(e -> entityConverter.apply(e)).collect(Collectors.toList()); + } + }); + } + + public LoadingCache> toLoadingCache() { + return toLoadingCache(e->e); + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusConverterUtils.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusConverterUtils.java new file mode 100644 index 0000000..c1dd7f8 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusConverterUtils.java @@ -0,0 +1,61 @@ +package cn.axzo.pokonyan.dao.mysql; + +import cn.axzo.pokonyan.dao.wrapper.SimpleWrapperConverter; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.TableFieldInfo; +import com.baomidou.mybatisplus.core.metadata.TableInfo; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.google.common.base.Strings; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +public class MybatisPlusConverterUtils { + + private static Map> converters = new ConcurrentHashMap<>(32); + + public static SimpleWrapperConverter getWrapperConverter(Class entityClass) { + return converters.computeIfAbsent(entityClass, clazz -> + SimpleWrapperConverter.builder() + .operatorProcessor(new MybatisPlusOperatorProcessor()) + .fieldColumnMap(getFieldMapping(clazz)) + .fieldTypeMap(getFieldTypeMapping(clazz)) + .build()); + } + + /** + * 返回class中property 对应的 column + * + * @param clazz + * @return + */ + public static Map getFieldMapping(Class clazz) { + // XXX: TableInfoHelper.getTableInfo(clazz).getFieldList返回的映射关系是不包含@TableId注解(会导致根据id查询的列找不到。) + // 在获取property和column映射关系的时候,需要聚合filedList和@TableId + TableInfo tableInfo = TableInfoHelper.getTableInfo(clazz); + Map fieldMap = tableInfo.getFieldList().stream() + .collect(Collectors.toMap(TableFieldInfo::getProperty, TableFieldInfo::getColumn)); + if (!Strings.isNullOrEmpty(tableInfo.getKeyProperty())) { + fieldMap.put(tableInfo.getKeyProperty(), tableInfo.getKeyColumn()); + } + return fieldMap; + } + + /** + * 返回class中property 对应的 propertyClass + * + * @param clazz + * @return + */ + public static Map> getFieldTypeMapping(Class clazz) { + TableInfo tableInfo = TableInfoHelper.getTableInfo(clazz); + Map> fieldTypeMap = tableInfo.getFieldList().stream() + .collect(Collectors.toMap(TableFieldInfo::getProperty, TableFieldInfo::getPropertyType)); + + if (!Strings.isNullOrEmpty(tableInfo.getKeyProperty())) { + fieldTypeMap.put(tableInfo.getKeyProperty(), tableInfo.getKeyType()); + } + return fieldTypeMap; + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusHelper.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusHelper.java new file mode 100644 index 0000000..b9ed965 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusHelper.java @@ -0,0 +1,73 @@ +package cn.axzo.pokonyan.dao.mysql; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.toolkit.support.SFunction; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * @author yuanyi + * Created on 2020/9/27. + */ +public class MybatisPlusHelper { + + private static final Integer LIMIT = 1_000; + + public static List drainAll(BaseMapper baseMapper, QueryWrapper queryWrapper, + SFunction idFunction) { + return drainAll(baseMapper, queryWrapper::lambda, idFunction, LIMIT); + } + + public static List drainAll(BaseMapper baseMapper, QueryWrapper queryWrapper, + SFunction idFunction, int limit) { + return drainAll(baseMapper, queryWrapper::lambda, idFunction, limit); + } + + public static List drainAll(BaseMapper baseMapper, Supplier> wrapperSupplier, + SFunction idFunction) { + return drainAll(baseMapper, wrapperSupplier, idFunction, LIMIT); + } + + public static List drainAll(BaseMapper baseMapper, Supplier> wrapperSupplier, + SFunction idFunction, int limit) { + LambdaQueryWrapper queryWrapper = wrapperSupplier.get(); + Preconditions.checkArgument(!StringUtils.containsIgnoreCase(queryWrapper.getSqlSegment(), "order by"), + "queryWrapper不能含有order by"); + + LambdaQueryWrapper nextQueryWrapper = wrapperSupplier.get(); + Preconditions.checkArgument(queryWrapper != nextQueryWrapper, + "wrapperSupplier需要返回不同的LambdaQueryWrapper实例"); + + Function> wrapperFunc = startId -> wrapperSupplier.get() + .gt(idFunction, startId) + .orderByAsc(idFunction) + .last(" limit " + limit); + return drainAll(baseMapper, wrapperFunc, idFunction::apply, limit); + } + + private static List drainAll(BaseMapper baseMapper, Function> wrapperFunc, + Function startIdFunc, int limit) { + List totalData = Lists.newArrayList(); + Long startId = 0L; + while (true) { + List records = baseMapper.selectList(wrapperFunc.apply(startId)); + totalData.addAll(records); + + if (records.size() < limit) { + break; + } + + T lastOne = records.get(records.size() - 1); + startId = startIdFunc.apply(lastOne); + } + return totalData; + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusOperatorProcessor.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusOperatorProcessor.java new file mode 100644 index 0000000..0e6480c --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/MybatisPlusOperatorProcessor.java @@ -0,0 +1,141 @@ +package cn.axzo.pokonyan.dao.mysql; + +import cn.axzo.pokonyan.dao.wrapper.CriteriaWrapper; +import cn.axzo.pokonyan.dao.wrapper.Operator; +import cn.axzo.pokonyan.dao.wrapper.OperatorProcessor; +import cn.axzo.pokonyan.dao.wrapper.TriConsumer; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.google.common.collect.ImmutableListMultimap; +import org.springframework.data.domain.Sort; + +import java.util.Collection; +import java.util.List; + +public class MybatisPlusOperatorProcessor implements OperatorProcessor> { + + @Override + public QueryWrapper assembleAllQueryWrapper(ImmutableListMultimap queryColumnMap, boolean andOperator) { + QueryWrapper queryWrapper = Wrappers.query(); + queryColumnMap.asMap().forEach((column, queryFields) -> { + // 获取processor,拼装wrapper + if (andOperator) { + queryFields.forEach(queryField -> get(queryField.getOperator()) + .accept(queryWrapper, queryField.getColumnWithPrefix(column), queryField.getValue())); + } else { + queryFields.forEach(queryField -> { + get(queryField.getOperator()) + .accept(queryWrapper, queryField.getColumnWithPrefix(column), queryField.getValue()); + queryWrapper.or(); + }); + } + }); + return queryWrapper; + } + + public TriConsumer, String, Object> get(Operator operator) { + switch (operator) { + case LIKE: + return QueryWrapper::like; + case EQ: + return QueryWrapper::eq; + case LT: + return QueryWrapper::lt; + case LE: + return QueryWrapper::le; + case GT: + return QueryWrapper::gt; + case GE: + return QueryWrapper::ge; + case NE: + return QueryWrapper::ne; + case SW: + return QueryWrapper::likeRight; + case EW: + return QueryWrapper::likeLeft; + case IN: + return this::in; + case IS_NULL: + return this::isNull; + case IS_NOT_NULL: + return this::isNotNull; + case BETWEEN: + return this::between; + case ORDER: + return this::order; + case OR: + return this::or; + case JSON: + return this::json; + case JSON_OR: + return this::jsonOr; + case FS: + throw new UnsupportedOperationException("暂不支持的操作符"); + + default: + throw new UnsupportedOperationException("暂不支持的操作符"); + } + } + + private void or(QueryWrapper queryWrapper, String column, Object value) { + List criterials = (List) value; + queryWrapper.and(e -> { + for (CriteriaWrapper.QueryField q : criterials) { + get(q.getOperator()).accept(e, q.getFieldWithPrefix(), q.getValue()); + e.or(); + } + }); + } + + private void json(QueryWrapper queryWrapper, String column, Object value) { + List criterials = (List) value; + queryWrapper.and(e -> { + for (CriteriaWrapper.QueryField q : criterials) { + get(q.getOperator()).accept(e, q.getFieldWithPrefix(), q.getValue()); + } + }); + } + + private void jsonOr(QueryWrapper queryWrapper, String column, Object value) { + List criterials = (List) value; + queryWrapper.and(e -> { + for (CriteriaWrapper.QueryField q : criterials) { + get(q.getOperator()).accept(e, q.getFieldWithPrefix(), q.getValue()); + e.or(); + } + }); + } + + private void isNull(QueryWrapper queryWrapper, String column, Object value) { + queryWrapper.isNull(column); + } + + private void isNotNull(QueryWrapper queryWrapper, String column, Object value) { + queryWrapper.isNotNull(column); + } + + private void between(QueryWrapper queryWrapper, String column, Object value) { + List valueList = (List) value; + queryWrapper.between(column, valueList.get(0), valueList.get(1)); + } + + private void in(QueryWrapper queryWrapper, String column, Object v) { + Collection values = (Collection) v; + boolean isJson = CriteriaWrapper.QueryField.isJsonQueryField(column); + if (isJson) { + queryWrapper.and(w -> values.forEach(value -> w.or().apply(column + "={0}", value))); + } else { + queryWrapper.in(column, values); + } + } + + private void order(QueryWrapper queryWrapper, String column, Object value) { + // 如果value不能转成Direction,会抛异常 + if (Sort.Direction.fromString(String.valueOf(value)).isDescending()) { + queryWrapper.orderByDesc(column); + return; + } + + queryWrapper.orderByAsc(column); + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/MysqlJsonHelper.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/MysqlJsonHelper.java new file mode 100644 index 0000000..12dee2a --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/MysqlJsonHelper.java @@ -0,0 +1,117 @@ +package cn.axzo.pokonyan.dao.mysql; + +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import lombok.NonNull; +import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.StringUtils; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * 一个 mysql 的 json 操作 helper,方便构建 json 操作sql + */ +@UtilityClass +public class MysqlJsonHelper { + + /** + * 构建 json 更新sql。支持嵌套内容的更新。 + * mysql 的 json_set 需要嵌套的路径中对象存在,否则更新失败。 + * https://stackoverflow.com/questions/40896920/mysql-json-set-cant-insert-into-column-with-null-value5-7 + *

+ * String sql = MysqlJsonHelper.buildJsonSetSql("ext", + * ImmutableMap.of("test2.test4.test5", "100", "test2.test3", "55", "test2.test4.test6", "999")); + * 产生结果: + * ext = COALESCE(ext, JSON_OBJECT()), + * ext = JSON_SET(ext, '$.test2', IFNULL(ext->'$.test2',JSON_OBJECT())), + * ext = JSON_SET(ext, '$.test2.test4', IFNULL(ext->'$.test2.test4',JSON_OBJECT())), + * ext = JSON_SET(ext, '$.test2.test4.test5', '100'), + * ext = JSON_SET(ext, '$.test2.test3', '55'), + * ext = JSON_SET(ext, '$.test2.test4.test6', '999') + * + * @return json_set sql 语句 + */ + public String buildJsonSetSql(@NonNull String colName, @NonNull Map values) { + Preconditions.checkArgument(!values.isEmpty()); + + return values.entrySet().stream().flatMap(e -> { + List nodes = Splitter.on(".").splitToList(e.getKey()); + List sqls = Lists.newArrayList(); + String jsonPath = "$"; + // json_set 不支持上级节点是null,这里需要产生 sql 来初始化上级节点。 + sqls.add(String.format("%s = COALESCE(%s, JSON_OBJECT())", colName, colName)); + for (int i = 0; i < nodes.size() - 1; i++) { + jsonPath = Joiner.on(".").join(jsonPath, nodes.get(i)); + String sql = String.format("%s = JSON_SET(%s, '%s', IFNULL(%s,JSON_OBJECT()))", colName, colName, jsonPath, buildJsonField(colName, jsonPath)); + sqls.add(sql); + } + String sqlValue = String.format("%s = JSON_SET(%s, '%s', %s)", colName, colName, "$." + e.getKey(), resolveValue(e.getValue())); + sqls.add(sqlValue); + return sqls.stream(); + }).distinct().collect(Collectors.joining(",\n")); + } + + private Object resolveValue(Object value) { + if (value instanceof String) { + return "'" + value + "'"; + } + + if (value instanceof List) { + return "JSON_ARRAY(JSON_OBJECT" + Joiner.on(",JSON_OBJECT").join((List) value) + .replace("\":\"", "\",\"") + .replace("{", "(") + .replace("}", ")") + ")"; + } + return value; + } + + public String buildJsonField(@NonNull String colName, @NonNull String path) { + Preconditions.checkArgument(StringUtils.isNotBlank(colName)); + Preconditions.checkArgument(StringUtils.isNotBlank(path)); + + return String.format("%s->'%s'", colName, (path.startsWith("$") ? path : "$." + path)); + } + + /** + * mysql 暂不支持JSON IN 查询 + * 这里把 in 转换成 or 查询。 + * //https://dev.mysql.com/doc/refman/5.7/en/json.html#json-comparison + */ + public QueryWrapper buildJsonIn(@NonNull QueryWrapper wrapper, @NonNull String colName, @NonNull String path, @NonNull Collection values) { + Preconditions.checkArgument(StringUtils.isNotBlank(colName)); + Preconditions.checkArgument(StringUtils.isNotBlank(path)); + + wrapper.and(w -> values.forEach(value -> { + w.or().apply(buildJsonField(colName, path) + "={0}", value); + })); + + return wrapper; + } + + /** + * mysql 暂不支持JSON ARRAY 的 IN 查询 + * 这里把 in 转换成 or 查询。 + * //https://dev.mysql.com/doc/refman/5.7/en/json.html#json-comparison + */ + public QueryWrapper buildJsonContains(@NonNull QueryWrapper wrapper, @NonNull String colName, @NonNull String path, @NonNull Collection values) { + Preconditions.checkArgument(StringUtils.isNotBlank(colName)); + Preconditions.checkArgument(StringUtils.isNotBlank(path)); + + String normalizedPath = (path.startsWith("$") ? path : "$." + path); + wrapper.and(w -> values.forEach(value -> { + String sql = String.format("JSON_CONTAINS(%s, '%s', '%s')", colName, + // XXX Long类型可能会被序列化为带引号的字符串(为了兼容前端),因此Long类型不使用json来序列化 + (value instanceof Long) ? value : JSONObject.toJSONString(value), normalizedPath); + w.or().apply(sql); + })); + + return wrapper; + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/QueryWrapperHelper.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/QueryWrapperHelper.java new file mode 100644 index 0000000..acd8902 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/QueryWrapperHelper.java @@ -0,0 +1,50 @@ +package cn.axzo.pokonyan.dao.mysql; + +import cn.axzo.pokonyan.dao.wrapper.CriteriaWrapper; +import cn.axzo.pokonyan.dao.wrapper.SimpleWrapperConverter; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class QueryWrapperHelper { + + public static QueryWrapper fromBean(Object bean, Class clazz) { + SimpleWrapperConverter converter = MybatisPlusConverterUtils.getWrapperConverter(clazz); + return converter.toWrapper(CriteriaWrapper.fromBean(bean)); + } + + public static QueryWrapper fromMap(Map map, Class clazz) { + SimpleWrapperConverter converter = MybatisPlusConverterUtils.getWrapperConverter(clazz); + return converter.toWrapper(CriteriaWrapper.builder().queryField(map).build()); + } + + public static QueryWrapper fromBean(Object bean, Class clazz, + Function> fieldConverter) { + SimpleWrapperConverter converter = MybatisPlusConverterUtils.getWrapperConverter(clazz); + return converter.toWrapper(CriteriaWrapper.fromBean(bean, true, fieldConverter)); + } + + public static QueryWrapper fromBean(Object bean, Class clazz, Class... others) { + SimpleWrapperConverter converter = MybatisPlusConverterUtils.getWrapperConverter(clazz); + return converter.toWrapper(CriteriaWrapper.fromBean(bean), Arrays.stream(others) + .map(MybatisPlusConverterUtils::getWrapperConverter).toArray(SimpleWrapperConverter[]::new)); + } + + public static QueryWrapper fromMap(Map map, Class clazz, Class... others) { + SimpleWrapperConverter converter = MybatisPlusConverterUtils.getWrapperConverter(clazz); + return converter.toWrapper(CriteriaWrapper.builder().queryField(map).build(), Arrays.stream(others) + .map(MybatisPlusConverterUtils::getWrapperConverter).toArray(SimpleWrapperConverter[]::new)); + } + + public static QueryWrapper fromBean(Object bean, Class clazz, + Function> fieldConverter, + Class... others) { + SimpleWrapperConverter converter = MybatisPlusConverterUtils.getWrapperConverter(clazz); + return converter.toWrapper(CriteriaWrapper.fromBean(bean, true, fieldConverter), Arrays.stream(others) + .map(MybatisPlusConverterUtils::getWrapperConverter).toArray(SimpleWrapperConverter[]::new)); + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/type/BaseListTypeHandler.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/type/BaseListTypeHandler.java new file mode 100644 index 0000000..a4a35ab --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/type/BaseListTypeHandler.java @@ -0,0 +1,51 @@ +package cn.axzo.pokonyan.dao.mysql.type; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.serializer.SerializerFeature; +import org.apache.ibatis.type.BaseTypeHandler; +import org.apache.ibatis.type.JdbcType; +import org.apache.ibatis.type.MappedJdbcTypes; +import org.apache.ibatis.type.MappedTypes; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +@MappedTypes({List.class}) +@MappedJdbcTypes(JdbcType.VARCHAR) +public abstract class BaseListTypeHandler extends BaseTypeHandler> { + + private Class type = getGenericType(); + + @Override + public void setNonNullParameter(PreparedStatement preparedStatement, int i, + List list, JdbcType jdbcType) throws SQLException { + preparedStatement.setString(i, JSONArray.toJSONString(list, SerializerFeature.WriteMapNullValue, + SerializerFeature.WriteNullListAsEmpty, SerializerFeature.WriteNullStringAsEmpty)); + } + + @Override + public List getNullableResult(ResultSet resultSet, String s) throws SQLException { + return JSONArray.parseArray(resultSet.getString(s), type); + } + + @Override + public List getNullableResult(ResultSet resultSet, int i) throws SQLException { + return JSONArray.parseArray(resultSet.getString(i), type); + } + + @Override + public List getNullableResult(CallableStatement callableStatement, int i) throws SQLException { + return JSONArray.parseArray(callableStatement.getString(i), type); + } + + private Class getGenericType() { + Type t = getClass().getGenericSuperclass(); + Type[] params = ((ParameterizedType) t).getActualTypeArguments(); + return (Class) params[0]; + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/type/LinkedHashSetTypeHandler.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/type/LinkedHashSetTypeHandler.java new file mode 100644 index 0000000..51a2aae --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/type/LinkedHashSetTypeHandler.java @@ -0,0 +1,54 @@ +package cn.axzo.pokonyan.dao.mysql.type; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.Sets; +import org.apache.ibatis.type.BaseTypeHandler; +import org.apache.ibatis.type.JdbcType; +import org.apache.ibatis.type.MappedJdbcTypes; + +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.LinkedHashSet; +import java.util.Optional; + +/** + * 1. 用于将数据库中的, 逗号分割的String, 转换为LinkedHashSet. + * 2. 存储时将LinkedHashSet直接存String, 多个使用逗号分割 + */ +@MappedJdbcTypes({JdbcType.VARCHAR}) +public class LinkedHashSetTypeHandler extends BaseTypeHandler> { + + private static final String DELIMITER = ","; + + @Override + public void setNonNullParameter(PreparedStatement ps, int i, LinkedHashSet parameter, JdbcType jdbcType) throws SQLException { + String value = Joiner.on(DELIMITER).join(Optional.ofNullable(parameter).orElseGet(Sets::newLinkedHashSet)); + ps.setString(i, value); + } + + @Override + public LinkedHashSet getNullableResult(ResultSet rs, String columnName) throws SQLException { + return getSet(rs.getString(columnName)); + } + + @Override + public LinkedHashSet getNullableResult(ResultSet rs, int columnIndex) throws SQLException { + return getSet(rs.getString(columnIndex)); + } + + @Override + public LinkedHashSet getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { + return getSet(cs.getString(columnIndex)); + } + + private LinkedHashSet getSet(String dbValue) { + return Sets.newLinkedHashSet(Splitter.on(DELIMITER) + .omitEmptyStrings() + .trimResults() + .splitToList(Strings.nullToEmpty(dbValue))); + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/mysql/type/SetTypeHandler.java b/src/main/java/cn/axzo/pokonyan/dao/mysql/type/SetTypeHandler.java new file mode 100644 index 0000000..5bece50 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/mysql/type/SetTypeHandler.java @@ -0,0 +1,60 @@ +package cn.axzo.pokonyan.dao.mysql.type; + +import com.google.common.base.Splitter; +import com.google.common.collect.Sets; +import org.apache.commons.lang3.StringUtils; +import org.apache.ibatis.type.BaseTypeHandler; +import org.apache.ibatis.type.JdbcType; +import org.apache.ibatis.type.MappedJdbcTypes; + +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 1. 用于将数据库中的, 逗号分割的String, 转换为Set. + * 2. 存储时将Set直接存String, 多个使用逗号分割 + */ +@MappedJdbcTypes({JdbcType.VARCHAR}) +public class SetTypeHandler extends BaseTypeHandler> { + + private static final String DELIMITER = ","; + + @Override + public void setNonNullParameter(PreparedStatement ps, int i, Set parameter, JdbcType jdbcType) throws SQLException { + String value = Sets.newLinkedHashSet(Optional.ofNullable(parameter).orElse(Collections.emptySet())) + .stream() + .collect(Collectors.joining(DELIMITER)); + + ps.setString(i, value); + } + + @Override + public Set getNullableResult(ResultSet rs, String columnName) throws SQLException { + return getSet(rs.getString(columnName)); + } + + @Override + public Set getNullableResult(ResultSet rs, int columnIndex) throws SQLException { + return getSet(rs.getString(columnIndex)); + } + + @Override + public Set getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { + return getSet(cs.getString(columnIndex)); + } + + private Set getSet(String dbValue) { + return Splitter.on(",") + .omitEmptyStrings() + .trimResults() + .splitToList(Optional.of(dbValue).orElse(StringUtils.EMPTY)) + .stream() + .collect(Collectors.toSet()); + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/page/IPageParam.java b/src/main/java/cn/axzo/pokonyan/dao/page/IPageParam.java new file mode 100644 index 0000000..eab36eb --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/page/IPageParam.java @@ -0,0 +1,37 @@ +package cn.axzo.pokonyan.dao.page; + +import com.google.common.collect.ImmutableList; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.List; + +public interface IPageParam { + Integer DEFAULT_PAGE_NUMBER = 1; + Integer DEFAULT_PAGE_SIZE = 20; + Integer MAX_PAGE_SIZE = 1000; + + String SORT_DELIMITER = "__"; + String SORT_DESC = OrderEnum.DESC.name(); + String SORT_ASC = OrderEnum.ASC.name(); + + default Integer getPageNumber() { + return DEFAULT_PAGE_NUMBER; + } + + default Integer getPageSize() { + return DEFAULT_PAGE_SIZE; + } + + default List getSort() { + return ImmutableList.of(); + } + + @RequiredArgsConstructor + @Getter + public static enum OrderEnum { + DESC, + ASC; + } + +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/utils/RepairDataHelper.java b/src/main/java/cn/axzo/pokonyan/dao/utils/RepairDataHelper.java new file mode 100644 index 0000000..af14cd6 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/utils/RepairDataHelper.java @@ -0,0 +1,255 @@ +package cn.axzo.pokonyan.dao.utils; + +import cn.axzo.pokonyan.dao.mysql.MybatisPlusConverterUtils; +import cn.axzo.pokonyan.dao.wrapper.SimpleWrapperConverter; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.TableInfo; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.google.common.base.Preconditions; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cglib.beans.BeanMap; +import org.springframework.util.CollectionUtils; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * 提供修复数据的工具类, 支持mysql. 修复方式为通过指定起始id批量查询并修复 + * + * @param mapper {@link BaseMapper}对象 + * @param clz 实体类 + * @param updater 用于更新数据库中查询出来的记录 + * @param queryWrapper 可选. 查询数据库时指定的查询条件. 注意因为需要clone该wrapper, 要求其中的查询参数是serializable的,否则可能出错 + * @param selectFields 可选. 查询数据库时指定返回的字段 + */ +@Slf4j +public class RepairDataHelper { + + private RepositoryWrapper repositoryWrapper; + private Function, List> updater; + + @Builder + public RepairDataHelper(BaseMapper mapper, Class clz, Function, List> updater, + QueryWrapper queryWrapper, Set selectFields) { + Preconditions.checkArgument(mapper != null, "mapper不能为空"); + Preconditions.checkArgument(clz != null, "clz不能为空"); + Preconditions.checkArgument(updater != null, "updater不能为空"); + + if (queryWrapper == null) { + queryWrapper = Wrappers.query(); + } + repositoryWrapper = new MysqlRepositoryWrapper(mapper, clz, queryWrapper, selectFields); + this.updater = updater; + + log.info("---Repair Data Helper--- created with repository: {}, queryWrapper: {}, selectFields: {}", + mapper, queryWrapper, selectFields); + } + + /** + * 执行数据修复 + * + * @param req startId 从指定的起始id开始修复数据. 默认从第一条记录开始 + * batchSize 一个批次处理的数据量. 默认一个批次仅处理20条数据 + * limit 必填项, 限制处理数据总条数. + * onlyPrint true表示仅打印需要修复的数据, 不做修复动作. 默认不执行修复动作 + * @return 总的修复记录数 + */ + public int repair(RepairReq req) { + req = Optional.ofNullable(req).orElse(RepairReq.DEFAULT); + Preconditions.checkArgument(req.getBatchSize() > 0, "batchCount必须大于0"); + Preconditions.checkArgument(req.getLimit() > 0, "limit必须大于0"); + + log.info("---Repair Data Helper--- repair with req: {}", req); + + int batchIndex = 0; + int totalProcessed = 0; + int totalRepaired = 0; + Serializable startId = req.getStartId(); + + while (totalProcessed < req.getLimit()) { + log.info("[{}]---Repair Data Helper--- start process, startId: {}", batchIndex, startId); + + boolean includeStartId = batchIndex == 0; + int batchSize = Math.min(req.getBatchSize(), req.getLimit() - totalProcessed); + List selectedRecords = repositoryWrapper.selectByStartId(startId, batchSize, includeStartId); + log.info("[{}]---Repair Data Helper--- selected records count: {}", batchIndex, selectedRecords.size()); + + if (CollectionUtils.isEmpty(selectedRecords)) { + break; + } + totalProcessed += selectedRecords.size(); + + List updateRecords = updater.apply(selectedRecords); + log.info("[{}]---Repair Data Helper--- update records count: {}", batchIndex, updateRecords.size()); + + List updateIds = updateRecords.stream() + .map(r -> getIdValue(r, repositoryWrapper.getIdFieldName())) + .collect(Collectors.toList()); + + + if (req.getOnlyPrint() || CollectionUtils.isEmpty(updateIds)) { + log.info("[{}]---Repair Data Helper--- to be updated ids: {}", batchIndex, updateIds); + } else { + int updatedCount; + if (needBatchUpdate(updateRecords)) { + updatedCount = repositoryWrapper.batchUpdate(updateRecords); + } else { + // 所有待更新的数据都是相同的,可以通过条件更新一次性更新 + updatedCount = repositoryWrapper.updateByIds(updateRecords.get(0), updateIds); + } + + log.info("[{}]---Repair Data Helper--- updated records count: {}", batchIndex, updatedCount); + totalRepaired += updatedCount; + } + + startId = getIdValue(selectedRecords.get(selectedRecords.size() - 1), repositoryWrapper.getIdFieldName()); + batchIndex += 1; + } + + log.info("---Repair Data Helper--- repair finished, req: {}, processed: {}, repaired: {}, ignored: {}", + req, totalProcessed, totalRepaired, totalProcessed - totalRepaired); + + return totalRepaired; + } + + private interface RepositoryWrapper { + String getIdFieldName(); + + List selectByStartId(Serializable startId, int count, boolean includeStartId); + + int updateByIds(T updateEntity, List ids); + + int batchUpdate(List updateEntities); + } + + private class MysqlRepositoryWrapper implements RepositoryWrapper { + private QueryWrapper queryWrapper; + private Set selectFields; + + private BaseMapper baseMapper; + private String idFieldName; + private String idColumnName; + + private SimpleWrapperConverter converter; + + private MysqlRepositoryWrapper(BaseMapper baseMapper, Class entityClass, + QueryWrapper queryWrapper, Set selectFields) { + try { + TableInfo tableInfo = TableInfoHelper.getTableInfo(entityClass); + idFieldName = tableInfo.getKeyProperty(); + idColumnName = tableInfo.getKeyColumn(); + converter = MybatisPlusConverterUtils.getWrapperConverter(entityClass); + } catch (Exception e) { + } + Preconditions.checkArgument(baseMapper != null, "repository找不到baseMapper"); + Preconditions.checkArgument(entityClass != null, "repository找不到entityClass"); + Preconditions.checkArgument(idFieldName != null, "repository找不到idFieldName"); + Preconditions.checkArgument(idColumnName != null, "repository找不到idColumnName"); + Preconditions.checkArgument(converter != null, "repository找不到converter"); + + this.baseMapper = baseMapper; + this.queryWrapper = queryWrapper; + this.selectFields = selectFields; + } + + @Override + public String getIdFieldName() { + return idFieldName; + } + + @Override + public List selectByStartId(Serializable startId, int count, boolean includeStartId) { + // 直接操作queryWrapper将带来副作用,需要将它clone出来 + QueryWrapper clonedWrapper = queryWrapper.clone(); + + if (startId != null) { + if (includeStartId) { + clonedWrapper.ge(idColumnName, startId); + } else { + clonedWrapper.gt(idColumnName, startId); + } + } + + clonedWrapper.orderByAsc(idColumnName).last("LIMIT " + count); + + if (selectFields != null) { + Set columns = selectFields.stream() + .map(converter::getColumnNotNull) + .collect(Collectors.toSet()); + columns.add(idColumnName); + // 注意wrapper的select()方法不支持重复调用,因此需要将所有查询字段放在一个数组里传入 + clonedWrapper.select(columns.toArray(new String[0])); + } + + return baseMapper.selectList(clonedWrapper); + } + + @Override + public int updateByIds(T updateEntity, List ids) { + UpdateWrapper updateWrapper = Wrappers.update().in(idColumnName, ids); + return baseMapper.update(updateEntity, updateWrapper); + } + + @Override + public int batchUpdate(List updateEntities) { + return updateEntities.stream().mapToInt(baseMapper::updateById).sum(); + } + } + + private Serializable getIdValue(Object entity, String idFieldName) { + Map entityFields = BeanMap.create(entity); + Serializable value = (Serializable) entityFields.get(idFieldName); + Preconditions.checkState(value != null, "记录的id不能为空"); + return value; + } + + private boolean needBatchUpdate(List records) { + return records.stream() + .map(record -> ((Map) BeanMap.create(record)).entrySet().stream() + .filter(e -> !repositoryWrapper.getIdFieldName().equals(e.getKey())) + .collect(Collectors.toList())) + .distinct() + .count() > 1; + } + + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + public static class RepairReq { + private String startId; + private Integer batchSize; + private Integer limit; + private Boolean onlyPrint; + + private static final RepairReq DEFAULT = RepairReq.builder() + .batchSize(20) + .limit(1) + .onlyPrint(true) + .build(); + + public Integer getBatchSize() { + return Optional.ofNullable(batchSize).orElse(DEFAULT.batchSize); + } + + public Integer getLimit() { + return Optional.ofNullable(limit).orElse(DEFAULT.limit); + } + + public Boolean getOnlyPrint() { + return Optional.ofNullable(onlyPrint).orElse(DEFAULT.onlyPrint); + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaField.java b/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaField.java new file mode 100644 index 0000000..660ec82 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaField.java @@ -0,0 +1,52 @@ +package cn.axzo.pokonyan.dao.wrapper; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD}) +public @interface CriteriaField { + /** + * 对应Mysql Entity的字段名或者"Field__Operator"格式 + * @return String + */ + String field() default ""; + + Operator operator() default Operator.EQ; + + /** + * 默认为true,当value为空集合或空map的时候,自动过滤该查询条件. + * 主要是考虑到调用方在用fastjson序列化的时候,会将null集合序列化为空集合 + * 所以默认将空集合过滤掉 + * + * @return boolean + */ + boolean filterEmpty() default true; + + /** + * 默认为true,当value为null的时候,自动过滤该查询条件. + * + * @return boolean + */ + boolean filterNull() default true; + + /** + * 是否忽略该字段的查询条件 + * + * @return + */ + boolean ignore() default false; + + /** + * 字段名前缀, 在放入sql时允许指定一个前缀。如前缀'a',则转换后的字段为a.id + * 用于联表查询,比如: + * @Select("select bill.* from bms_plus_pay_bill as bill, bms_plus_pay_audit as audit ${ew.customSqlSegment} " + + * "group by bill.id") + * IPage pageWithQueryPayAudit(IPage page, @Param(Constants.WRAPPER) Wrapper wrapper); + * + * @return + */ + String prefix() default ""; +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaWrapper.java b/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaWrapper.java new file mode 100644 index 0000000..3ff907b --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaWrapper.java @@ -0,0 +1,408 @@ +package cn.axzo.pokonyan.dao.wrapper; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.springframework.cglib.beans.BeanMap; +import org.springframework.util.CollectionUtils; + +import java.lang.reflect.Field; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +@Slf4j +public class CriteriaWrapper { + + private static final String DELIMITER = "__"; + + /** + * 使用ConcurrentMap,HashMap线程不安全 + */ + private static ConcurrentMap> criteriaFieldAnnotations = Maps.newConcurrentMap(); + + @Getter + private Boolean andOperator = true; + + @Getter + private ImmutableListMultimap queryFieldMap; + + public CriteriaWrapper() { + queryFieldMap = ImmutableListMultimap.of(); + } + + public CriteriaWrapper(ImmutableListMultimap queryFieldMap, boolean andOperator) { + this.queryFieldMap = queryFieldMap; + this.andOperator = andOperator; + } + + public static CriteriaWrapperBuilder builder() { + return new CriteriaWrapperBuilder(); + } + + public static class CriteriaWrapperBuilder { + private List queryFields = Lists.newArrayList(); + private boolean andOperator = true; + + /** + * @param key java bean的字段名或者"Field__Operator"格式 + * @param value 除了IS_NULL, IS_NOT_NULL允许null值外,其他情况如果为null该查询条件会被过滤 + * @return CriteriaWrapperBuilder + */ + public CriteriaWrapperBuilder queryField(String key, Object value) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(key)); + return queryField(key, QueryField.getDefaultOperator(key, value), value); + } + + /** + * + * @param key java bean的字段名或者"Field__Operator"格式 + * @param defaultOperator Operator + * @param value 除了IS_NULL, IS_NOT_NULL允许null值外,其他情况如果为null该查询条件会被过滤 + * @return CriteriaWrapperBuilder + */ + public CriteriaWrapperBuilder queryField(String key, Operator defaultOperator, Object value) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(key)); + + QueryField queryField = QueryField.buildWithNullFilter(key, defaultOperator, value); + if (queryField != null) { + this.queryFields.add(queryField); + } + + return this; + } + + /** + * + * @param condition false的时候,该查询条件会被过滤 + * @param key java bean的字段名或者"Field__Operator"格式 + * @param value 如果为null,该查询条件会被过滤 + * @return CriteriaWrapperBuilder + */ + public CriteriaWrapperBuilder queryField(boolean condition, String key, Object value) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(key)); + return queryField(condition, key, QueryField.getDefaultOperator(key, value), value); + } + + /** + * + * @param condition false的时候,该查询条件会被过滤 + * @param key java bean的字段名或者"Field__Operator"格式 + * @param defaultOperator 默认的查询Operator + * @param value 如果为null,该查询条件会被过滤 + * @return CriteriaWrapperBuilder + */ + public CriteriaWrapperBuilder queryField(boolean condition, String key, Operator defaultOperator, Object value) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(key)); + return queryField(condition, key, defaultOperator, value, StringUtils.EMPTY); + } + + /** + * + * @param condition false的时候,该查询条件会被过滤 + * @param key java bean的字段名或者"Field__Operator"格式 + * @param defaultOperator 默认的查询Operator + * @param value 如果为null,该查询条件会被过滤 + * @param prefix 字段名添加一个前缀 + * @return CriteriaWrapperBuilder + */ + public CriteriaWrapperBuilder queryField(boolean condition, String key, Operator defaultOperator, + Object value, String prefix) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(key)); + + if (!condition) { + return this; + } + + queryFields.add(QueryField.build(key, defaultOperator, value, prefix)); + return this; + } + + /** + * + * @param params {key, value}, key支持"Field__Operator"格式;value如果为null,该查询条件会被过滤 + * @return CriteriaWrapperBuilder + */ + public CriteriaWrapperBuilder queryField(Map params) { + return queryField(params, null); + } + + /** + * + * @param params {key, value}, key支持"Field__Operator"格式;value如果为null,该查询条件会被过滤 + * @param converter 对querfield进行转换. 比如查询条件, 查询值. + * @return CriteriaWrapperBuilder + */ + public CriteriaWrapperBuilder queryField(Map params, Function converter) { + List queryFields = params.entrySet().stream() + .map(entry -> QueryField.build(entry.getKey(), entry.getValue())) + .map(queryField -> converter == null ? queryField : converter.apply(queryField)) + .filter(Objects::nonNull) + .filter(queryField -> queryField.getValue() != null || queryField.getOperator().allowNullValue()) + .collect(Collectors.toList()); + this.queryFields.addAll(queryFields); + return this; + } + + public CriteriaWrapperBuilder queryFields(List queryFields) { + this.queryFields.addAll(queryFields); + return this; + } + + public CriteriaWrapperBuilder andOperator(boolean andOperator) { + this.andOperator = andOperator; + return this; + } + + public CriteriaWrapper build() { + ImmutableListMultimap multimap = queryFields.stream() + .collect(ImmutableListMultimap.toImmutableListMultimap(QueryField::getField, Function.identity())); + return new CriteriaWrapper(multimap, this.andOperator); + } + } + + /** + * 从bean对象来构造一个CriteriaWrapper + * 需要这个bean对象中查询的属性通过{@link CriteriaField}来声明 + * @param bean 查询条件的bean对象 + * @return 查询对象 + */ + public static CriteriaWrapper fromBean(Object bean) { + return fromBean(bean, true); + } + + + /** + * 从bean对象来构造一个CriteriaWrapper + * 需要这个bean对象中查询的属性通过{@link CriteriaField}来声明 + * @param bean 查询条件的bean对象 + * @param andOperator true查询条件是and操作, false查询条件or操作. + * @return 查询对象 + */ + public static CriteriaWrapper fromBean(Object bean, boolean andOperator) { + return fromBean(bean, andOperator, null); + } + + /** + * 从bean对象来构造一个CriteriaWrapper + * 需要这个bean对象中查询的属性通过{@link CriteriaField}来声明 + * @param bean 查询条件的bean对象 + * @param andOperator true查询条件是and操作, false查询条件or操作. + * @return 查询对象 + */ + public static CriteriaWrapper fromBean(Object bean, boolean andOperator, + Function> converter) { + return fromBean(bean, andOperator, converter, ImmutableMap.of()); + } + + /** + * 从bean对象来构造一个CriteriaWrapper + * 需要这个bean对象中查询的属性通过{@link CriteriaField}来声明 + * + * @param bean 查询条件的bean对象 + * @param andOperator true查询条件是and操作, false查询条件or操作. + * @return 查询对象 + */ + public static CriteriaWrapper fromBean(Object bean, boolean andOperator, + Function> converter, Map defaultOperators) { + Map annotations = getFieldAnnotations(bean.getClass()); + Map beanMap = (bean instanceof Map) ? (Map) bean : BeanMap.create(bean); + List queryFields = beanMap.entrySet().stream() + .map(entry -> QueryField.build(entry.getKey(), entry.getValue(), annotations, + (defaultOperators != null ? defaultOperators.get(entry.getKey()): null))) + .filter(Objects::nonNull) + .flatMap(queryField -> converter == null ? Stream.of(queryField) : converter.apply(queryField).stream()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + return CriteriaWrapper.builder().queryFields(queryFields).andOperator(andOperator).build(); + } + + + private static Map getFieldAnnotations(Class beanClazz) { + return criteriaFieldAnnotations.computeIfAbsent(beanClazz, clazz -> + FieldUtils.getFieldsListWithAnnotation(clazz, CriteriaField.class).stream() + .collect(Collectors.toMap(Field::getName, field -> field.getAnnotation(CriteriaField.class)))); + } + + @Builder(toBuilder = true) + @Data + @AllArgsConstructor + public static class QueryField { + /** + * json字段查询条件构造模板,example:content->'$.filed1'__LIKE + */ + private final static String JSON_QUERY_FILED_TEMPLATE = "%s->'$.%s'%s"; + + /** + * json查询条件作为filed时的特殊操作符 + */ + private final static String JSON_QUERY_FILED_FLAG = "->"; + /** + * 逻辑控制字段名称前缀 + */ + private final static String LOGICAL_CONTROL_FIELD_PREFIX = "$$"; + /** + * java bean的property name + */ + private String field; + /** + * 查询的operator + */ + private Operator operator; + /** + * 查询的值 + */ + private Object value; + /** + * 字段前缀名称 + */ + private String prefix; + + public boolean isLogicalControlField() { + return operator == Operator.OR; + } + + /** + * 是否逻辑控制字段 + * @param fieldName + * @return 比如 or,and + */ + public static boolean isLogicalControlField(String fieldName) { + return fieldName.startsWith(LOGICAL_CONTROL_FIELD_PREFIX); + } + + /** + * 是否是json查询字段,fieldName包含"->"符号表示是json查询 + * @param fieldName + * @return + */ + public static boolean isJsonQueryField(String fieldName) { + return fieldName.contains(JSON_QUERY_FILED_FLAG); + } + + public String getFieldWithPrefix() { + if (Strings.isNullOrEmpty(prefix)) { + return field; + } + return prefix + '.' + field; + } + + public String getColumnWithPrefix(String column) { + if (Strings.isNullOrEmpty(prefix)) { + return column; + } + return prefix + '.' + column; + } + + private static QueryField build(String field, Object value, Map annotations, Operator defaultOperator) { + CriteriaField fieldAnnotation = annotations.get(field); + if (fieldAnnotation == null) { + // 如果没有Annotation,默认会过滤Null + return QueryField.buildWithNullFilter(field, (defaultOperator != null ? defaultOperator : getDefaultOperator(field, value)), value); + } + + if (fieldAnnotation.ignore()) { + return null; + } + + if (fieldAnnotation.filterEmpty() + && (value instanceof Collection) + && CollectionUtils.isEmpty((Collection)value)) { + return null; + } + + if (fieldAnnotation.filterEmpty() + && (value instanceof Map) + && CollectionUtils.isEmpty((Map) value)) { + return null; + } + + if (fieldAnnotation.filterNull() && Objects.isNull(value)) { + return null; + } + + String fieldName = Strings.isNullOrEmpty(fieldAnnotation.field()) ? field : fieldAnnotation.field(); + // XXX 注解中获取的operator如果为EQ,无法判断是用户手动设置为EQ还是使用的默认operator.EQ,这里暂时保持原状 + return build(fieldName, fieldAnnotation.operator(), value, fieldAnnotation.prefix()); + } + + private static QueryField build(String key, Object value) { + return QueryField.build(key, getDefaultOperator(key, value), value, StringUtils.EMPTY); + } + + private static QueryField build(String key, Operator defaultOperator, Object value, String prefix) { + String field = StringUtils.substringBefore(key, DELIMITER); + String expression = StringUtils.substringAfter(key, DELIMITER); + Operator operator = Strings.isNullOrEmpty(expression) ? defaultOperator : Operator.valueOf(expression); + if (operator == Operator.OR) { + List nestedFields = ((Map)value).entrySet() + .stream().map(e -> build(e.getKey(), Operator.EQ, e.getValue(), prefix)).collect(Collectors.toList()); + // FIXME: 因为为了减少修改,这里修改了 field 的名称,通过名称将状态传递下去。 + return new QueryField(LOGICAL_CONTROL_FIELD_PREFIX + field, operator, nestedFields, prefix); + } + if (operator == Operator.JSON || operator == Operator.JSON_OR) { + List nestedFields = ((Map)value).entrySet() + .stream().map(e -> build(buildJsonQueryFieldKey(field, e.getKey()), Operator.EQ, e.getValue(), prefix)).collect(Collectors.toList()); + // FIXME: 因为为了减少修改,这里修改了 field 的名称,通过名称将状态传递下去。 + return new QueryField(LOGICAL_CONTROL_FIELD_PREFIX + field, operator, nestedFields, prefix); + + } + return new QueryField(field, operator, value, prefix); + } + + + private static QueryField buildWithNullFilter(String key, Operator defaultOperator, Object value) { + QueryField queryField = build(key, defaultOperator, value, StringUtils.EMPTY); + if (queryField.getValue() == null && !queryField.getOperator().allowNullValue()) { + return null; + } + return queryField; + } + + private static Operator getDefaultOperator(String field, Object value) { + if (!(value instanceof Collection)) { + return Operator.EQ; + } + + // 当value instanceof Collection,检查值是否为空 + if (((Collection)value).isEmpty()) { + log.error("value is collection and is empty, field={}", field); + } + return Operator.IN; + } + + /** + * 构造json查询条件key + * 例如fieldName=content, jsonFieldName=text__LIKE,返回:content->'$.text'__LIKE + * + * @param fieldName 数据库字段名 + * @param jsonFieldName json内的字段名 + * @return + */ + private static String buildJsonQueryFieldKey(String fieldName, String jsonFieldName) { + String jsonField = StringUtils.substringBefore(jsonFieldName, DELIMITER); + String jsonExpression = StringUtils.substringAfter(jsonFieldName, DELIMITER); + if (Strings.isNullOrEmpty(jsonExpression)) { + return String.format(JSON_QUERY_FILED_TEMPLATE, fieldName, jsonField, ""); + } + return String.format(JSON_QUERY_FILED_TEMPLATE, fieldName, jsonField, DELIMITER + jsonExpression); + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/wrapper/Operator.java b/src/main/java/cn/axzo/pokonyan/dao/wrapper/Operator.java new file mode 100644 index 0000000..bc98753 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/wrapper/Operator.java @@ -0,0 +1,47 @@ +package cn.axzo.pokonyan.dao.wrapper; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum Operator { + IN, + LIKE, + EQ, + NE, + GT, + GE, + LT, + LE, + IS_NULL, + IS_NOT_NULL, + BETWEEN, + /** + * start with + */ + SW, + /** + * end with + */ + EW, + ORDER, + OR, + /** + * 添加 + * FULL_SEARCH + */ + FS, + /** + * JSON解析,现只支持mysql json类型字段 + * 查询条件示例:{"name":"张三","time__GT":"1595235995326","carNo__LIKE":"川A"} + */ + JSON, + JSON_OR, + @Deprecated + CUSTOM; + + public boolean allowNullValue() { + return this == IS_NULL || this == IS_NOT_NULL; + } +} \ No newline at end of file diff --git a/src/main/java/cn/axzo/pokonyan/dao/wrapper/OperatorProcessor.java b/src/main/java/cn/axzo/pokonyan/dao/wrapper/OperatorProcessor.java new file mode 100644 index 0000000..d0b1796 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/wrapper/OperatorProcessor.java @@ -0,0 +1,19 @@ +package cn.axzo.pokonyan.dao.wrapper; + +import com.google.common.collect.ImmutableListMultimap; + +/** + * 封装底层的查询实现 + * + */ +public interface OperatorProcessor { + + /** + * 组装所有查询条件 + * + * @param queryColumnMap + * @param andOperator + * @return + */ + T assembleAllQueryWrapper(ImmutableListMultimap queryColumnMap, boolean andOperator); +} diff --git a/src/main/java/cn/axzo/pokonyan/dao/wrapper/SimpleWrapperConverter.java b/src/main/java/cn/axzo/pokonyan/dao/wrapper/SimpleWrapperConverter.java new file mode 100644 index 0000000..61bf200 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/wrapper/SimpleWrapperConverter.java @@ -0,0 +1,158 @@ +package cn.axzo.pokonyan.dao.wrapper; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableTable; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * 将CriteriaWrapper转换为mysql的QueryWrapper + */ +@Slf4j +@Builder +@AllArgsConstructor +public class SimpleWrapperConverter { + + private OperatorProcessor operatorProcessor; + /** + * key = property value = column + */ + private Map fieldColumnMap; + /** + * key = property value = property class + */ + private Map> fieldTypeMap; + + /** + * value转换. key1 = value原来的type, key2 = targetType, value = 转换func + */ + private static final Table, Class, Function> VALUE_CONVERTERS = + ImmutableTable., Class, Function>builder() + .put(Long.class, LocalDateTime.class, o -> Instant.ofEpochMilli((Long) o).atZone(ZoneId.systemDefault()).toLocalDateTime()) + .put(Long.class, LocalDate.class, o -> Instant.ofEpochMilli((Long) o).atZone(ZoneId.systemDefault()).toLocalDate()) + .put(Long.class, Date.class, o -> new Date((Long) o)) + .build(); + + + public T toWrapper(CriteriaWrapper criteriaWrapper) { + ListMultimap multimap = criteriaWrapper.getQueryFieldMap() + .asMap().entrySet().stream() + .collect(Multimaps.flatteningToMultimap( + query -> getColumnNotNull(query.getKey()), + query -> convertValue(query.getValue()).stream(), + ArrayListMultimap::create)); + return operatorProcessor.assembleAllQueryWrapper(ImmutableListMultimap.copyOf(multimap), criteriaWrapper.getAndOperator()); + } + + public T toWrapper(CriteriaWrapper criteriaWrapper, SimpleWrapperConverter... anotherConverters) { + ListMultimap multimap = criteriaWrapper.getQueryFieldMap() + .asMap().entrySet().stream() + .collect(Multimaps.flatteningToMultimap( + query -> getColumnByConverters(query.getKey(), anotherConverters), + query -> convertValue(query.getValue()).stream(), + ArrayListMultimap::create)); + return operatorProcessor.assembleAllQueryWrapper(ImmutableListMultimap.copyOf(multimap), criteriaWrapper.getAndOperator()); + } + + public Optional getColumn(String fieldName) { + if (CriteriaWrapper.QueryField.isLogicalControlField(fieldName) || + CriteriaWrapper.QueryField.isJsonQueryField(fieldName)) { + // 逻辑控制字段是虚拟字段,不需要查找;json查询字段是特殊语法,构造的字段不需要查找。 + return Optional.of(fieldName); + } + return Optional.ofNullable(fieldColumnMap.get(fieldName)); + } + + public String getColumnNotNull(String fieldName) { + return getColumn(fieldName).orElseThrow(() -> new RuntimeException("参数验证失败")); + } + + public String getColumnByConverters(String fieldName, SimpleWrapperConverter... anotherConverters) { + // 优先查找当前class的字段 + Optional column = getColumn(fieldName); + if (column.isPresent()) { + return column.get(); + } + for (SimpleWrapperConverter anotherConverter : anotherConverters) { + column = anotherConverter.getColumn(fieldName); + if (column.isPresent()) { + return column.get(); + } + } + throw new RuntimeException("参数验证失败"); + } + + /** + * 转换QueryField的value + * + * @param queryFields + * @return + */ + private List convertValue(Collection queryFields) { + return queryFields.stream() + .map(queryField -> queryField.toBuilder().value(getConvertedValue(queryField)).build()) + .collect(Collectors.toList()); + } + + private Object getConvertedValue(CriteriaWrapper.QueryField queryField) { + if (queryField.isLogicalControlField()) { + List subfields = (List) queryField.getValue(); + // resolve 嵌套字段的字段和值 + for (final CriteriaWrapper.QueryField subfield : subfields) { + subfield.setField(getColumnNotNull(subfield.getField())); + } + } + + if (queryField.getValue() == null) { + return null; + } + + //找到属性对应的类型与当前value的类型. 根据类型找到convert. 有则处理. 没有则直接返回 + Class fieldType = fieldTypeMap.get(queryField.getField()); + Class valueType = queryField.getValue().getClass(); + + //如果sourceType + if (Collection.class.isAssignableFrom(valueType)) { + Collection collection = (Collection) queryField.getValue(); + return collection.stream() + .filter(Objects::nonNull) + .map(e -> { + Function valueConverter = VALUE_CONVERTERS.get(e.getClass(), fieldType); + return valueConverter == null ? e : valueConverter.apply(e); + }).collect(getCollectionCollector(valueType)); + } + + Function valueConverter = VALUE_CONVERTERS.get(valueType, fieldType); + return valueConverter == null ? queryField.getValue() : valueConverter.apply(queryField.getValue()); + } + + private Collector getCollectionCollector(Class clazz) { + if (List.class.isAssignableFrom(clazz)) { + return Collectors.toList(); + } + if (Set.class.isAssignableFrom(clazz)) { + return Collectors.toSet(); + } + throw new UnsupportedOperationException(String.format("unsupported collection type of %s", clazz.getSimpleName())); + } +} \ No newline at end of file diff --git a/src/main/java/cn/axzo/pokonyan/dao/wrapper/TriConsumer.java b/src/main/java/cn/axzo/pokonyan/dao/wrapper/TriConsumer.java new file mode 100644 index 0000000..8e72156 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/dao/wrapper/TriConsumer.java @@ -0,0 +1,17 @@ +package cn.axzo.pokonyan.dao.wrapper; + +/** + * 仅包内可访问 + * + */ +@FunctionalInterface +public interface TriConsumer { + /** + * 三入参的consumer + * + * @param k + * @param v + * @param s + */ + void accept(K k, V v, S s); +} From 6d88c0e1c378753fd79e7129c98e28cfd8af8dea Mon Sep 17 00:00:00 2001 From: lilong Date: Fri, 15 Mar 2024 17:16:01 +0800 Subject: [PATCH 03/17] =?UTF-8?q?feat:=E5=A2=9E=E5=8A=A0redissonClient?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/pokonyan/client/impl/RateLimiterClientImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/RateLimiterClientImpl.java b/src/main/java/cn/axzo/pokonyan/client/impl/RateLimiterClientImpl.java index f7e0196..96dc8f0 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/RateLimiterClientImpl.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/RateLimiterClientImpl.java @@ -35,7 +35,7 @@ public class RateLimiterClientImpl implements RateLimiterClient { private RedissonClient redissonClient; - public Builder redisTemplate(RedissonClient redissonClient) { + public Builder redissonClient(RedissonClient redissonClient) { this.redissonClient = redissonClient; return this; } From 99bae07f9372e0b7153fae1dbb5de35291ef4517 Mon Sep 17 00:00:00 2001 From: lilong Date: Fri, 15 Mar 2024 17:58:51 +0800 Subject: [PATCH 04/17] =?UTF-8?q?feat:=E5=8E=BB=E6=8E=89availablePermits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java b/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java index 5c1c675..689d9d8 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java @@ -83,11 +83,8 @@ public class RedisRateLimiterImpl implements RateLimiter { class SlidingWindowRateLimiter implements RateLimiterWorker { public boolean tryAcquire(Object value) { String key = buildRedisKey(value); - long now = System.currentTimeMillis(); RRateLimiter rateLimiter = redissonClient.getRateLimiter(key); - - rateLimiter.availablePermits(); if (!rateLimiter.isExists()) { rateLimiter.trySetRate(RateType.OVERALL, limitRule.getPermits(), limitRule.getSeconds(), RateIntervalUnit.SECONDS); } From ea47b8f094087fdba1e452e63bdbf1a55be83d3b Mon Sep 17 00:00:00 2001 From: lilong Date: Fri, 15 Mar 2024 18:22:03 +0800 Subject: [PATCH 05/17] =?UTF-8?q?feat:=E4=BF=AE=E6=94=B9ratelimiter?= =?UTF-8?q?=E7=9A=84=E9=87=8D=E5=A4=8D=E5=88=9B=E5=BB=BA=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/pokonyan/client/RateLimiter.java | 11 +------ .../client/impl/RedisRateLimiterImpl.java | 29 +++++++++---------- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java b/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java index e440feb..f93af78 100644 --- a/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java +++ b/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java @@ -1,31 +1,22 @@ package cn.axzo.pokonyan.client; -import com.google.common.base.Splitter; -import com.google.common.base.Strings; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.Getter; import lombok.ToString; -import org.springframework.util.CollectionUtils; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; public interface RateLimiter { /** * 尝试获得锁, 获取失败则返回Optional.empty() * 如果获取锁成功. 则返回Optional. 同时计数器增加 * Permit支持取消 - * - * @param value 业务标识 * @return */ - boolean tryAcquire(Object value); + boolean tryAcquire(); /** * 获取窗口类型 diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java b/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java index 689d9d8..01d537f 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java @@ -24,12 +24,13 @@ public class RedisRateLimiterImpl implements RateLimiter { private LimitRule limitRule; private WindowType windowType; + private RRateLimiter rateLimiter; + @Builder RedisRateLimiterImpl(RedissonClient redissonClient, WindowType windowType, String limiterKey, - LimitRule limitRule, - Integer maxWindowDurationHour) { + LimitRule limitRule) { Objects.requireNonNull(redissonClient); Objects.requireNonNull(windowType); Objects.requireNonNull(limitRule); @@ -43,11 +44,16 @@ public class RedisRateLimiterImpl implements RateLimiter { this.limitRule = limitRule; this.limiterKey = limiterKey; this.rateLimiterWorker = buildWorker(windowType); + + String key = buildRedisKey(); + RRateLimiter rateLimiter = redissonClient.getRateLimiter(key); + rateLimiter.trySetRate(RateType.OVERALL, limitRule.getPermits(), limitRule.getSeconds(), RateIntervalUnit.SECONDS); + this.rateLimiter = rateLimiter; } @Override - public boolean tryAcquire(Object value) { - return rateLimiterWorker.tryAcquire(value); + public boolean tryAcquire() { + return rateLimiterWorker.tryAcquire(); } @Override @@ -62,10 +68,9 @@ public class RedisRateLimiterImpl implements RateLimiter { throw new RuntimeException(String.format("unsupported window type, window type = %s", windowType)); } - private String buildRedisKey(Object value) { + private String buildRedisKey() { String hash = Hashing.murmur3_128().newHasher() .putString(limiterKey, Charsets.UTF_8) - .putString(String.valueOf(value), Charsets.UTF_8) .hash() .toString(); @@ -81,14 +86,7 @@ public class RedisRateLimiterImpl implements RateLimiter { * */ class SlidingWindowRateLimiter implements RateLimiterWorker { - public boolean tryAcquire(Object value) { - String key = buildRedisKey(value); - - RRateLimiter rateLimiter = redissonClient.getRateLimiter(key); - if (!rateLimiter.isExists()) { - rateLimiter.trySetRate(RateType.OVERALL, limitRule.getPermits(), limitRule.getSeconds(), RateIntervalUnit.SECONDS); - } - + public boolean tryAcquire() { return rateLimiter.tryAcquire(1); } } @@ -97,9 +95,8 @@ public class RedisRateLimiterImpl implements RateLimiter { /** * 尝试获取令牌 * - * @param value * @return 如果获取成功则返回true, 失败则为false */ - boolean tryAcquire(Object value); + boolean tryAcquire(); } } From 341c31cb3aa57709e22aa744da39381b73d4162e Mon Sep 17 00:00:00 2001 From: lilong Date: Tue, 19 Mar 2024 10:32:58 +0800 Subject: [PATCH 06/17] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E5=BC=82?= =?UTF-8?q?=E5=B8=B8check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/pokonyan/exception/Aassert.java | 132 +++++ .../pokonyan/exception/BusinessException.java | 39 ++ .../axzo/pokonyan/exception/ResultCode.java | 35 ++ .../pokonyan/exception/VarParamFormatter.java | 486 ++++++++++++++++++ 4 files changed, 692 insertions(+) create mode 100644 src/main/java/cn/axzo/pokonyan/exception/Aassert.java create mode 100644 src/main/java/cn/axzo/pokonyan/exception/BusinessException.java create mode 100644 src/main/java/cn/axzo/pokonyan/exception/ResultCode.java create mode 100644 src/main/java/cn/axzo/pokonyan/exception/VarParamFormatter.java diff --git a/src/main/java/cn/axzo/pokonyan/exception/Aassert.java b/src/main/java/cn/axzo/pokonyan/exception/Aassert.java new file mode 100644 index 0000000..f315f44 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/exception/Aassert.java @@ -0,0 +1,132 @@ +package cn.axzo.pokonyan.exception; + +import com.google.common.base.Strings; + +import java.util.Collection; +import java.util.Objects; +import java.util.function.Supplier; + +public abstract class Aassert { + + public Aassert() { + } + + public static void throwError(ResultCode errorCode) { + throw new BusinessException(errorCode); + } + + public static void throwError(ResultCode errorCode, String overrideMessage) { + throw new BusinessException(errorCode.getErrorCode(), overrideMessage); + } + + public static void isTrue(boolean expression, ResultCode errorCode) { + if (!expression) { + throw new BusinessException(errorCode); + } + } + + public static void isTrue(boolean expression, ResultCode errorCode, String overrideMessage) { + if (!expression) { + throw new BusinessException(errorCode.getErrorCode(), overrideMessage); + } + } + + public static void isFalse(boolean expression, ResultCode errorCode) { + isTrue(!expression, errorCode); + } + + public static void isFalse(boolean expression, ResultCode errorCode, String overrideMessage) { + isTrue(!expression, errorCode, overrideMessage); + } + + public static void isNull(Object object, ResultCode errorCode) { + isTrue(object == null, errorCode); + } + + public static void isNull(Object object, ResultCode errorCode, String overrideMessage) { + isTrue(object == null, errorCode, overrideMessage); + } + + public static void notNull(Object object, ResultCode errorCode) { + isTrue(object != null, errorCode); + } + + public static void notNull(Object object, ResultCode errorCode, String overrideMessage) { + isTrue(object != null, errorCode, overrideMessage); + } + + public static void check(boolean expect, String code, String msg) { + if (!expect) { + throw new BusinessException(code, msg); + } + } + + public static void check(boolean expect, ResultCode resultCode) { + if (!expect) { + throw resultCode.toException(); + } + } + + public static void check(boolean expect, ResultCode resultCode, String msg, Object... objects) { + if (!expect) { + throw resultCode.toException(msg, objects); + } + } + + public static void check(boolean expect, Supplier supplier) throws Throwable { + if (!expect) { + throw (Throwable)supplier.get(); + } + } + + public static void checkEquals(Object source, Object target, ResultCode resultCode) { + if (!Objects.equals(source, target)) { + throw resultCode.toException(); + } + } + + public static void checkEquals(Object source, Object target, ResultCode resultCode, String msg, Object... objects) { + if (!Objects.equals(source, target)) { + throw resultCode.toException(msg, objects); + } + } + + + public static void checkNonNull(Object target, ResultCode resultCode) { + if (Objects.isNull(target)) { + throw resultCode.toException(); + } + } + + + public static void checkNonNull(Object target, ResultCode resultCode, String msg, Object... objects) { + if (Objects.isNull(target)) { + throw resultCode.toException(msg, objects); + } + } + + public static void checkNotEmpty(Collection coll, ResultCode resultCode) { + if (coll == null || coll.isEmpty()) { + throw resultCode.toException(); + } + } + + public static void checkNotEmpty(Collection coll, ResultCode resultCode, String msg, Object... objects) { + if (coll == null || coll.isEmpty()) { + throw resultCode.toException(msg, objects); + } + } + + public static void checkStringNotEmpty(String str, ResultCode resultCode) { + if (Strings.isNullOrEmpty(str)) { + throw resultCode.toException(); + } + } + + public static void checkStringNotEmpty(String str, ResultCode resultCode, String msg, Object... objects) { + if (Strings.isNullOrEmpty(str)) { + throw resultCode.toException(msg, objects); + } + } + +} diff --git a/src/main/java/cn/axzo/pokonyan/exception/BusinessException.java b/src/main/java/cn/axzo/pokonyan/exception/BusinessException.java new file mode 100644 index 0000000..6aebee4 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/exception/BusinessException.java @@ -0,0 +1,39 @@ +package cn.axzo.pokonyan.exception; + +public class BusinessException extends RuntimeException { + private static final long serialVersionUID = -4949212560571865637L; + private final String errorCode; + private final String errorMsg; + + public BusinessException(ResultCode resultCode) { + super(String.format("BusinessException{errorCode:%s, errorMsg:%s}", resultCode.getErrorCode(), resultCode.getErrorMessage())); + this.errorCode = resultCode.getErrorCode(); + this.errorMsg = resultCode.getErrorMessage(); + } + + public BusinessException(ResultCode resultCode, Throwable cause) { + super(String.format("BusinessException{errorCode:%s, errorMsg:%s}", resultCode.getErrorCode(), resultCode.getErrorMessage()), cause); + this.errorCode = resultCode.getErrorCode(); + this.errorMsg = resultCode.getErrorMessage(); + } + + public BusinessException(String errorCode, String errorMsg) { + super(String.format("BusinessException{errorCode:%s, errorMsg:%s}", errorCode, errorMsg)); + this.errorCode = errorCode; + this.errorMsg = errorMsg; + } + + public BusinessException(String errorCode, String errorMsg, Throwable cause) { + super(String.format("BusinessException{errorCode:%s, errorMsg:%s}", errorCode, errorMsg), cause); + this.errorCode = errorCode; + this.errorMsg = errorMsg; + } + + public String getErrorCode() { + return this.errorCode; + } + + public String getErrorMsg() { + return this.errorMsg; + } +} \ No newline at end of file diff --git a/src/main/java/cn/axzo/pokonyan/exception/ResultCode.java b/src/main/java/cn/axzo/pokonyan/exception/ResultCode.java new file mode 100644 index 0000000..731ce4f --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/exception/ResultCode.java @@ -0,0 +1,35 @@ +package cn.axzo.pokonyan.exception; + +public interface ResultCode { + Integer DEFAULT_HTTP_ERROR_CODE = 400; + + default Integer getHttpCode() { + return DEFAULT_HTTP_ERROR_CODE; + } + + String getErrorCode(); + + String getErrorMessage(); + + default BusinessException toException() { + return new BusinessException(this); + } + + default BusinessException toException(String customMsg) { + return new BusinessException(this.getErrorCode(), customMsg); + } + + default BusinessException toException(String customMsg, Object... objects) { + if (objects != null && objects.length != 0) { + String msg = VarParamFormatter.format(customMsg, objects); + if (objects[objects.length - 1] instanceof Throwable) { + Throwable throwable = (Throwable)objects[objects.length - 1]; + msg = String.format("%s (%s)", msg, throwable.getClass().getSimpleName()); + } + + return this.toException(msg); + } else { + return this.toException(customMsg); + } + } +} \ No newline at end of file diff --git a/src/main/java/cn/axzo/pokonyan/exception/VarParamFormatter.java b/src/main/java/cn/axzo/pokonyan/exception/VarParamFormatter.java new file mode 100644 index 0000000..9b2113e --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/exception/VarParamFormatter.java @@ -0,0 +1,486 @@ +package cn.axzo.pokonyan.exception; + +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +public class VarParamFormatter { + static final String RECURSION_PREFIX = "[..."; + static final String RECURSION_SUFFIX = "...]"; + static final String ERROR_PREFIX = "[!!!"; + static final String ERROR_SEPARATOR = "=>"; + static final String ERROR_MSG_SEPARATOR = ":"; + static final String ERROR_SUFFIX = "!!!]"; + private static final char DELIM_START = '{'; + private static final char DELIM_STOP = '}'; + private static final char ESCAPE_CHAR = '\\'; + private static ThreadLocal threadLocalSimpleDateFormat = new ThreadLocal(); + + private VarParamFormatter() { + } + + static int countArgumentPlaceholders(final String messagePattern) { + if (messagePattern == null) { + return 0; + } else { + int length = messagePattern.length(); + int result = 0; + boolean isEscaped = false; + + for(int i = 0; i < length - 1; ++i) { + char curChar = messagePattern.charAt(i); + if (curChar == '\\') { + isEscaped = !isEscaped; + } else if (curChar == '{') { + if (!isEscaped && messagePattern.charAt(i + 1) == '}') { + ++result; + ++i; + } + + isEscaped = false; + } else { + isEscaped = false; + } + } + + return result; + } + } + + static int countArgumentPlaceholders2(final String messagePattern, final int[] indices) { + if (messagePattern == null) { + return 0; + } else { + int length = messagePattern.length(); + int result = 0; + boolean isEscaped = false; + + for(int i = 0; i < length - 1; ++i) { + char curChar = messagePattern.charAt(i); + if (curChar == '\\') { + isEscaped = !isEscaped; + indices[0] = -1; + ++result; + } else if (curChar == '{') { + if (!isEscaped && messagePattern.charAt(i + 1) == '}') { + indices[result] = i; + ++result; + ++i; + } + + isEscaped = false; + } else { + isEscaped = false; + } + } + + return result; + } + } + + static int countArgumentPlaceholders3(final char[] messagePattern, final int length, final int[] indices) { + int result = 0; + boolean isEscaped = false; + + for(int i = 0; i < length - 1; ++i) { + char curChar = messagePattern[i]; + if (curChar == '\\') { + isEscaped = !isEscaped; + } else if (curChar == '{') { + if (!isEscaped && messagePattern[i + 1] == '}') { + indices[result] = i; + ++result; + ++i; + } + + isEscaped = false; + } else { + isEscaped = false; + } + } + + return result; + } + + public static String format(final String messagePattern, final Object[] arguments) { + StringBuilder result = new StringBuilder(); + int argCount = arguments == null ? 0 : arguments.length; + formatMessage(result, messagePattern, arguments, argCount); + return result.toString(); + } + + static void formatMessage2(final StringBuilder buffer, final String messagePattern, final Object[] arguments, final int argCount, final int[] indices) { + if (messagePattern != null && arguments != null && argCount != 0) { + int previous = 0; + + for(int i = 0; i < argCount; ++i) { + buffer.append(messagePattern, previous, indices[i]); + previous = indices[i] + 2; + recursiveDeepToString(arguments[i], buffer, (Set)null); + } + + buffer.append(messagePattern, previous, messagePattern.length()); + } else { + buffer.append(messagePattern); + } + } + + static void formatMessage3(final StringBuilder buffer, final char[] messagePattern, final int patternLength, final Object[] arguments, final int argCount, final int[] indices) { + if (messagePattern != null) { + if (arguments != null && argCount != 0) { + int previous = 0; + + for(int i = 0; i < argCount; ++i) { + buffer.append(messagePattern, previous, indices[i]); + previous = indices[i] + 2; + recursiveDeepToString(arguments[i], buffer, (Set)null); + } + + buffer.append(messagePattern, previous, patternLength); + } else { + buffer.append(messagePattern); + } + } + } + + static void formatMessage(final StringBuilder buffer, final String messagePattern, final Object[] arguments, final int argCount) { + if (messagePattern != null && arguments != null && argCount != 0) { + int escapeCounter = 0; + int currentArgument = 0; + int i = 0; + + int len; + for(len = messagePattern.length(); i < len - 1; ++i) { + char curChar = messagePattern.charAt(i); + if (curChar == '\\') { + ++escapeCounter; + } else { + if (isDelimPair(curChar, messagePattern, i)) { + ++i; + writeEscapedEscapeChars(escapeCounter, buffer); + if (isOdd(escapeCounter)) { + writeDelimPair(buffer); + } else { + writeArgOrDelimPair(arguments, argCount, currentArgument, buffer); + ++currentArgument; + } + } else { + handleLiteralChar(buffer, escapeCounter, curChar); + } + + escapeCounter = 0; + } + } + + handleRemainingCharIfAny(messagePattern, len, buffer, escapeCounter, i); + } else { + buffer.append(messagePattern); + } + } + + private static boolean isDelimPair(final char curChar, final String messagePattern, final int curCharIndex) { + return curChar == '{' && messagePattern.charAt(curCharIndex + 1) == '}'; + } + + private static void handleRemainingCharIfAny(final String messagePattern, final int len, final StringBuilder buffer, final int escapeCounter, final int i) { + if (i == len - 1) { + char curChar = messagePattern.charAt(i); + handleLastChar(buffer, escapeCounter, curChar); + } + + } + + private static void handleLastChar(final StringBuilder buffer, final int escapeCounter, final char curChar) { + if (curChar == '\\') { + writeUnescapedEscapeChars(escapeCounter + 1, buffer); + } else { + handleLiteralChar(buffer, escapeCounter, curChar); + } + + } + + private static void handleLiteralChar(final StringBuilder buffer, final int escapeCounter, final char curChar) { + writeUnescapedEscapeChars(escapeCounter, buffer); + buffer.append(curChar); + } + + private static void writeDelimPair(final StringBuilder buffer) { + buffer.append('{'); + buffer.append('}'); + } + + private static boolean isOdd(final int number) { + return (number & 1) == 1; + } + + private static void writeEscapedEscapeChars(final int escapeCounter, final StringBuilder buffer) { + int escapedEscapes = escapeCounter >> 1; + writeUnescapedEscapeChars(escapedEscapes, buffer); + } + + private static void writeUnescapedEscapeChars(int escapeCounter, final StringBuilder buffer) { + while(escapeCounter > 0) { + buffer.append('\\'); + --escapeCounter; + } + + } + + private static void writeArgOrDelimPair(final Object[] arguments, final int argCount, final int currentArgument, final StringBuilder buffer) { + if (currentArgument < argCount) { + recursiveDeepToString(arguments[currentArgument], buffer, (Set)null); + } else { + writeDelimPair(buffer); + } + + } + + static String deepToString(final Object o) { + if (o == null) { + return null; + } else if (o instanceof String) { + return (String)o; + } else { + StringBuilder str = new StringBuilder(); + Set dejaVu = new HashSet(); + recursiveDeepToString(o, str, dejaVu); + return str.toString(); + } + } + + private static void recursiveDeepToString(final Object o, final StringBuilder str, final Set dejaVu) { + if (!appendSpecialTypes(o, str)) { + if (isMaybeRecursive(o)) { + appendPotentiallyRecursiveValue(o, str, dejaVu); + } else { + tryObjectToString(o, str); + } + + } + } + + private static boolean appendSpecialTypes(final Object o, final StringBuilder str) { + if (o != null && !(o instanceof String)) { + if (o instanceof CharSequence) { + str.append((CharSequence)o); + return true; + } else if (o instanceof Integer) { + str.append((Integer)o); + return true; + } else if (o instanceof Long) { + str.append((Long)o); + return true; + } else if (o instanceof Double) { + str.append((Double)o); + return true; + } else if (o instanceof Boolean) { + str.append((Boolean)o); + return true; + } else if (o instanceof Character) { + str.append((Character)o); + return true; + } else if (o instanceof Short) { + str.append((Short)o); + return true; + } else if (o instanceof Float) { + str.append((Float)o); + return true; + } else { + return appendDate(o, str); + } + } else { + str.append((String)o); + return true; + } + } + + private static boolean appendDate(final Object o, final StringBuilder str) { + if (!(o instanceof Date)) { + return false; + } else { + Date date = (Date)o; + SimpleDateFormat format = getSimpleDateFormat(); + str.append(format.format(date)); + return true; + } + } + + private static SimpleDateFormat getSimpleDateFormat() { + SimpleDateFormat result = (SimpleDateFormat)threadLocalSimpleDateFormat.get(); + if (result == null) { + result = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + threadLocalSimpleDateFormat.set(result); + } + + return result; + } + + private static boolean isMaybeRecursive(final Object o) { + return o.getClass().isArray() || o instanceof Map || o instanceof Collection; + } + + private static void appendPotentiallyRecursiveValue(final Object o, final StringBuilder str, final Set dejaVu) { + Class oClass = o.getClass(); + if (oClass.isArray()) { + appendArray(o, str, dejaVu, oClass); + } else if (o instanceof Map) { + appendMap(o, str, dejaVu); + } else if (o instanceof Collection) { + appendCollection(o, str, dejaVu); + } + + } + + private static void appendArray(final Object o, final StringBuilder str, Set dejaVu, final Class oClass) { + if (oClass == byte[].class) { + str.append(Arrays.toString((byte[])o)); + } else if (oClass == short[].class) { + str.append(Arrays.toString((short[])o)); + } else if (oClass == int[].class) { + str.append(Arrays.toString((int[])o)); + } else if (oClass == long[].class) { + str.append(Arrays.toString((long[])o)); + } else if (oClass == float[].class) { + str.append(Arrays.toString((float[])o)); + } else if (oClass == double[].class) { + str.append(Arrays.toString((double[])o)); + } else if (oClass == boolean[].class) { + str.append(Arrays.toString((boolean[])o)); + } else if (oClass == char[].class) { + str.append(Arrays.toString((char[])o)); + } else { + if (dejaVu == null) { + dejaVu = new HashSet(); + } + + String id = identityToString(o); + if (((Set)dejaVu).contains(id)) { + str.append("[...").append(id).append("...]"); + } else { + ((Set)dejaVu).add(id); + Object[] oArray = (Object[])o; + str.append('['); + boolean first = true; + Object[] var7 = oArray; + int var8 = oArray.length; + + for(int var9 = 0; var9 < var8; ++var9) { + Object current = var7[var9]; + if (first) { + first = false; + } else { + str.append(", "); + } + + recursiveDeepToString(current, str, new HashSet((Collection)dejaVu)); + } + + str.append(']'); + } + } + + } + + private static void appendMap(final Object o, final StringBuilder str, Set dejaVu) { + if (dejaVu == null) { + dejaVu = new HashSet(); + } + + String id = identityToString(o); + if (((Set)dejaVu).contains(id)) { + str.append("[...").append(id).append("...]"); + } else { + ((Set)dejaVu).add(id); + Map oMap = (Map)o; + str.append('{'); + boolean isFirst = true; + Iterator var6 = oMap.entrySet().iterator(); + + while(var6.hasNext()) { + Object o1 = var6.next(); + Map.Entry current = (Map.Entry)o1; + if (isFirst) { + isFirst = false; + } else { + str.append(", "); + } + + Object key = current.getKey(); + Object value = current.getValue(); + recursiveDeepToString(key, str, new HashSet((Collection)dejaVu)); + str.append('='); + recursiveDeepToString(value, str, new HashSet((Collection)dejaVu)); + } + + str.append('}'); + } + + } + + private static void appendCollection(final Object o, final StringBuilder str, Set dejaVu) { + if (dejaVu == null) { + dejaVu = new HashSet(); + } + + String id = identityToString(o); + if (((Set)dejaVu).contains(id)) { + str.append("[...").append(id).append("...]"); + } else { + ((Set)dejaVu).add(id); + Collection oCol = (Collection)o; + str.append('['); + boolean isFirst = true; + + Object anOCol; + for(Iterator var6 = oCol.iterator(); var6.hasNext(); recursiveDeepToString(anOCol, str, new HashSet((Collection)dejaVu))) { + anOCol = var6.next(); + if (isFirst) { + isFirst = false; + } else { + str.append(", "); + } + } + + str.append(']'); + } + + } + + private static void tryObjectToString(final Object o, final StringBuilder str) { + try { + str.append(o.toString()); + } catch (Throwable var3) { + handleErrorInObjectToString(o, str, var3); + } + + } + + private static void handleErrorInObjectToString(final Object o, final StringBuilder str, final Throwable t) { + str.append("[!!!"); + str.append(identityToString(o)); + str.append("=>"); + String msg = t.getMessage(); + String className = t.getClass().getName(); + str.append(className); + if (!className.equals(msg)) { + str.append(":"); + str.append(msg); + } + + str.append("!!!]"); + } + + static String identityToString(final Object obj) { + if (obj == null) { + return null; + } else { + String var10000 = obj.getClass().getName(); + return var10000 + "@" + Integer.toHexString(System.identityHashCode(obj)); + } + } +} From 851ec5bf81b71144757ae74d1aa0f3a6866e6a95 Mon Sep 17 00:00:00 2001 From: lilong Date: Tue, 19 Mar 2024 18:09:18 +0800 Subject: [PATCH 07/17] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0dataSheet?= =?UTF-8?q?=E6=8F=90=E4=BE=9B=E9=80=9A=E7=94=A8=E7=9A=84excel=20import?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 14 + .../axzo/pokonyan/client/DataSheetClient.java | 608 ++++++++++++++++++ .../client/impl/DataSheetClientImpl.java | 54 ++ .../client/impl/DataSheetImporter.java | 536 +++++++++++++++ .../pokonyan/exception/BizResultCode.java | 16 + .../java/cn/axzo/pokonyan/util/Regex.java | 40 ++ 6 files changed, 1268 insertions(+) create mode 100644 src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java create mode 100644 src/main/java/cn/axzo/pokonyan/client/impl/DataSheetClientImpl.java create mode 100644 src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java create mode 100644 src/main/java/cn/axzo/pokonyan/exception/BizResultCode.java create mode 100644 src/main/java/cn/axzo/pokonyan/util/Regex.java diff --git a/pom.xml b/pom.xml index de9f7fe..6d4a7b2 100644 --- a/pom.xml +++ b/pom.xml @@ -20,6 +20,8 @@ 2.0.0-SNAPSHOT 2.0.0-SNAPSHOT 2.0.0-SNAPSHOT + 5.2.2 + 3.3.3 @@ -113,6 +115,18 @@ org.redisson redisson-spring-boot-starter + + org.apache.poi + poi-ooxml + ${poi.version} + + + + + com.alibaba + easyexcel + ${easyexcel.version} + diff --git a/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java b/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java new file mode 100644 index 0000000..16a2c07 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java @@ -0,0 +1,608 @@ +package cn.axzo.pokonyan.client; + +import cn.axzo.pokonyan.exception.BizResultCode; +import cn.axzo.pokonyan.exception.BusinessException; +import cn.axzo.pokonyan.exception.VarParamFormatter; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSONPath; +import com.alibaba.fastjson.util.TypeUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.experimental.Accessors; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; + +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_DATETIME_FORMAT; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_RANGE_LOWER_TYPE; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_RANGE_UPPER_TYPE; + +/** + * 用于发送sms & email验证码, 或者获取图片验证码. 并提供校验验证码是否正确的能力 + */ +public interface DataSheetClient { + + /** + * 数据导入的builder + * + * @return + */ + ImporterBuilder importBuilder(); + + @Data + @NoArgsConstructor + @AllArgsConstructor + @Accessors(fluent = true) + abstract class ImporterBuilder { + /** + * 导入的场景名称, 必填 + */ + @NonNull + private String scene; + + /** + * 导入格式 + */ + @NonNull + private ImportFormat format; + + /** + * EXCEL格式的sheetName + */ + private String tableName; + + private Meta meta; + + private boolean debugEnabled; + + /** + * 解析结果ImportResp中的lines是否包含解析失败的行 + * true: 解析失败的时候,不会抛异常,会在每行的JSONObject中添加 + * "errors":[{"columnIndex":1,"columnKey":"","columnName":"","rawValue":"","errorCode":"","errorMsg":""}] + * false: 解析失败的时候,抛第一个异常 + */ + private boolean includeLineErrors; + /** + * 允许最大的导入行数 + */ + private Integer allowMaxLineCount; + + public abstract Importer build(); + } + + interface Importer { + /** + * 所有字段读取为String类型 + * + * @param inputStream + * @return + */ + ImportResp readAll(InputStream inputStream); + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class ImportResp { + /** + * 导入的场景名称 + */ + private String scene; + + /** + * 模版对应的code以及版本 + */ + private String templateCode; + private String version; + + /** + * headers + */ + private List headers; + /** + * 每一行的数据 + */ + private List lines; + + /** + * header的行数 + */ + private Integer headerRowCount; + /** + * 导入数据总行数, 不包含header + */ + private Integer rowCount; + /** + * 导入数据列数 + */ + private Integer columnCount; + + /** + * Meta数据 + */ + private Meta meta; + + /** + * 消耗的时间 + */ + private Long elapsedMillis; + } + + @AllArgsConstructor + @Getter + enum ImportFormat { + EXCEL("xlsx"), + // TODO: 支持其他类型 + // CSV("csv") + ; + + private String suffix; + } + + @Data + class ExportField { + private String column; + private String header; + private CellMeta.Type type; + private Boolean mandatory; + + private String lowerType; + private String upperType; + private String dateTimeFormat; + private List options; + + private BiFunction reader; + + @Builder + public ExportField(String column, String header, CellMeta.Type type, Boolean mandatory, + Function resultConverter, BiFunction reader, + List options) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(column), "column不能为空"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(header), "header不能为空"); + Preconditions.checkArgument(!(resultConverter != null && reader != null), "reader & resultConverter不能同时存在"); + + + this.column = column; + this.header = header; + this.type = Optional.ofNullable(type).orElse(CellMeta.Type.STRING); + this.mandatory = Optional.ofNullable(mandatory).orElse(Boolean.FALSE); + if (reader != null) { + this.reader = reader; + } else { + this.reader = Optional.ofNullable(resultConverter) + .map(e -> (BiFunction) (row, integer) -> e.apply(Strings.nullToEmpty(row.getString(column)))) + .orElseGet(() -> DataSheetClient.stringCellReader(column)); + } + this.options = options; + } + + public CellMeta toCellMeta() { + JSONObject params = new JSONObject(); + if (CellMeta.Type.RANGE == type) { + params.put(EXT_KEY_RANGE_LOWER_TYPE, lowerType); + params.put(EXT_KEY_RANGE_UPPER_TYPE, upperType); + } + if (CellMeta.Type.DATE == type || CellMeta.Type.DATETIME == type) { + params.put(EXT_KEY_DATETIME_FORMAT, dateTimeFormat); + } + + return CellMeta.builder() + .key(column).name(header).mandatory(mandatory).type(type) + .params(params).options(options).build(); + } + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class Meta { + public static final String PATTERN_VERSION_OPEN = "$V{"; + public static final String PATTERN_CELL_OPEN = "$C{"; + public static final String PATTERN_IGNORE_OPEN = "$I{"; + public static final String PATTERN_EXTRA_OPEN = "$E{"; + public static final String PATTERN_CLOSE = "}"; + + public static final String RANGE_TYPE_OPEN_OPEN = "1"; + public static final String RANGE_TYPE_OPEN_CLOSED = "2"; + public static final String RANGE_TYPE_CLOSED_OPEN = "3"; + public static final String RANGE_TYPE_CLOSED_CLOSED = "4"; + + public static final String IGNORE_ROW_KEY = "r"; + public static final String IGNORE_COLUMN_KEY = "c"; + public static final String IGNORE_ROW_AND_COLUMN_KEY = "b"; + + private String templateCode; + private String version; + private List cellMetas; + /** + * 忽略掉的行号 + */ + private List ignoreRowIndexes; + /** + * 忽略掉的列号 + */ + private List ignoreColumnIndexes; + + public List getIgnoreRowIndexes() { + return Optional.ofNullable(ignoreRowIndexes).orElseGet(ImmutableList::of); + } + + public List getIgnoreColumnIndexes() { + return Optional.ofNullable(ignoreColumnIndexes).orElseGet(ImmutableList::of); + } + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + class CellMeta { + public static final String EXT_KEY_DATETIME_FORMAT = "dateTimeFormat"; + public static final String EXT_KEY_RANGE_LOWER_TYPE = "lowerType"; + public static final String EXT_KEY_RANGE_UPPER_TYPE = "upperType"; + + public static final String RANGE_BOUND_TYPE_OPEN = "open"; + public static final String RANGE_BOUND_TYPE_CLOSED = "closed"; + public static final Set RANGE_TYPES = ImmutableSet.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED); + + private static final List DEFAULT_DATE_FORMATS = ImmutableList.of( + "yyyy-MM-dd", + "yyyy-MM-dd HH:mm", + "yyyy-MM-dd HH:mm:ss", + "yyyy.MM.dd", + "yyyy.MM.dd HH:mm", + "yyyy.MM.dd HH:mm:ss", + "yyyy年MM月dd日", + "yyyy年MM月dd日 HH:mm", + "yyyy年MM月dd日 HH:mm:ss", + "yyyy年MM月dd日 HH时mm分", + "yyyy年MM月dd日 HH时mm分ss秒", + "yyyy/MM/dd", + "yyyy/MM/dd HH:mm", + "yyyy/MM/dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss'Z'", + "yyyy-MM-dd'T'HH:mm:ssZ", + "yyyy-MM-dd'T'HH:mm:ssz", + "yyyy-MM-dd'T'HH:mm:ss", + "MM/dd/yyyy HH:mm:ss a", + "yyyyMMddHHmmss", + "yyyyMMdd" + ); + + private static final Map BOOLEAN_MAP = ImmutableMap.builder() + .put("是", Boolean.TRUE) + .put("yes", Boolean.TRUE) + .put("y", Boolean.TRUE) + .put("1", Boolean.TRUE) + .put("否", Boolean.FALSE) + .put("no", Boolean.FALSE) + .put("n", Boolean.FALSE) + .put("0", Boolean.FALSE) + .build(); + + /** + * cell的key + */ + private String key; + /** + * cell的header,一般为中文 + */ + private String name; + /** + * 类型 + */ + private Type type; + /** + * 是否必需 + */ + private Boolean mandatory; + /** + * 可选值 + */ + private List options; + /** + * 存放一些和type相关的参数,例如DateTime的pattern格式,Range的开闭区间 + */ + private JSONObject params; + /** + * 存放一些额外的信息 + */ + private JSONObject ext; + + /** + * 导入的时候,将原始的String转换为特定的类型 + */ + private Function importConverter; + + public void validate() { + if (type == Type.RANGE + && (!RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_LOWER_TYPE)) + || !RANGE_TYPES.contains(params.getString(EXT_KEY_RANGE_UPPER_TYPE)))) { + throw ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR.toException(); + } + } + + public Object convertType(String value) { + if (importConverter != null) { + return importConverter.apply(value); + } + + if (Boolean.TRUE.equals(mandatory) && Strings.isNullOrEmpty(value)) { + throw ResultCode.IMPORT_CELL_META_MISSING_MANDATORY_VALUE.toException(); + } + // 非必填的字段如果为空,直接返回 + if (value == null) { + return type == Type.STRING ? StringUtils.EMPTY : null; + } + + switch (type) { + case INT: + return Integer.valueOf(value); + case LONG: + return Long.valueOf(value); + case FLOAT: + return Float.valueOf(value); + case DOUBLE: + return Double.valueOf(value); + case BIG_DECIMAL: + return new BigDecimal(value); + case DATE: + case DATETIME: + return parseDate(value, DEFAULT_DATE_FORMATS); + case RANGE: + // 格式 100-200 + List values = Splitter.on("-").omitEmptyStrings().trimResults().splitToList(value) + .stream() + .map(Integer::valueOf) + .collect(Collectors.toList()); + if (values.size() != 2) { + throw ResultCode.IMPORT_CELL_RANGE_FORMAT_ERROR.toException(); + } + if (values.get(0) >= values.get(1)) { + throw ResultCode.IMPORT_CELL_RANGE_VALUE_ERROR.toException(); + } + return new JSONObject() + .fluentPut("lower", values.get(0)) + .fluentPut("upper", values.get(1)) + .fluentPut(EXT_KEY_RANGE_LOWER_TYPE, params.getString(EXT_KEY_RANGE_LOWER_TYPE)) + .fluentPut(EXT_KEY_RANGE_UPPER_TYPE, params.getString(EXT_KEY_RANGE_UPPER_TYPE)); + case BOOLEAN: + return Optional.ofNullable(BOOLEAN_MAP.get(value.toLowerCase())) + .orElseThrow(ResultCode.IMPORT_CELL_BOOLEAN_VALUE_ERROR::toException); + default: + return Strings.nullToEmpty(value); + } + } + + private LocalDateTime parseDate(String dateStr, List formats) { + try { + Date date = DateUtils.parseDate(dateStr, formats.toArray(new String[0])); + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } catch (Exception e) { + throw ResultCode.IMPORT_CELL_DATETIME_CONVERT_FAILED.toException("不支持的日期格式{}", dateStr); + } + } + + @AllArgsConstructor + public enum Type { + INT("00"), + LONG("01"), + FLOAT("02"), + DOUBLE("03"), + STRING("04"), + BIG_DECIMAL("05"), + DATE("06"), + DATETIME("07"), + RANGE("08"), + BOOLEAN("09") + ; + + @Getter + private String code; + + private static Map map = Stream.of(Type.values()) + .collect(Collectors.toMap(Type::getCode, Function.identity())); + + public static Type from(String code) { + return map.get(code); + } + } + } + + @AllArgsConstructor + @Getter + enum ExportFormat { + EXCEL("application/vnd.ms-excel", ".xlsx"), + CSV("text/csv", ".csv"); + + private String contentType; + private String suffix; + } + + static BiFunction indexCellReader() { + return (row, index) -> String.valueOf(index + 1); + } + + static BiFunction stringCellReader(String columnName) { + return (row, index) -> Strings.nullToEmpty(row.getString(columnName)); + } + + static BiFunction dateTimeCellReader(String columnName, String pattern) { + return (row, index) -> { + Long epochMillis = row.getLong(columnName); + if (epochMillis == null) { + return StringUtils.EMPTY; + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); + Instant instant = Instant.ofEpochMilli(epochMillis); + LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); + return formatter.format(localDateTime); + }; + } + + static BiFunction bigDecimalCellReader(String columnName) { + return bigDecimalCellReader(columnName, 2); + } + + static BiFunction bigDecimalCellReader(String columnName, int scale) { + return (row, index) -> { + BigDecimal number = row.getBigDecimal(columnName); + if (number == null) { + number = BigDecimal.ZERO; + } + + return number.stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString(); + }; + } + + static BiFunction jsonPathCellReader(String jsonPath) { + return (row, index) -> JSONPath.eval(row, jsonPath); + } + + static BiFunction jsonPathBigDecimalCellReader(String jsonPath) { + return jsonPathBigDecimalCellReader(jsonPath, 2); + } + + static BiFunction jsonPathBigDecimalCellReader(String jsonPath, int scale) { + return (row, index) -> { + Object val = JSONPath.eval(row, jsonPath); + if (val == null) { + return BigDecimal.ZERO; + } + return TypeUtils.castToBigDecimal(val).stripTrailingZeros().setScale(scale, RoundingMode.HALF_UP).toPlainString(); + }; + } + + @Getter + class DataSheetException extends BusinessException { + private String subErrorCode; + /** 异常相关的行号,列号,0开始 */ + private Integer rowIndex; + private Integer columnIndex; + /** 异常相关的列名称,中文 */ + private List columnNames; + + public DataSheetException(ResultCode resultCode) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + } + + public DataSheetException(ResultCode resultCode, Integer rowIndex, Integer columnIndex) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + this.rowIndex = rowIndex; + this.columnIndex = columnIndex; + } + + public DataSheetException(ResultCode resultCode, List columnNames) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), resultCode.getMessage()); + this.subErrorCode = resultCode.getSubBizCode(); + this.columnNames = columnNames; + } + + public DataSheetException(ResultCode resultCode, String message) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), message); + this.subErrorCode = resultCode.getSubBizCode(); + } + + public DataSheetException(String subErrorCode, String errorMsg, Integer rowIndex, Integer columnIndex) { + super(BizResultCode.SYSTEM_PARAM_NOT_VALID_EXCEPTION.getErrorCode(), errorMsg); + this.subErrorCode = subErrorCode; + this.rowIndex = rowIndex; + this.columnIndex = columnIndex; + } + + public DataSheetException(String errorCode, String subErrorCode, String errorMsg) { + super(errorCode, errorMsg); + this.subErrorCode = subErrorCode; + } + } + + @AllArgsConstructor + @Getter + enum ResultCode { + /** 解析Excel的批注时,相关的errorCode*/ + IMPORT_PARSE_MISSING_VERSION("C00", "批注中没有找到版本信息"), + IMPORT_PARSE_VERSION_FORMAT_ERROR("C01", "批注中的版本格式不对"), + IMPORT_PARSE_CELL_META_MISSING_TYPE("C02", "批注中的字段没有找到类型信息"), + IMPORT_PARSE_CELL_META_FORMAT_ERROR("C03", "批注中的字段格式不对"), + IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR("C04", "批注中的范围类型字段的格式不对"), + IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR("C05", "批注中的范围类型字段的区间类型不对"), + + /** 导入数据时,相关的errorCode*/ + IMPORT_LINES_REACHED_LIMIT("C20", "导入的数据超过了最大行数"), + IMPORT_COLUMN_RANGE_MISSING_TYPES("C21", "范围类型缺少区间类型的定义:开区间还是闭区间"), + IMPORT_COLUMN_DUPLICATED_NAME("C22", "列的名称不能重复"), + IMPORT_COLUMN_MISSING_CELL_META("C23", "字段缺少类型定义"), + IMPORT_COLUMN_NAME_NOT_MATCHED("C24", "字段的名称与类型定义的名称不一致"), + IMPORT_CELL_RANGE_FORMAT_ERROR("C25", "范围类型的格式不对"), + IMPORT_CELL_RANGE_VALUE_ERROR("C26", "范围类型的下限值必须小于上限值"), + IMPORT_CELL_DATETIME_CONVERT_FAILED("C27", "时间类型解析失败"), + IMPORT_CELL_CONVERT_FAILED("C28", "类型转换失败"), + IMPORT_CELL_META_MISSING_MANDATORY_VALUE("C29", "必填字段不能为空"), + IMPORT_CELL_BOOLEAN_VALUE_ERROR("C30", "布尔类型的值不支持"), + ; + + private String subBizCode; + private String message; + + public DataSheetException toException() { + return new DataSheetException(this); + } + + public DataSheetException toException(String message) { + return new DataSheetException(this, message); + } + + public DataSheetException toException(List columnNames) { + return new DataSheetException(this, columnNames); + } + + public DataSheetException toException(Integer rowIndex, Integer columnIndex) { + return new DataSheetException(this, rowIndex, columnIndex); + } + + public DataSheetException toException(String customMsg, Object... objects) { + if (objects == null) { + return toException(customMsg); + } + + String msg = VarParamFormatter.format(customMsg, objects); + //如果最后一个参数是Throwable. 则将SimpleName附加到msg中 + if (objects[objects.length - 1] instanceof Throwable) { + Throwable throwable = (Throwable) objects[objects.length - 1]; + msg = String.format("%s (%s)", msg, throwable.getClass().getSimpleName()); + } + + return toException(msg); + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetClientImpl.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetClientImpl.java new file mode 100644 index 0000000..f189e12 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetClientImpl.java @@ -0,0 +1,54 @@ +package cn.axzo.pokonyan.client.impl; + +import cn.axzo.pokonyan.client.DataSheetClient; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; + +@Slf4j +public class DataSheetClientImpl implements DataSheetClient { + + @Override + public ImporterBuilder importBuilder() { + return new DataSheetImporter(); + } + + + @Data + @lombok.Builder + @NoArgsConstructor + @AllArgsConstructor + private static class ReportReq { + String appName; + String scene; + ReportReq.Action action; + Map resp; + Long rowCount; + Long elapsedMillis; + String filePath; + String operatorId; + String operatorName; + String operatorTenantId; + + public enum Action { + IMPORT, + EXPORT; + } + } + + + public static Builder builder() { + return new Builder(); + } + + @Data + public static class Builder { + + public DataSheetClient build() { + return new DataSheetClientImpl(); + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java new file mode 100644 index 0000000..93d0f1d --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java @@ -0,0 +1,536 @@ +package cn.axzo.pokonyan.client.impl; + +import cn.axzo.pokonyan.client.DataSheetClient; +import cn.axzo.pokonyan.exception.BusinessException; +import cn.axzo.pokonyan.util.Regex; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.enums.CellExtraTypeEnum; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.excel.metadata.CellExtra; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.io.InputStream; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_RANGE_LOWER_TYPE; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.EXT_KEY_RANGE_UPPER_TYPE; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.RANGE_BOUND_TYPE_CLOSED; +import static cn.axzo.pokonyan.client.DataSheetClient.CellMeta.RANGE_BOUND_TYPE_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.IGNORE_COLUMN_KEY; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.IGNORE_ROW_AND_COLUMN_KEY; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.IGNORE_ROW_KEY; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_CELL_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_CLOSE; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_EXTRA_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_IGNORE_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.PATTERN_VERSION_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.RANGE_TYPE_CLOSED_CLOSED; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.RANGE_TYPE_CLOSED_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.RANGE_TYPE_OPEN_CLOSED; +import static cn.axzo.pokonyan.client.DataSheetClient.Meta.RANGE_TYPE_OPEN_OPEN; +import static cn.axzo.pokonyan.client.DataSheetClient.ResultCode.IMPORT_CELL_CONVERT_FAILED; + +/** + * 使用ali的EasyExcel来读入excel/csv文件 + * + * @author yuanyi + */ +@Slf4j +public class DataSheetImporter extends DataSheetClient.ImporterBuilder { + + private static final String LINE_KEY_ERRORS = "errors"; + private static final String LINE_KEY_ROW_INDEX = "rowIndex"; + + /** + * 完成后处理事件. + */ + @Setter + private Consumer onCompleted; + + @Override + public DataSheetClient.Importer build() { + Preconditions.checkArgument(this.scene() != null, "scene不能为空"); + Preconditions.checkArgument(this.format() != null, "format不能为空"); + + // TODO: 当支持更多format的时候,需要生成对应的Importer实例 + return ExcelImporter.builder() + .scene(scene()) + .tableName(tableName()) + .meta(meta()) + .debugEnabled(debugEnabled()) + .allowMaxLineCount(allowMaxLineCount()) + .includeLineErrors(includeLineErrors()) + .onCompleted(onCompleted) + .build(); + } + + @Builder + private static class ExcelImporter implements DataSheetClient.Importer { + + private static final Map> rangeBoundTypes = ImmutableMap.of( + RANGE_TYPE_OPEN_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_OPEN), + RANGE_TYPE_OPEN_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_OPEN, RANGE_BOUND_TYPE_CLOSED), + RANGE_TYPE_CLOSED_OPEN, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_OPEN), + RANGE_TYPE_CLOSED_CLOSED, ImmutableList.of(RANGE_BOUND_TYPE_CLOSED, RANGE_BOUND_TYPE_CLOSED)); + + private String scene; + private String tableName; + private DataSheetClient.Meta meta; + private boolean debugEnabled; + private Integer allowMaxLineCount; + private boolean includeLineErrors; + private Consumer onCompleted; + + @Override + public DataSheetClient.ImportResp readAll(InputStream inputStream) { + Stopwatch stopwatch = Stopwatch.createStarted(); + + NoModelDataListener dataListener = new NoModelDataListener(); + EasyExcel.read(inputStream, dataListener).extraRead(CellExtraTypeEnum.COMMENT).sheet(tableName).doRead(); + if (allowMaxLineCount != null && dataListener.lines.size() > allowMaxLineCount) { + throw DataSheetClient.ResultCode.IMPORT_LINES_REACHED_LIMIT + .toException("导入的数据超过了最大行数" + allowMaxLineCount); + } + + // 聚合来自入参和文件批注的meta信息;如果都存在,使用入参的meta覆盖文件批注的meta + this.meta = mergeMeta(this.meta, parseMetaFromData(dataListener)); + filterHeadMap(dataListener); + filterLines(dataListener); + + validateHeaders(dataListener); + validateMeta(dataListener); + + List headers = parseHeaders(dataListener); + List lines = parseLines(dataListener); + if (!includeLineErrors) { + // 没有设置includeLineErrors时,抛第一个发现的异常 + Optional errorLine = lines.stream() + .filter(line -> line.containsKey(LINE_KEY_ERRORS)) + .findFirst(); + if (errorLine.isPresent()) { + JSONObject error = errorLine.get().getJSONArray(LINE_KEY_ERRORS).getJSONObject(0); + Integer rowIndex = lines.indexOf(errorLine.get()); + Integer columnIndex = error.getInteger("columnIndex"); + + String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1, + error.getString("columnName"), error.getString("errorMsg")); + throw new DataSheetClient.DataSheetException(error.getString("subErrorCode"), errMsg, rowIndex, columnIndex); + } + } + + DataSheetClient.ImportResp resp = DataSheetClient.ImportResp.builder() + .scene(scene) + .templateCode(Optional.ofNullable(meta).map(DataSheetClient.Meta::getTemplateCode).orElse(null)) + .version(Optional.ofNullable(meta).map(DataSheetClient.Meta::getVersion).orElse(null)) + .headers(headers) + .lines(lines) + .headerRowCount(1) + .rowCount(lines.size()) + .columnCount(headers.size()) + .meta(meta) + .elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS)) + .build(); + + if (null != onCompleted) { + onCompleted.accept(resp); + } + return resp; + } + + private void filterHeadMap(NoModelDataListener dataListener) { + if (meta == null) { + return; + } + + Map headMap = dataListener.getHeadMap(); + meta.getIgnoreColumnIndexes().forEach(headMap::remove); + } + + private void filterLines(NoModelDataListener dataListener) { + if (meta == null) { + return; + } + Set toRemoveLines = ImmutableSet.copyOf(meta.getIgnoreRowIndexes()); + List> lines = dataListener.getLines(); + dataListener.setLines(IntStream.range(0, lines.size()) + // 这里要加1,是因为removeIndex是整个文档的行数来计算,包括了header + // 但是lines的数据,已经排除了header + .filter(index -> !toRemoveLines.contains(index + 1)) + .mapToObj(lines::get) + .collect(Collectors.toList())); + } + + private DataSheetClient.Meta parseMetaFromData(NoModelDataListener dataListener) { + Map headMap = dataListener.getHeadMap(); + List headComments = dataListener.getCellComments().stream() + // 获取第一行Head的批注信息 + .filter(cellExtra -> cellExtra.getRowIndex() == 0) + // 过滤带有Cell参数配置的批注信息 + .filter(cellExtra -> !Strings.isNullOrEmpty(StringUtils.substringBetween(cellExtra.getText(), + PATTERN_CELL_OPEN, PATTERN_CLOSE))) + // 排序,方便找到第一列,获取templateCode与version + .sorted(Comparator.comparing(CellExtra::getColumnIndex)) + .collect(Collectors.toList()); + // 没有批注信息,直接返回null;会以String来解析值 + if (headComments.isEmpty()) { + return null; + } + + // 从文件中解析meta信息 + JSONObject codeAndVersion = parseTemplateCodeAndVersion(headComments); + List cellMetas = headComments.stream() + .map(cellExtra -> parseCellMeta(cellExtra.getText(), headMap.get(cellExtra.getColumnIndex()))) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (cellMetas.isEmpty()) { + return null; + } + + return DataSheetClient.Meta.builder() + .templateCode(codeAndVersion.getString("templateCode")) + .version(codeAndVersion.getString("version")) + .cellMetas(cellMetas) + .ignoreColumnIndexes(parseIgnoreColumns(dataListener)) + .ignoreRowIndexes(parseIgnoreRows(dataListener)) + .build(); + } + + private DataSheetClient.Meta mergeMeta(DataSheetClient.Meta metaFromParam, DataSheetClient.Meta metaFromHeader) { + if (metaFromHeader == null || metaFromParam == null) { + return metaFromHeader == null ? metaFromParam : metaFromHeader; + } + + // 在两者都存在的时候,以文件的meta为准;仅仅将入参的CellMetas覆盖文件的CellMetas + if (metaFromHeader.getCellMetas() == null) { + metaFromHeader.setCellMetas(metaFromParam.getCellMetas()); + return metaFromHeader; + } + + // 使用入参中定义的列信息,覆盖文件的cellMetas + Map cellMetaMap = + Maps.uniqueIndex(metaFromParam.getCellMetas(), DataSheetClient.CellMeta::getKey); + List cellMetas = metaFromHeader.getCellMetas().stream() + .map(cellMeta -> cellMetaMap.getOrDefault(cellMeta.getKey(), cellMeta)) + .collect(Collectors.toList()); + metaFromHeader.setCellMetas(cellMetas); + return metaFromHeader; + } + + private List parseIgnoreRows(NoModelDataListener dataListener) { + return dataListener.getCellComments().stream() + .map(cellExtra -> { + String ignoreKey = StringUtils.substringBetween(cellExtra.getText(), + PATTERN_IGNORE_OPEN, PATTERN_CLOSE); + if (IGNORE_ROW_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) { + return cellExtra.getRowIndex(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private List parseIgnoreColumns(NoModelDataListener dataListener) { + return dataListener.getCellComments().stream() + .map(cellExtra -> { + String ignoreKey = StringUtils.substringBetween(cellExtra.getText(), + PATTERN_IGNORE_OPEN, PATTERN_CLOSE); + if (IGNORE_COLUMN_KEY.equals(ignoreKey) || IGNORE_ROW_AND_COLUMN_KEY.equals(ignoreKey)) { + return cellExtra.getColumnIndex(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + public JSONObject parseTemplateCodeAndVersion(List headComments) { + // 从整个头部的批注,获取模版code和版本信息 + Optional parsedText = headComments.stream() + .map(comment -> StringUtils.substringBetween(comment.getText(), PATTERN_VERSION_OPEN, PATTERN_CLOSE)) + .filter(str -> !Strings.isNullOrEmpty(str)) + .findFirst(); + if (!parsedText.isPresent()) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_MISSING_VERSION.toException(); + } + // 格式为"CODE_VERSION" + List values = Splitter.on("_").splitToList(parsedText.get()); + if (values.size() != 2) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_VERSION_FORMAT_ERROR.toException(); + } + return new JSONObject() + .fluentPut("templateCode", values.get(0)) + .fluentPut("version", values.get(1)); + } + + private DataSheetClient.CellMeta parseCellMeta(String text, String name) { + String value = StringUtils.substringBetween(text, PATTERN_CELL_OPEN, PATTERN_CLOSE); + if (Strings.isNullOrEmpty(value)) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_MISSING_TYPE + .toException("没有找到列[{}]的类型信息", name); + } + List values = Splitter.on("_").splitToList(value); + // 格式: key_type_mandatory_params... + if (values.size() < 3) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_FORMAT_ERROR + .toException("列[{}]类型的批注格式不对[{}]", name, text); + } + + DataSheetClient.CellMeta.Type type = DataSheetClient.CellMeta.Type.from(values.get(1)); + JSONObject params = new JSONObject(); + if (type == DataSheetClient.CellMeta.Type.RANGE) { + if (values.size() != 4 || Strings.isNullOrEmpty(values.get(3))) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_FORMAT_ERROR + .toException("列[{}]范围类型的批注格式不对[{}]", name, text); + } + List boundType = rangeBoundTypes.get(values.get(3)); + if (boundType == null) { + throw DataSheetClient.ResultCode.IMPORT_PARSE_CELL_META_RANGE_TYPE_ERROR + .toException(String.format("列[{}]范围类型的值不对", name)); + } + params.put(EXT_KEY_RANGE_LOWER_TYPE, boundType.get(0)); + params.put(EXT_KEY_RANGE_UPPER_TYPE, boundType.get(1)); + } + + JSONObject ext = new JSONObject(); + value = StringUtils.substringBetween(text, PATTERN_EXTRA_OPEN, PATTERN_CLOSE); + if (!Strings.isNullOrEmpty(value)) { + ext.putAll(Splitter.on("&").withKeyValueSeparator("=").split(value)); + } + + return DataSheetClient.CellMeta.builder() + .key(values.get(0)) + .name(name) + .type(type) + .mandatory("1".equals(values.get(2))) + .params(params) + .ext(ext) + .build(); + } + + private List parseHeaders(NoModelDataListener dataListener) { + return dataListener.getHeadMap().keySet().stream() + .sorted() + .map(key -> Strings.nullToEmpty(dataListener.getHeadMap().get(key))) + .collect(Collectors.toList()); + } + + private List parseLines(NoModelDataListener dataListener) { + // 如果没有找到meta信息,按照key=header(很可能是中文), 类型就为String + if (this.meta == null) { + Map headerMap = dataListener.getHeadMap(); + return dataListener.getLines().stream() + .map(line -> new JSONObject().fluentPutAll(line.entrySet().stream() + .map(entry -> Pair.of(headerMap.get(entry.getKey()), Strings.nullToEmpty(entry.getValue()))) + .collect(Collectors.toMap(Pair::getKey, Pair::getValue)))) + .collect(Collectors.toList()); + } + + // 根据meta来校验cell的类型 + Map headerMap = dataListener.getHeadMap(); + Map cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), + DataSheetClient.CellMeta::getName); + List> lines = dataListener.getLines(); + return IntStream.range(0, lines.size()) + .mapToObj(lineIndex -> { + Map line = lines.get(lineIndex); + // 收集每一行每一列的转换结果 + Map> convertRespMap = headerMap.entrySet().stream() + .map(entry -> { + Integer columnIndex = entry.getKey(); + String header = entry.getValue(); + DataSheetClient.CellMeta cellMeta = cellMetaMap.get(header); + return convertType(cellMeta, line.get(columnIndex), lineIndex, columnIndex); + }) + .collect(Collectors.groupingBy(ColumnConvertResp::getSuccess)); + + JSONObject convertedLine = new JSONObject() + .fluentPutAll(convertRespMap.getOrDefault(Boolean.TRUE, ImmutableList.of()).stream() + // convertValue可能为null, 有非必需的字段 + .collect(HashMap::new, (m, v) -> m.put(v.getKey(), v.getConvertedValue()), HashMap::putAll)); + if (convertRespMap.get(Boolean.FALSE) != null) { + // 转换失败的,将失败信息放到errors字段中 + convertedLine.put(LINE_KEY_ERRORS, convertRespMap.get(Boolean.FALSE).stream() + .map(ColumnConvertResp::getError) + .collect(Collectors.toList())); + } + convertedLine.put(LINE_KEY_ROW_INDEX, lineIndex); + return convertedLine; + }) + .collect(Collectors.toList()); + } + + private ColumnConvertResp convertType(DataSheetClient.CellMeta cellMeta, String rawValue, int rowIndex, int columnIndex) { + try { + return ColumnConvertResp.builder() + .success(true).cellMeta(cellMeta) + .rawValue(rawValue).convertedValue(cellMeta.convertType(rawValue)) + .build(); + } catch (BusinessException e) { + log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}", + cellMeta, rawValue, rowIndex, columnIndex, e); + String subErrorCode = null; + if (e instanceof DataSheetClient.DataSheetException) { + subErrorCode = ((DataSheetClient.DataSheetException) e).getSubErrorCode(); + } + return ColumnConvertResp.builder() + .success(false).cellMeta(cellMeta).rawValue(rawValue) + .columnIndex(columnIndex) + .errorCode(e.getErrorCode()) + .errorMsg(e.getErrorMsg()) + .subErrorCode(subErrorCode) + .build(); + } catch (Exception e) { + log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}", + cellMeta, rawValue, rowIndex, columnIndex, e); + String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1, + cellMeta.getName(), Optional.ofNullable(e.getMessage()) + .orElse(IMPORT_CELL_CONVERT_FAILED.getMessage())); + throw new DataSheetClient.DataSheetException(IMPORT_CELL_CONVERT_FAILED.getSubBizCode(), errMsg, rowIndex, columnIndex); + } + } + + private void validateHeaders(NoModelDataListener dataListener) { + Map headMap = dataListener.getHeadMap(); + Set headerNames = ImmutableSet.copyOf(headMap.values()); + if (headerNames.size() != headMap.size()) { + List columnNames = headMap.values().stream() + .collect(Collectors.groupingBy(Function.identity())) + .entrySet().stream() + .filter(grouped -> grouped.getValue().size() > 1) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + throw DataSheetClient.ResultCode.IMPORT_COLUMN_DUPLICATED_NAME.toException(columnNames); + } + } + + private void validateMeta(NoModelDataListener dataListener) { + if (this.meta == null) { + return; + } + + if (meta.getCellMetas().size() != dataListener.getHeadMap().size()) { + throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException(); + } + + Set headerNames = ImmutableSet.copyOf(dataListener.getHeadMap().values()); + Set cellNames = meta.getCellMetas().stream() + .map(DataSheetClient.CellMeta::getName) + .collect(Collectors.toSet()); + if (!headerNames.equals(cellNames)) { + Set missingNames = Sets.difference(cellNames, headerNames); + Set redundantNames = Sets.difference(headerNames, cellNames); + List columnNames = Stream.of(missingNames, redundantNames) + .flatMap(Set::stream) + .collect(Collectors.toList()); + throw DataSheetClient.ResultCode.IMPORT_COLUMN_NAME_NOT_MATCHED.toException(columnNames); + } + meta.getCellMetas().forEach(DataSheetClient.CellMeta::validate); + } + + @Slf4j + @Data + private static class NoModelDataListener extends AnalysisEventListener> { + private Map headMap = Maps.newHashMap(); + private List> lines = Lists.newArrayList(); + private List cellComments = Lists.newArrayList(); + + @Override + public void invoke(Map data, AnalysisContext context) { + lines.add(strip(data)); + } + + @Override + public void invokeHeadMap(Map headMap, AnalysisContext context) { + this.headMap.putAll(strip(headMap)); + } + + private Map strip(Map data) { + return data.entrySet().stream() + .map(entry -> Maps.immutableEntry(entry.getKey(), StringUtils.strip(entry.getValue(), Regex.WHITESPACE_CHARS))) + // value有可能为null,不能直接用Collectors.toMap + .collect(Maps::newHashMap, (m, v) -> m.put(v.getKey(), v.getValue()), HashMap::putAll); + } + + /** + * extra是在整个文件被解析后才会调用 + * + * @param extra + * @param context + */ + @Override + public void extra(CellExtra extra, AnalysisContext context) { + if (extra.getType() == CellExtraTypeEnum.COMMENT) { + cellComments.add(extra); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + + } + } + + @Builder + @Data + @NoArgsConstructor + @AllArgsConstructor + private static class ColumnConvertResp { + private Boolean success; + private DataSheetClient.CellMeta cellMeta; + private String rawValue; + private Object convertedValue; + + private Integer columnIndex; + private String errorCode; + private String errorMsg; + private String subErrorCode; + + public String getKey() { + return cellMeta.getKey(); + } + + public JSONObject getError() { + return new JSONObject() + .fluentPut("columnIndex", columnIndex) + .fluentPut("columnKey", cellMeta.getKey()) + .fluentPut("columnName", cellMeta.getName()) + .fluentPut("rawValue", rawValue) + .fluentPut("errorCode", errorCode) + .fluentPut("errorMsg", errorMsg) + .fluentPut("subErrorCode", subErrorCode); + } + } + } +} diff --git a/src/main/java/cn/axzo/pokonyan/exception/BizResultCode.java b/src/main/java/cn/axzo/pokonyan/exception/BizResultCode.java new file mode 100644 index 0000000..c313ba4 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/exception/BizResultCode.java @@ -0,0 +1,16 @@ +package cn.axzo.pokonyan.exception; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum BizResultCode implements ResultCode { + + SYSTEM_PARAM_NOT_VALID_EXCEPTION("001", "参与异常"),; + + + private String errorCode; + private String errorMessage; + +} diff --git a/src/main/java/cn/axzo/pokonyan/util/Regex.java b/src/main/java/cn/axzo/pokonyan/util/Regex.java new file mode 100644 index 0000000..3ccd516 --- /dev/null +++ b/src/main/java/cn/axzo/pokonyan/util/Regex.java @@ -0,0 +1,40 @@ +package cn.axzo.pokonyan.util; + +import org.apache.commons.lang3.StringUtils; + +import java.util.regex.Pattern; + +/** + * 重用的数据格式验证正则表达式 + */ +public class Regex { + + public final static String MOBILE_REGEX = "^1\\d{10}$"; + public final static String MOBILE_REGEX_MESSAGE = "手机号格式不正确"; + public final static Pattern MOBILE_PATTERN = Pattern.compile(MOBILE_REGEX); + + public static final String ID_NO_REGEX = "^(\\d{15}$|^\\d{18}$|^\\d{17}(\\d|X|x))$"; + public final static String ID_NO_REGEX_MESSAGE = "身份证格式不正确"; + public final static Pattern ID_NO_PATTERN = Pattern.compile(ID_NO_REGEX); + + public static final String COMPANY_LICENSE_NO_REGEX = "^(\\w{15}|\\w{18})$"; + public final static String COMPANY_LICENSE_NO_REGEX_MESSAGE = "营业执照号格式不正确"; + public final static Pattern COMPANY_LICENSE_NO_PATTERN = Pattern.compile(COMPANY_LICENSE_NO_REGEX); + + public static final String TEL_REGEX = "^((0\\d{2,3})-?)(\\d{7,8})?$"; + public final static String TEL_REGEX_MESSAGE = "固定电话格式不正确"; + public final static Pattern TEL_PATTERN = Pattern.compile(TEL_REGEX); + + public static final String EMAIL_REGEX = "^([-|\\w])+(\\.[-|\\w]+)*@(\\w)+((\\.\\w+)+)$"; + public final static String EMAIL_REGEX_MESSAGE = "邮件格式不正确"; + public final static Pattern EMAIL_PATTERN = Pattern.compile(EMAIL_REGEX); + + public static final String HTTP_REGEX = "^(http|https)://([\\w.]+/?)\\S*$"; + + public static final String REAL_NAME_REGEX = "^[\\u4E00-\\u9FA5]{2,32}$|^$"; + public static final String REAL_NAME_REGEX_MESSAGE = "姓名格式不正确"; + public final static Pattern REAL_NAME_PATTERN = Pattern.compile(REAL_NAME_REGEX); + + /**常用的制表符*/ + public static final String WHITESPACE_CHARS = "\r\n\0\t\b" + StringUtils.SPACE; +} From dda80932a7103b5c12f759e06140c980bfdf49dd Mon Sep 17 00:00:00 2001 From: lilong Date: Wed, 20 Mar 2024 10:22:43 +0800 Subject: [PATCH 08/17] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9dataSheet?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=A0=B9=E6=8D=AEcolumn=E6=88=96=E8=80=85tit?= =?UTF-8?q?le=E6=9D=A5=E8=A7=A3=E6=9E=90excel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../axzo/pokonyan/client/DataSheetClient.java | 6 ++++++ .../client/impl/DataSheetImporter.java | 20 ++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java b/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java index 16a2c07..28d221c 100644 --- a/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java +++ b/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java @@ -316,6 +316,12 @@ public interface DataSheetClient { * cell的header,一般为中文 */ private String name; + + /** + * excel的列顺序,从0开始,可以name或者column二选一 + */ + private Integer column; + /** * 类型 */ diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java index 93d0f1d..c8a9246 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java @@ -25,6 +25,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -359,8 +360,9 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { // 根据meta来校验cell的类型 Map headerMap = dataListener.getHeadMap(); - Map cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), - DataSheetClient.CellMeta::getName); + boolean cellTypeColumn = meta.getCellMetas().stream().anyMatch(cellMeta -> cellMeta.getColumn() == null); + Map cellMetaMap = resolveCellMetaMap(cellTypeColumn); + List> lines = dataListener.getLines(); return IntStream.range(0, lines.size()) .mapToObj(lineIndex -> { @@ -370,7 +372,7 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { .map(entry -> { Integer columnIndex = entry.getKey(); String header = entry.getValue(); - DataSheetClient.CellMeta cellMeta = cellMetaMap.get(header); + DataSheetClient.CellMeta cellMeta = cellTypeColumn ? cellMetaMap.get(columnIndex.toString()) : cellMetaMap.get(header); return convertType(cellMeta, line.get(columnIndex), lineIndex, columnIndex); }) .collect(Collectors.groupingBy(ColumnConvertResp::getSuccess)); @@ -391,6 +393,18 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { .collect(Collectors.toList()); } + + private Map resolveCellMetaMap(boolean cellTypeColumn) { + if (BooleanUtils.isTrue(cellTypeColumn)) { + // 根据列顺序解析 + return Maps.uniqueIndex(meta.getCellMetas(), + cellMeta -> cellMeta.getColumn().toString()); + } + // 根据列名字解析 + return Maps.uniqueIndex(meta.getCellMetas(), + DataSheetClient.CellMeta::getName); + } + private ColumnConvertResp convertType(DataSheetClient.CellMeta cellMeta, String rawValue, int rowIndex, int columnIndex) { try { return ColumnConvertResp.builder() From 89d99f0f46e41e70a7659f6a588a037b91dc0947 Mon Sep 17 00:00:00 2001 From: lilong Date: Wed, 20 Mar 2024 10:40:05 +0800 Subject: [PATCH 09/17] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9dataSheet?= =?UTF-8?q?=E7=9A=84check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/axzo/pokonyan/client/DataSheetClient.java | 10 ++++++++++ .../axzo/pokonyan/client/impl/DataSheetImporter.java | 11 +++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java b/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java index 28d221c..8b32709 100644 --- a/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java +++ b/src/main/java/cn/axzo/pokonyan/client/DataSheetClient.java @@ -251,6 +251,8 @@ public interface DataSheetClient { */ private List ignoreColumnIndexes; + private ResolveRowType resolveRowType; + public List getIgnoreRowIndexes() { return Optional.ofNullable(ignoreRowIndexes).orElseGet(ImmutableList::of); } @@ -260,6 +262,14 @@ public interface DataSheetClient { } } + @AllArgsConstructor + @Getter + enum ResolveRowType { + COLUMN, + TITLE, + ; + } + @Builder @Data @NoArgsConstructor diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java index c8a9246..b20a47c 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java @@ -360,7 +360,7 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { // 根据meta来校验cell的类型 Map headerMap = dataListener.getHeadMap(); - boolean cellTypeColumn = meta.getCellMetas().stream().anyMatch(cellMeta -> cellMeta.getColumn() == null); + boolean cellTypeColumn = meta.getResolveRowType() == DataSheetClient.ResolveRowType.COLUMN; Map cellMetaMap = resolveCellMetaMap(cellTypeColumn); List> lines = dataListener.getLines(); @@ -458,6 +458,14 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException(); } + if (meta.getResolveRowType() == DataSheetClient.ResolveRowType.TITLE) { + checkMetaTitle(dataListener); + } + + meta.getCellMetas().forEach(DataSheetClient.CellMeta::validate); + } + + private void checkMetaTitle(NoModelDataListener dataListener) { Set headerNames = ImmutableSet.copyOf(dataListener.getHeadMap().values()); Set cellNames = meta.getCellMetas().stream() .map(DataSheetClient.CellMeta::getName) @@ -470,7 +478,6 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { .collect(Collectors.toList()); throw DataSheetClient.ResultCode.IMPORT_COLUMN_NAME_NOT_MATCHED.toException(columnNames); } - meta.getCellMetas().forEach(DataSheetClient.CellMeta::validate); } @Slf4j From d52e2cf0fc86c5d4167db43de40582286cff3a95 Mon Sep 17 00:00:00 2001 From: lilong Date: Mon, 25 Mar 2024 20:31:37 +0800 Subject: [PATCH 10/17] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java index b20a47c..d3631e6 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java @@ -428,7 +428,7 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { } catch (Exception e) { log.error("failed to convert type, cellMeta={}, rawValue={}, rowIndex={}, columnIndex={}", cellMeta, rawValue, rowIndex, columnIndex, e); - String errMsg = String.format("第%d行, 第%d列字段[%s]%s", rowIndex + 1, columnIndex + 1, + String errMsg = String.format("第%d行, 第%d列字段, %s", rowIndex + 1, columnIndex + 1, cellMeta.getName(), Optional.ofNullable(e.getMessage()) .orElse(IMPORT_CELL_CONVERT_FAILED.getMessage())); throw new DataSheetClient.DataSheetException(IMPORT_CELL_CONVERT_FAILED.getSubBizCode(), errMsg, rowIndex, columnIndex); From 2a3e906a1addc2687924bab63bb0272ceb4c2e28 Mon Sep 17 00:00:00 2001 From: lilong Date: Sat, 30 Mar 2024 08:15:32 +0800 Subject: [PATCH 11/17] =?UTF-8?q?feat:=E4=BF=AE=E6=94=B9=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E5=8F=AA=E6=94=AF=E6=8C=81column?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/impl/DataSheetImporter.java | 31 +++++-------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java index d3631e6..e52445b 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java @@ -25,7 +25,6 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -126,13 +125,10 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { // 聚合来自入参和文件批注的meta信息;如果都存在,使用入参的meta覆盖文件批注的meta this.meta = mergeMeta(this.meta, parseMetaFromData(dataListener)); - filterHeadMap(dataListener); filterLines(dataListener); - validateHeaders(dataListener); validateMeta(dataListener); - List headers = parseHeaders(dataListener); List lines = parseLines(dataListener); if (!includeLineErrors) { // 没有设置includeLineErrors时,抛第一个发现的异常 @@ -154,11 +150,10 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { .scene(scene) .templateCode(Optional.ofNullable(meta).map(DataSheetClient.Meta::getTemplateCode).orElse(null)) .version(Optional.ofNullable(meta).map(DataSheetClient.Meta::getVersion).orElse(null)) - .headers(headers) .lines(lines) .headerRowCount(1) .rowCount(lines.size()) - .columnCount(headers.size()) + .columnCount(lines.size()) .meta(meta) .elapsedMillis(stopwatch.elapsed(TimeUnit.MILLISECONDS)) .build(); @@ -341,9 +336,9 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { } private List parseHeaders(NoModelDataListener dataListener) { - return dataListener.getHeadMap().keySet().stream() + return dataListener.getHeadMap().entrySet().stream() .sorted() - .map(key -> Strings.nullToEmpty(dataListener.getHeadMap().get(key))) + .map(entry -> Strings.nullToEmpty(dataListener.getHeadMap().get(entry.getKey()))) .collect(Collectors.toList()); } @@ -360,8 +355,8 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { // 根据meta来校验cell的类型 Map headerMap = dataListener.getHeadMap(); - boolean cellTypeColumn = meta.getResolveRowType() == DataSheetClient.ResolveRowType.COLUMN; - Map cellMetaMap = resolveCellMetaMap(cellTypeColumn); + Map cellMetaMap = Maps.uniqueIndex(meta.getCellMetas(), + DataSheetClient.CellMeta::getColumn);; List> lines = dataListener.getLines(); return IntStream.range(0, lines.size()) @@ -371,8 +366,7 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { Map> convertRespMap = headerMap.entrySet().stream() .map(entry -> { Integer columnIndex = entry.getKey(); - String header = entry.getValue(); - DataSheetClient.CellMeta cellMeta = cellTypeColumn ? cellMetaMap.get(columnIndex.toString()) : cellMetaMap.get(header); + DataSheetClient.CellMeta cellMeta = cellMetaMap.get(columnIndex); return convertType(cellMeta, line.get(columnIndex), lineIndex, columnIndex); }) .collect(Collectors.groupingBy(ColumnConvertResp::getSuccess)); @@ -393,18 +387,6 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { .collect(Collectors.toList()); } - - private Map resolveCellMetaMap(boolean cellTypeColumn) { - if (BooleanUtils.isTrue(cellTypeColumn)) { - // 根据列顺序解析 - return Maps.uniqueIndex(meta.getCellMetas(), - cellMeta -> cellMeta.getColumn().toString()); - } - // 根据列名字解析 - return Maps.uniqueIndex(meta.getCellMetas(), - DataSheetClient.CellMeta::getName); - } - private ColumnConvertResp convertType(DataSheetClient.CellMeta cellMeta, String rawValue, int rowIndex, int columnIndex) { try { return ColumnConvertResp.builder() @@ -440,6 +422,7 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { Set headerNames = ImmutableSet.copyOf(headMap.values()); if (headerNames.size() != headMap.size()) { List columnNames = headMap.values().stream() + .filter(Objects::nonNull) .collect(Collectors.groupingBy(Function.identity())) .entrySet().stream() .filter(grouped -> grouped.getValue().size() > 1) From ead6b9af5fbbc292399ffeb88fe422b9c1ea8782 Mon Sep 17 00:00:00 2001 From: lilong Date: Sat, 30 Mar 2024 08:42:23 +0800 Subject: [PATCH 12/17] =?UTF-8?q?feat:=E5=8E=BB=E6=8E=89=E6=A0=B9=E6=8D=AE?= =?UTF-8?q?header=E7=9A=84=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/pokonyan/client/impl/DataSheetImporter.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java index e52445b..64a4f1e 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java @@ -437,14 +437,10 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { return; } - if (meta.getCellMetas().size() != dataListener.getHeadMap().size()) { + if (meta.getCellMetas().size() > dataListener.getLines().size()) { throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException(); } - if (meta.getResolveRowType() == DataSheetClient.ResolveRowType.TITLE) { - checkMetaTitle(dataListener); - } - meta.getCellMetas().forEach(DataSheetClient.CellMeta::validate); } From e994e7342e20f40f51acec420cb790049cc099c9 Mon Sep 17 00:00:00 2001 From: lilong Date: Sat, 30 Mar 2024 08:48:05 +0800 Subject: [PATCH 13/17] =?UTF-8?q?feat:=E5=8E=BB=E6=8E=89=E6=A0=B9=E6=8D=AE?= =?UTF-8?q?header=E7=9A=84=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java index 64a4f1e..97f88b5 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java @@ -437,7 +437,7 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { return; } - if (meta.getCellMetas().size() > dataListener.getLines().size()) { + if (meta.getCellMetas().size() > dataListener.getHeadMap().size()) { throw DataSheetClient.ResultCode.IMPORT_COLUMN_MISSING_CELL_META.toException(); } From 32b866c97462b5ba77a8dcf3aae874159cbf00ea Mon Sep 17 00:00:00 2001 From: lilong Date: Sat, 30 Mar 2024 09:03:43 +0800 Subject: [PATCH 14/17] =?UTF-8?q?feat:=20=E6=A0=B9=E6=8D=AEcolumn=E8=A7=A3?= =?UTF-8?q?=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../axzo/pokonyan/client/impl/DataSheetImporter.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java index 97f88b5..55ee5da 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/DataSheetImporter.java @@ -363,11 +363,11 @@ public class DataSheetImporter extends DataSheetClient.ImporterBuilder { .mapToObj(lineIndex -> { Map line = lines.get(lineIndex); // 收集每一行每一列的转换结果 - Map> convertRespMap = headerMap.entrySet().stream() - .map(entry -> { - Integer columnIndex = entry.getKey(); - DataSheetClient.CellMeta cellMeta = cellMetaMap.get(columnIndex); - return convertType(cellMeta, line.get(columnIndex), lineIndex, columnIndex); + Map> convertRespMap = cellMetaMap.entrySet() + .stream() + .map(cellMeta -> { + Integer columnIndex = cellMeta.getKey(); + return convertType(cellMeta.getValue(), line.get(columnIndex), lineIndex, columnIndex); }) .collect(Collectors.groupingBy(ColumnConvertResp::getSuccess)); From 0bfd1ef2170809ccb6cfa8233d6b1eacb0951dfb Mon Sep 17 00:00:00 2001 From: lilong Date: Tue, 2 Apr 2024 15:18:10 +0800 Subject: [PATCH 15/17] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9pageNumber?= =?UTF-8?q?=E4=B8=BApage=EF=BC=8C=E5=9B=A0=E4=B8=BA=E5=89=8D=E7=AB=AF?= =?UTF-8?q?=E7=BB=84=E4=BB=B6=E6=98=AF=E8=BF=99=E6=A0=B7=E5=AE=9A=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/cn/axzo/pokonyan/dao/converter/PageConverter.java | 2 +- src/main/java/cn/axzo/pokonyan/dao/page/IPageParam.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/cn/axzo/pokonyan/dao/converter/PageConverter.java b/src/main/java/cn/axzo/pokonyan/dao/converter/PageConverter.java index 3770871..efebbe0 100644 --- a/src/main/java/cn/axzo/pokonyan/dao/converter/PageConverter.java +++ b/src/main/java/cn/axzo/pokonyan/dao/converter/PageConverter.java @@ -31,7 +31,7 @@ public class PageConverter { */ public static Page convertToMybatis(IPageParam page, Class entityClz) { int pageSize = Math.min(Optional.ofNullable(page.getPageSize()).orElse(IPageParam.DEFAULT_PAGE_SIZE), IPageParam.MAX_PAGE_SIZE); - Integer current = Optional.ofNullable(page.getPageNumber()).orElse(IPageParam.DEFAULT_PAGE_NUMBER); + Integer current = Optional.ofNullable(page.getPage()).orElse(IPageParam.DEFAULT_PAGE_NUMBER); Page myBatisPage = new Page<>(current, pageSize); diff --git a/src/main/java/cn/axzo/pokonyan/dao/page/IPageParam.java b/src/main/java/cn/axzo/pokonyan/dao/page/IPageParam.java index eab36eb..f410f9c 100644 --- a/src/main/java/cn/axzo/pokonyan/dao/page/IPageParam.java +++ b/src/main/java/cn/axzo/pokonyan/dao/page/IPageParam.java @@ -15,7 +15,7 @@ public interface IPageParam { String SORT_DESC = OrderEnum.DESC.name(); String SORT_ASC = OrderEnum.ASC.name(); - default Integer getPageNumber() { + default Integer getPage() { return DEFAULT_PAGE_NUMBER; } From 4b0d92cbe9bd8971c82f76919f157b65c602760b Mon Sep 17 00:00:00 2001 From: lilong Date: Tue, 26 Mar 2024 16:49:41 +0800 Subject: [PATCH 16/17] =?UTF-8?q?feat:=20=E8=A7=A3=E5=86=B3=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=9D=A1=E4=BB=B6string=E6=9C=89=E4=BC=A0""=E7=9A=84?= =?UTF-8?q?=E6=83=85=E5=86=B5=EF=BC=8C=E8=BF=99=E7=A7=8D=E6=83=85=E5=86=B5?= =?UTF-8?q?=E7=9B=B4=E6=8E=A5=E5=BF=BD=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/axzo/pokonyan/dao/wrapper/CriteriaField.java | 6 ++++++ .../java/cn/axzo/pokonyan/dao/wrapper/CriteriaWrapper.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaField.java b/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaField.java index 660ec82..3275bc8 100644 --- a/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaField.java +++ b/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaField.java @@ -32,6 +32,12 @@ public @interface CriteriaField { */ boolean filterNull() default true; + /** + * 默认为true,当value为""的时候,自动过滤该查询条件. + * @return + */ + boolean filterBlank() default true; + /** * 是否忽略该字段的查询条件 * diff --git a/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaWrapper.java b/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaWrapper.java index 3ff907b..ee7f244 100644 --- a/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaWrapper.java +++ b/src/main/java/cn/axzo/pokonyan/dao/wrapper/CriteriaWrapper.java @@ -338,6 +338,12 @@ public class CriteriaWrapper { return null; } + if (fieldAnnotation.filterBlank() + && (value instanceof String) + && StringUtils.isBlank((String) value)) { + return null; + } + String fieldName = Strings.isNullOrEmpty(fieldAnnotation.field()) ? field : fieldAnnotation.field(); // XXX 注解中获取的operator如果为EQ,无法判断是用户手动设置为EQ还是使用的默认operator.EQ,这里暂时保持原状 return build(fieldName, fieldAnnotation.operator(), value, fieldAnnotation.prefix()); From cfe659ca8f61acddcceef7ee5ba0c149f7537a03 Mon Sep 17 00:00:00 2001 From: lilong Date: Thu, 21 Mar 2024 17:10:06 +0800 Subject: [PATCH 17/17] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E4=BB=A4=E7=89=8C=E7=AD=89=E5=BE=85=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/axzo/pokonyan/client/RateLimiter.java | 2 ++ .../client/impl/RedisRateLimiterImpl.java | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java b/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java index f93af78..36643cd 100644 --- a/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java +++ b/src/main/java/cn/axzo/pokonyan/client/RateLimiter.java @@ -18,6 +18,8 @@ public interface RateLimiter { */ boolean tryAcquire(); + boolean tryAcquire(long timeoutMillis); + /** * 获取窗口类型 * diff --git a/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java b/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java index 01d537f..2ea1490 100644 --- a/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java +++ b/src/main/java/cn/axzo/pokonyan/client/impl/RedisRateLimiterImpl.java @@ -12,6 +12,7 @@ import org.redisson.api.RateType; import org.redisson.api.RedissonClient; import java.util.Objects; +import java.util.concurrent.TimeUnit; @Slf4j public class RedisRateLimiterImpl implements RateLimiter { @@ -26,6 +27,12 @@ public class RedisRateLimiterImpl implements RateLimiter { private RRateLimiter rateLimiter; + /** + * 默认超时时间(毫秒) + */ + private static final long DEFAULT_TIME_OUT_MILLIS = 5 * 1000; + + @Builder RedisRateLimiterImpl(RedissonClient redissonClient, WindowType windowType, @@ -56,6 +63,11 @@ public class RedisRateLimiterImpl implements RateLimiter { return rateLimiterWorker.tryAcquire(); } + @Override + public boolean tryAcquire(long timeoutMillis) { + return rateLimiterWorker.tryAcquire(timeoutMillis); + } + @Override public WindowType getWindowType() { return windowType; @@ -89,6 +101,11 @@ public class RedisRateLimiterImpl implements RateLimiter { public boolean tryAcquire() { return rateLimiter.tryAcquire(1); } + + @Override + public boolean tryAcquire(long timeoutMillis) { + return rateLimiter.tryAcquire(1, timeoutMillis, TimeUnit.MILLISECONDS); + } } interface RateLimiterWorker { @@ -98,5 +115,7 @@ public class RedisRateLimiterImpl implements RateLimiter { * @return 如果获取成功则返回true, 失败则为false */ boolean tryAcquire(); + + boolean tryAcquire(long timeoutMillis); } }