From ab0b0cdcbde425a3811e654358f6b79b5e1e0ed2 Mon Sep 17 00:00:00 2001 From: yanglin Date: Tue, 23 Apr 2024 18:53:53 +0800 Subject: [PATCH 01/33] REQ-2405: backup --- .../config/PendingMessageBizConfig.java | 6 + .../impl/MessageRecordServiceImpl.java | 11 + .../controller/PrivateMessageController.java | 8 + .../impl/GeneralMessageOldServiceImpl.java | 20 +- .../impl/GeneralMessageServiceImpl.java | 37 +-- .../service/impl/oldmsg/CacheValue.java | 48 +++ .../impl/oldmsg/OldMsgIdentifyFilter.java | 52 ++++ .../impl/oldmsg/OldMsgIdentifyInfo.java | 17 + .../service/impl/oldmsg/OldMsgStatCache.java | 292 ++++++++++++++++++ .../oldmsg/OldMsgStatCacheRefreshJob.java | 26 ++ .../impl/oldmsg/PersonAndCacheValue.java | 14 + .../cn/axzo/msg/center/mq/MqMessageType.java | 10 +- .../cn/axzo/msg/center/mq/MqProducer.java | 6 +- .../cn/axzo/msg/center/mq/RocketMQConfig.java | 19 ++ .../api/mq/SendMessageRecordMessage.java | 16 + .../center/api/request/GeneralMessageReq.java | 1 - .../msg/center/service/dto/IdentityDTO.java | 13 + 17 files changed, 545 insertions(+), 51 deletions(-) create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/CacheValue.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyFilter.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyInfo.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheRefreshJob.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java create mode 100644 msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java index 2934a2cb..53074767 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java @@ -100,6 +100,12 @@ public class PendingMessageBizConfig { @Getter private boolean formatExecSQL; + @Getter + private int oldMsgStatDataExpireHours = 6; + + @Getter + private int oldMsgStateDataBackendUpdateBatchSize = 100; + public boolean hasMessageDetailStyle(String code) { return name2DetailStyle != null && name2DetailStyle.containsKey(code); } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java index d38f8d34..7e2996ae 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java @@ -7,6 +7,7 @@ import cn.axzo.msg.center.api.enums.MsgRecordTerminalTypeEnum; import cn.axzo.msg.center.api.enums.MsgStateEnum; import cn.axzo.msg.center.api.enums.MsgTypeEnum; import cn.axzo.msg.center.api.enums.ReceiveTypeEnum; +import cn.axzo.msg.center.api.mq.SendMessageRecordMessage; import cn.axzo.msg.center.api.request.CmsMsgQueryReq; import cn.axzo.msg.center.api.request.CmsReadMsgReq; import cn.axzo.msg.center.api.request.GeneralMessageReq; @@ -49,6 +50,9 @@ import cn.axzo.msg.center.domain.request.InsideCmsReadMsgReq; import cn.axzo.msg.center.inside.notices.event.SendMessageEvent; import cn.axzo.msg.center.inside.notices.service.MessageRecordService; import cn.axzo.msg.center.message.service.GeneralMessageMapperService; +import cn.axzo.msg.center.mq.MqMessageRecord; +import cn.axzo.msg.center.mq.MqMessageType; +import cn.axzo.msg.center.mq.MqProducer; import cn.axzo.msg.center.service.dto.IdentifyAndReceiveType; import cn.axzo.msg.center.service.dto.IdentityDTO; import cn.axzo.msg.center.utils.PersonIdentityUtil; @@ -115,6 +119,8 @@ public class MessageRecordServiceImpl implements MessageRecordService { private MessageModuleDao messageModuleDao; @Resource private MessageRouterDao messageRouterDao; + @Resource + private MqProducer mqProducer; /*@Resource private IdentityProfileService identityProfileService;*/ @@ -179,6 +185,11 @@ public class MessageRecordServiceImpl implements MessageRecordService { // 不影响原有消息通道 pushMessages.addAll(messageRecords); }); + SendMessageRecordMessage mqMessage = new SendMessageRecordMessage(); + mqMessage.setRequest(message); + mqProducer.send(MqMessageRecord + .builder(MqMessageType.OLD_MSG_SEND, mqMessage) + .build()); if(pushAthena) { asyncPushAthena(message, messageTemplate.getAudioFileName(), messageModule.getModuleName(), pushMessages); } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/PrivateMessageController.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/PrivateMessageController.java index 687e6687..43a6085e 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/PrivateMessageController.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/PrivateMessageController.java @@ -9,8 +9,10 @@ import cn.axzo.msg.center.inside.notices.service.impl.TodoSearchService; import cn.axzo.msg.center.inside.notices.service.impl.v3.MessageRecordServiceV3; import cn.axzo.msg.center.message.domain.param.PendingMessagePushParam; import cn.axzo.msg.center.message.service.group.GroupTemplateService; +import cn.axzo.msg.center.message.service.impl.oldmsg.OldMsgStatCache; import cn.axzo.msg.center.message.service.todo.manage.TodoManager; import com.google.common.base.Splitter; +import com.google.common.collect.Sets; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -39,6 +41,7 @@ public class PrivateMessageController { private final TodoSearchService todoSearchService; private final TodoManager todoManager; private final GroupTemplateService groupTemplateService; + private final OldMsgStatCache oldMsgStatCache; @PostMapping("/sendPendingMessage") public Object sendPendingMessage(@RequestBody @Valid PendingMessagePushParam request) { @@ -70,6 +73,11 @@ public class PrivateMessageController { return groupTemplateService.collectTemplateCodes(groupNodeCode); } + @PostMapping("/refreshOldMsgStat") + public Object refreshOldMsgStat(@RequestParam("personId") Long personId) throws Exception { + return oldMsgStatCache.refreshBackend(Sets.newHashSet(personId)); + } + @PostMapping("/getPersonIdByPhone") public Object getPersonIdByPhone(@RequestParam("phones") String phones) { List phonesStr = Splitter.on(",") diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageOldServiceImpl.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageOldServiceImpl.java index ebb6b690..d478dd35 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageOldServiceImpl.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageOldServiceImpl.java @@ -16,8 +16,11 @@ import cn.axzo.msg.center.inside.notices.service.MessageModuleService; import cn.axzo.msg.center.inside.notices.service.MessageRelationService; import cn.axzo.msg.center.message.service.GeneralMessageOldService; import cn.axzo.msg.center.message.service.MessageSendTwiceRecordService; +import cn.axzo.msg.center.message.service.impl.oldmsg.OldMsgStatCache; import cn.axzo.msg.center.service.dto.IdentityDTO; import cn.axzo.msg.center.service.dto.PersonDTO; +import cn.axzo.msg.center.service.general.request.GeneralMessageOldDataStatisticRequest; +import cn.axzo.msg.center.service.general.response.GeneralMessageOldDataStatisticResponse; import cn.axzo.msg.center.utils.PersonIdentityUtil; import cn.azxo.framework.common.model.Page; import com.alibaba.fastjson.JSON; @@ -26,7 +29,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; -import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -45,6 +47,7 @@ public class GeneralMessageOldServiceImpl implements GeneralMessageOldService { private final MessageModuleService messageModuleService; private final MessageRelationService messageRelationService; private final MessageSendTwiceRecordService messageSendTwiceRecordService; + private final OldMsgStatCache oldMsgStatCache; @Override public int countUnread(PersonDTO person) { @@ -58,16 +61,11 @@ public class GeneralMessageOldServiceImpl implements GeneralMessageOldService { public int countUnreadWithIdentities(Long personId, List identities) { log.info("GeneralMessageOldServiceImpl#countUnreadWithIdentities. personId={}, identifies={}", personId, JSON.toJSONString(identities)); - if (CollectionUtils.isEmpty(identities)) { - return 0; - } - List sendTwiceMsgIds = messageSendTwiceRecordService.listByPerson(personId); - int count = 0; - for (IdentityDTO identity : identities) { - PersonDTO person = PersonDTO.from(personId, identity.getId(), identity.getType()); - count += countUnread(person, sendTwiceMsgIds); - } - return count; + GeneralMessageOldDataStatisticRequest request = new GeneralMessageOldDataStatisticRequest(); + request.setPersonId(personId); + request.setIdentities(identities); + GeneralMessageOldDataStatisticResponse statResp = oldMsgStatCache.getResponseOrRefresh(request); + return statResp == null ? 0 : statResp.getUnreadCount(); } public int countUnread(Long personId, List identities, List excludeMsgIds) { diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java index 58ae9ba6..fb0b6a07 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java @@ -5,8 +5,6 @@ import cn.axzo.im.center.api.feign.MessageApi; import cn.axzo.im.center.api.vo.req.MessageInfo; import cn.axzo.im.center.api.vo.resp.MessageDispatchResp; import cn.axzo.im.center.common.enums.AppTypeEnum; -import cn.axzo.msg.center.api.request.CmsMsgQueryReq; -import cn.axzo.msg.center.api.response.MessageNewRes; import cn.axzo.msg.center.common.exception.ServiceException; import cn.axzo.msg.center.common.utils.PlaceholderResolver; import cn.axzo.msg.center.dal.GeneralMessageRecordDao; @@ -17,21 +15,19 @@ import cn.axzo.msg.center.message.domain.dto.MessageTemplateRouterDTO; import cn.axzo.msg.center.message.domain.dto.SendImMessageDTO; import cn.axzo.msg.center.message.domain.vo.GeneralMessagePushVO; import cn.axzo.msg.center.message.service.GeneralMessageService; -import cn.axzo.msg.center.message.service.MessageSendTwiceRecordService; import cn.axzo.msg.center.message.service.MessageTemplateNewService; +import cn.axzo.msg.center.message.service.impl.oldmsg.OldMsgStatCache; import cn.axzo.msg.center.service.dto.IdentityDTO; import cn.axzo.msg.center.service.dto.MessageCardContentItemDTO; import cn.axzo.msg.center.service.dto.PersonDTO; import cn.axzo.msg.center.service.enums.GeneralMessageStateEnum; import cn.axzo.msg.center.service.enums.IdentityTypeEnum; -import cn.axzo.msg.center.service.enums.MessageCategoryEnum; import cn.axzo.msg.center.service.enums.PushTerminalEnum; import cn.axzo.msg.center.service.general.request.GeneralMessageOldDataStatisticRequest; import cn.axzo.msg.center.service.general.request.GeneralMessageSendRequest; import cn.axzo.msg.center.service.general.response.GeneralMessageOldDataStatisticResponse; import cn.axzo.msg.center.utils.MessageRouterUtil; import cn.axzo.msg.center.utils.UUIDUtil; -import cn.azxo.framework.common.model.Page; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.ImmutableMap; @@ -48,7 +44,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.stream.Collectors; /** @@ -74,9 +69,8 @@ public class GeneralMessageServiceImpl implements GeneralMessageService { private final MessageApi messageApi; private final MessageSystemConfig messageSystemConfig; private final GeneralMessageRecordDao generalMessageRecordDao; - private final GeneralMessageOldServiceImpl generalMessageOldService; private final MessageTemplateNewService messageTemplateNewService; - private final MessageSendTwiceRecordService messageSendTwiceRecordService; + private final OldMsgStatCache oldMsgStatCache; @Override @Transactional(rollbackFor = Exception.class, propagation = Propagation.REQUIRES_NEW) @@ -93,20 +87,7 @@ public class GeneralMessageServiceImpl implements GeneralMessageService { @Override public GeneralMessageOldDataStatisticResponse statisticOldData(GeneralMessageOldDataStatisticRequest request) { - // 查询双发的消息记录 - List sendTwiceMsgIds = messageSendTwiceRecordService.listByPerson(request.getPersonId()); - // 分页查询最新一条数据 - Page result = generalMessageOldService.pageMsgInfo(build(request, sendTwiceMsgIds)); - // 统计旧的未读普通消息数量 - int count = generalMessageOldService.countUnread( - request.getPersonId(), request.determineIdentities(), sendTwiceMsgIds); - // 编排组合成界面展示的数据结构 - MessageNewRes msg = CollectionUtils.isNotEmpty(result.getList()) ? result.getList().get(0) : null; - return GeneralMessageOldDataStatisticResponse.builder() - .unreadCount(count) - .latestMsgSendTimestamp(Optional.ofNullable(msg).map(v -> v.getCreateAt().getTime()).orElse(null)) - .latestMsgContent(Optional.ofNullable(msg).map(MessageNewRes::getContent).orElse(null)) - .build(); + return oldMsgStatCache.getResponseOrRefresh(request); } private List buildMessageRecord(GeneralMessageSendRequest request, MessageTemplateDTO template) { @@ -207,16 +188,4 @@ public class GeneralMessageServiceImpl implements GeneralMessageService { return PlaceholderResolver.getDefaultResolver().resolveByMap(string, params); } - private CmsMsgQueryReq build(GeneralMessageOldDataStatisticRequest request, List excludeMsgIds) { - CmsMsgQueryReq req = new CmsMsgQueryReq(); - req.setMsgType(MessageCategoryEnum.GENERAL_MESSAGE.getCode()); - // 这里查询消息中心全部状态的数据 - req.setMsgStatus(0); - req.setPage(1L); - req.setPageSize(1L); - req.setPersonId(request.getPersonId()); - req.setIdentities(request.determineIdentities()); - req.setExcludeMsgIds(excludeMsgIds); - return req; - } } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/CacheValue.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/CacheValue.java new file mode 100644 index 00000000..0894aab3 --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/CacheValue.java @@ -0,0 +1,48 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import cn.axzo.msg.center.service.dto.IdentityDTO; +import cn.axzo.msg.center.service.general.response.GeneralMessageOldDataStatisticResponse; +import lombok.Data; +import org.apache.commons.collections.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * @author yanglin + */ +@Data +public class CacheValue { + + private List identityResponses; + + public Optional findResponse(Collection identities) { + if (CollectionUtils.isEmpty(identityResponses)) + return Optional.empty(); + return identityResponses.stream() + .filter(response -> response.isIdentitiesMatch(identities)) + .map(IdentityResponse::getResponse) + .findFirst(); + } + + public void addIdentityResponse(IdentityResponse identityResponse) { + if (identityResponses == null) + identityResponses = new ArrayList<>(); + identityResponses.add(identityResponse); + } + + @Data + public static class IdentityResponse { + private GeneralMessageOldDataStatisticResponse response; + private Set identities; + + public boolean isIdentitiesMatch(Collection identities) { + // 比较时忽略顺序 + return this.identities.equals(new HashSet<>(identities)); + } + } +} diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyFilter.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyFilter.java new file mode 100644 index 00000000..998f7611 --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyFilter.java @@ -0,0 +1,52 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import cn.axzo.basics.profiles.api.UserProfileServiceApi; +import cn.axzo.basics.profiles.dto.basic.IdentityProfileDto; +import cn.axzo.msg.center.common.utils.BizAssertions; +import cn.axzo.msg.center.service.dto.IdentityDTO; +import cn.axzo.msg.center.service.enums.IdentityTypeEnum; +import com.google.common.collect.ImmutableSet; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * @author yanglin + */ +@Component +@RequiredArgsConstructor +public class OldMsgIdentifyFilter { + + private final UserProfileServiceApi userProfileServiceApi; + + public OldMsgIdentifyInfo getIdentifyInfo(Long personId, boolean asManager) { + Set identifiesFilter; + // cms或者管理端 + if (asManager) { + identifiesFilter = ImmutableSet.of(IdentityTypeEnum.PRACTITIONER, IdentityTypeEnum.WORKER_LEADER); + } + // 工人端 + else { + identifiesFilter = ImmutableSet.of(IdentityTypeEnum.WORKER); + } + List profiles = BizAssertions.assertResponse(userProfileServiceApi + .getPersonIdentityProfile(personId), "获取身份信息失败, personId={}", personId); + ArrayList identities = new ArrayList<>(); + for (IdentityProfileDto profile : profiles) { + IdentityTypeEnum identityType = IdentityTypeEnum + .codeOf(profile.getIdentityType().getCode()) + .orElse(null); + if (identityType == null || !identifiesFilter.contains(identityType)) { + continue; + } + IdentityDTO identify = new IdentityDTO(); + identify.setId(profile.getId()); + identify.setType(identityType); + identities.add(identify); + } + return new OldMsgIdentifyInfo(personId, identities); + } +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyInfo.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyInfo.java new file mode 100644 index 00000000..dddcddff --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyInfo.java @@ -0,0 +1,17 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import cn.axzo.msg.center.service.dto.IdentityDTO; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.List; + +/** + * @author yanglin + */ +@Getter +@RequiredArgsConstructor +public class OldMsgIdentifyInfo { + private final Long personId; + private final List identities; +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java new file mode 100644 index 00000000..bdc835f7 --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -0,0 +1,292 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import cn.axzo.basics.profiles.api.UserProfileServiceApi; +import cn.axzo.basics.profiles.common.enums.IdentityType; +import cn.axzo.basics.profiles.dto.basic.BasicDto; +import cn.axzo.basics.profiles.dto.basic.IdentityProfileDto; +import cn.axzo.framework.rocketmq.Event; +import cn.axzo.framework.rocketmq.EventConsumer; +import cn.axzo.framework.rocketmq.EventHandler; +import cn.axzo.msg.center.api.mq.SendMessageRecordMessage; +import cn.axzo.msg.center.api.request.CmsMsgQueryReq; +import cn.axzo.msg.center.api.request.GeneralMessageReq; +import cn.axzo.msg.center.api.response.MessageNewRes; +import cn.axzo.msg.center.inside.notices.config.PendingMessageBizConfig; +import cn.axzo.msg.center.message.service.MessageSendTwiceRecordService; +import cn.axzo.msg.center.message.service.impl.GeneralMessageOldServiceImpl; +import cn.axzo.msg.center.message.service.impl.oldmsg.CacheValue.IdentityResponse; +import cn.axzo.msg.center.mq.MqMessageType; +import cn.axzo.msg.center.notices.common.enums.ReturnCodeEnum; +import cn.axzo.msg.center.notices.common.exception.BizException; +import cn.axzo.msg.center.service.dto.IdentityDTO; +import cn.axzo.msg.center.service.enums.MessageCategoryEnum; +import cn.axzo.msg.center.service.general.request.GeneralMessageOldDataStatisticRequest; +import cn.axzo.msg.center.service.general.response.GeneralMessageOldDataStatisticResponse; +import cn.azxo.framework.common.model.CommonResponse; +import cn.azxo.framework.common.model.Page; +import com.alibaba.fastjson.JSON; +import com.google.common.collect.Lists; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.context.ApplicationContext; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import static java.util.stream.Collectors.toSet; + +/** + * @author yanglin + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class OldMsgStatCache implements EventHandler, InitializingBean { + + private final ApplicationContext applicationContext; + private final MessageSendTwiceRecordService messageSendTwiceRecordService; + private final OldMsgIdentifyFilter oldMsgIdentifyFilter; + private final StringRedisTemplate stringRedisTemplate; + private final PendingMessageBizConfig cfg; + private final EventConsumer eventConsumer; + private final UserProfileServiceApi userProfileServiceApi; + private final ForkJoinPool foregroundExecutor = new ForkJoinPool(20); + private final ForkJoinPool backgroundExecutor = new ForkJoinPool(20); + + public GeneralMessageOldDataStatisticResponse getResponseOrRefresh(GeneralMessageOldDataStatisticRequest request) { + GeneralMessageOldDataStatisticResponse resp = getCacheResponse(request); + return resp != null ? resp : refreshFrontend(request); + } + + private GeneralMessageOldDataStatisticResponse getCacheResponse(GeneralMessageOldDataStatisticRequest request) { + CacheValue cacheValue = getCacheValue(request.getPersonId()); + if (cacheValue == null) { + log.info("oldMsgStat: Can't fnd stat data in cache, try to refresh. request={}", request); + return null; + } + GeneralMessageOldDataStatisticResponse response = cacheValue + .findResponse(request.getIdentities()) + .orElse(null); + if (response != null) + return response; + // 身份发生了变化, 而且异步刷新不及时 + log.info("oldMsgStat: cached data may be expired. Need to refresh. request={}", request); + return null; + } + + private GeneralMessageOldDataStatisticResponse refreshFrontend(GeneralMessageOldDataStatisticRequest request) { + log.info("oldMsgStat: refresh cache for request. request={}", request); + CacheValue cacheValue = initPersonCacheValueFrontend(request.getPersonId()); + GeneralMessageOldDataStatisticResponse response = cacheValue + .findResponse(request.getIdentities()) + .orElse(null); + // 边界 + if (response == null) { + IdentityResponse newIdentityResponse = getIdentityResponse(request.getPersonId(), request.getIdentities()); + cacheValue.addIdentityResponse(newIdentityResponse); + } + setCacheValue(request.getPersonId(), cacheValue); + return cacheValue + .findResponse(request.getIdentities()) + // should never happen + .orElse(null); + } + + private CacheValue initPersonCacheValueFrontend(Long personId) { + Function buildResponseFun = asManager -> { + OldMsgIdentifyInfo identifyInfo = oldMsgIdentifyFilter.getIdentifyInfo(personId, asManager); + return getIdentityResponse(personId, identifyInfo.getIdentities()); + }; + CacheValue cacheValue = new CacheValue(); + CompletableFuture asManagerFuture = CompletableFuture + .supplyAsync(() -> buildResponseFun.apply(true), foregroundExecutor); + CompletableFuture asWorkerFuture = CompletableFuture + .supplyAsync(() -> buildResponseFun.apply(false), foregroundExecutor); + CompletableFuture.allOf(asManagerFuture, asWorkerFuture).join(); + try { + cacheValue.addIdentityResponse(asManagerFuture.get()); + cacheValue.addIdentityResponse(asWorkerFuture.get()); + } catch (Exception e) { + log.warn("oldMsgStat: fail to stat old msg. personId={}", personId, e); + throw new BizException(ReturnCodeEnum.SYSTEM_ERROR.getCode(), e); + } + return cacheValue; + } + + public Map refreshBackend(Set personIds) throws Exception { + log.info("oldMsgStat: refresh cache for person in background. personIds={}", personIds); + if (CollectionUtils.isEmpty(personIds)) { + log.info("oldMsgStat: personIds is empty"); + return Collections.emptyMap(); + } + HashMap personId2CacheValue = new HashMap<>(); + // 温柔点, 别一次性全丢到线程池 + int bachSize = cfg.getOldMsgStateDataBackendUpdateBatchSize(); + for (List batch : Lists.partition(new ArrayList<>(personIds), bachSize)) { + ArrayList> futures = new ArrayList<>(); + for (Long personId : batch) { + futures.add(CompletableFuture.supplyAsync(() -> { + CacheValue cacheValue = initPersonCacheValueBackend(personId); + setCacheValue(personId, cacheValue); + return new PersonAndCacheValue(personId, cacheValue); + }, backgroundExecutor)); + } + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + for (CompletableFuture future : futures) { + PersonAndCacheValue tmp = future.get(); + personId2CacheValue.put(tmp.getPersonId(), tmp.getCacheValue()); + } + } + return personId2CacheValue; + } + + private CacheValue initPersonCacheValueBackend(Long personId) { + Function buildResponseFun = asManager -> { + OldMsgIdentifyInfo identifyInfo = oldMsgIdentifyFilter.getIdentifyInfo(personId, asManager); + return getIdentityResponse(personId, identifyInfo.getIdentities()); + }; + CacheValue cacheValue = new CacheValue(); + cacheValue.addIdentityResponse(buildResponseFun.apply(true)); + cacheValue.addIdentityResponse(buildResponseFun.apply(false)); + return cacheValue; + } + + private IdentityResponse getIdentityResponse(Long personId, List identities) { + GeneralMessageOldDataStatisticRequest request = new GeneralMessageOldDataStatisticRequest(); + request.setPersonId(personId); + request.setIdentities(identities); + GeneralMessageOldDataStatisticResponse response = statisticOldDataImpl(request); + IdentityResponse identityResponse = new IdentityResponse(); + identityResponse.setResponse(response); + identityResponse.setIdentities(new HashSet<>(identities)); + return identityResponse; + } + + // !! cache helper + + private void setCacheValue(Long personId, CacheValue cacheValue) { + long secondsDelta = TimeUnit.HOURS.toSeconds(cfg.getOldMsgStatDataExpireHours()); + // with random seconds delta, 0-600 seconds + secondsDelta += new SecureRandom().nextInt(10 * 60); + + String redisKey = buildCacheKey(personId); + String redisValue = JSON.toJSONString(cacheValue); + stringRedisTemplate.opsForValue().set(redisKey, redisValue, secondsDelta, TimeUnit.SECONDS); + } + + private CacheValue getCacheValue(Long personId) { + String redisKey = buildCacheKey(personId); + String json = stringRedisTemplate.opsForValue().get(redisKey); + return StringUtils.isBlank(json) ? null : JSON.parseObject(json, CacheValue.class); + } + + private static String buildCacheKey(Long personId) { + return String.format("msg_center:old_msg_stat:v1:%d", personId); + } + + // !! 以下为原有的统计逻辑 + + private GeneralMessageOldDataStatisticResponse statisticOldDataImpl(GeneralMessageOldDataStatisticRequest request) { + if (request.determineIdentities().isEmpty()) { + return GeneralMessageOldDataStatisticResponse.builder() + .unreadCount(0) + .latestMsgSendTimestamp(null) + .latestMsgContent(null) + .build(); + } + // 查询双发的消息记录 + List sendTwiceMsgIds = messageSendTwiceRecordService.listByPerson(request.getPersonId()); + // 打破循环依赖, 避免大量改造 (反正快下线了) + GeneralMessageOldServiceImpl generalMessageOldService = applicationContext + .getBean(GeneralMessageOldServiceImpl.class); + // 分页查询最新一条数据 + Page result = generalMessageOldService.pageMsgInfo(build(request, sendTwiceMsgIds)); + // 统计旧的未读普通消息数量 + int count = generalMessageOldService.countUnread( + request.getPersonId(), request.determineIdentities(), sendTwiceMsgIds); + // 编排组合成界面展示的数据结构 + MessageNewRes msg = CollectionUtils.isNotEmpty(result.getList()) ? result.getList().get(0) : null; + return GeneralMessageOldDataStatisticResponse.builder() + .unreadCount(count) + .latestMsgSendTimestamp(Optional.ofNullable(msg).map(v -> v.getCreateAt().getTime()).orElse(null)) + .latestMsgContent(Optional.ofNullable(msg).map(MessageNewRes::getContent).orElse(null)) + .build(); + } + + private CmsMsgQueryReq build(GeneralMessageOldDataStatisticRequest request, List excludeMsgIds) { + CmsMsgQueryReq req = new CmsMsgQueryReq(); + req.setMsgType(MessageCategoryEnum.GENERAL_MESSAGE.getCode()); + // 这里查询消息中心全部状态的数据 + req.setMsgStatus(0); + req.setPage(1L); + req.setPageSize(1L); + req.setPersonId(request.getPersonId()); + req.setIdentities(request.determineIdentities()); + req.setExcludeMsgIds(excludeMsgIds); + return req; + } + + @Override + public void afterPropertiesSet() { + eventConsumer.registerHandler(MqMessageType.OLD_MSG_SEND.getEventCode(), this); + } + + @Override + public void onEvent(Event event, EventConsumer.Context context) { + log.info("oldMsgStat: start - handle mq event, event={}", JSON.toJSONString(event)); + try { + long start = System.currentTimeMillis(); + handleMqMessage(event); + long end = System.currentTimeMillis(); + log.info("oldMsgStat: end - handle mq event, used={}ms, event={}", end - start, JSON.toJSONString(event)); + } catch (Exception e) { + log.warn("oldMsgStat: error - handle mq event, event={}", JSON.toJSONString(event)); + } + } + + private void handleMqMessage(Event event) throws Exception { + SendMessageRecordMessage mqMessage = event.normalizedData(SendMessageRecordMessage.class); + GeneralMessageReq request = mqMessage.getRequest(); + Map identityId2PersonId = request.getToldIdPersonIdMap(); + if (identityId2PersonId == null) + identityId2PersonId = new HashMap<>(); + HashSet personIds = new HashSet<>(identityId2PersonId.values()); + Set identityIds = request.getToId(); + if (identityIds == null) + identityIds = Collections.emptySet(); + HashSet toQueryPersonIdIdentityIds = new HashSet<>(); + for (Long identityId : identityIds) { + if (!identityId2PersonId.containsKey(identityId)) + toQueryPersonIdIdentityIds.add(identityId2PersonId.get(identityId)); + } + if (!toQueryPersonIdIdentityIds.isEmpty()) { + CommonResponse> identitiesByIdSet = userProfileServiceApi.getIdentitiesByIdSet( + new ArrayList<>(toQueryPersonIdIdentityIds), IdentityType.NOT_SUPPORT.getCode()); + log.info("oldMsgStat: query person ids by identityIds. toQueryPersonIdIdentityIds={}, response={}", + JSON.toJSONString(toQueryPersonIdIdentityIds), JSON.toJSONString(identitiesByIdSet)); + Set queriedPersonIds = ListUtils.emptyIfNull(identitiesByIdSet.getData()).stream() + .map(BasicDto::getId) + .collect(toSet()); + personIds.addAll(queriedPersonIds); + } + refreshBackend(personIds); + } + +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheRefreshJob.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheRefreshJob.java new file mode 100644 index 00000000..3973c1a4 --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheRefreshJob.java @@ -0,0 +1,26 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import com.xxl.job.core.biz.model.ReturnT; +import com.xxl.job.core.handler.IJobHandler; +import com.xxl.job.core.handler.annotation.XxlJob; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +/** + * @author yanglin + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class OldMsgStatCacheRefreshJob extends IJobHandler { + + private final OldMsgStatCache oldMsgStatCache; + + @Override + @XxlJob("oldMsgStatCacheRefreshJob") + public ReturnT execute(String param) throws Exception { + return null; + } + +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java new file mode 100644 index 00000000..7ea644b0 --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java @@ -0,0 +1,14 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * @author yanglin + */ +@Getter +@RequiredArgsConstructor +public class PersonAndCacheValue { + private final Long personId; + private final CacheValue cacheValue; +} diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/mq/MqMessageType.java b/inside-notices/src/main/java/cn/axzo/msg/center/mq/MqMessageType.java index efc77d19..32476fdb 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/mq/MqMessageType.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/mq/MqMessageType.java @@ -1,5 +1,6 @@ package cn.axzo.msg.center.mq; +import cn.axzo.framework.rocketmq.Event; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -12,9 +13,16 @@ import lombok.RequiredArgsConstructor; public enum MqMessageType { TODO_PRESET_BUTTON_PRESSED("msg-center-todo", "msg-center-todo-preset-button-pressed", "预设按钮被点击"), + OLD_MSG_SEND("msg-center-old-msg", "old-msg-send", "发送旧消息"); - ; private final String model; private final String tag; private final String desc; + + public Event.EventCode getEventCode() { + return Event.EventCode.builder() + .module(model) + .name(tag) + .build(); + } } \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/mq/MqProducer.java b/inside-notices/src/main/java/cn/axzo/msg/center/mq/MqProducer.java index 8414e5b7..9d6537df 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/mq/MqProducer.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/mq/MqProducer.java @@ -1,7 +1,6 @@ package cn.axzo.msg.center.mq; import cn.axzo.framework.rocketmq.Event; -import cn.axzo.framework.rocketmq.Event.EventCode; import cn.axzo.framework.rocketmq.EventProducer; import cn.axzo.msg.center.api.mq.MqMessage; import cn.axzo.msg.center.common.utils.BizAssertions; @@ -25,12 +24,11 @@ public class MqProducer { log.info("开始 - 发送消息. messageRecord={}", record); try { sendImpl(record); + log.info("结束 - 发送消息. messageRecord={}", record); } catch (Exception e) { log.warn("异常 - 发送消息. messageRecord={}", record, e); // 由调用方决定怎么处理异常 throw e; - } finally { - log.info("结束 - 发送消息. messageRecord={}", record); } } @@ -40,7 +38,7 @@ public class MqProducer { .shardingKey(record.getShardingKey()) .targetId(record.getTargetId()) .targetType(messageType.getModel()) - .eventCode(new EventCode(messageType.getModel(), messageType.getTag())) + .eventCode(messageType.getEventCode()) .eventModule(messageType.getModel()) .eventName(messageType.getTag()) .operatorId(record.getOperatorId()) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/mq/RocketMQConfig.java b/inside-notices/src/main/java/cn/axzo/msg/center/mq/RocketMQConfig.java index 581eab00..3e4622a9 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/mq/RocketMQConfig.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/mq/RocketMQConfig.java @@ -9,6 +9,7 @@ import cn.axzo.framework.rocketmq.EventProducer.Context; import cn.axzo.framework.rocketmq.RocketMQEventProducer; import cn.axzo.framework.rocketmq.RocketMQEventProducer.RocketMQMessageMeta; import cn.axzo.msg.center.api.mq.MqMessage; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.spring.annotation.ConsumeMode; @@ -77,6 +78,24 @@ public class RocketMQConfig { } } + @Slf4j + @Component + @RequiredArgsConstructor + @RocketMQMessageListener(topic = "topic_msg_center_${spring.profiles.active}", + consumerGroup = "GID_topic_msg_center_self_consume_${spring.profiles.active}", + consumeMode = ConsumeMode.ORDERLY, + nameServer = "${rocketmq.name-server}" + ) + public static class MsgCenterListener extends BaseListener implements RocketMQListener { + + private final EventConsumer eventConsumer; + + @Override + public void onMessage(MessageExt message) { + super.onEvent(message, eventConsumer); + } + } + @Bean EventHandlerRepository eventHandlerRepository() { return new EventHandlerRepository((ex, logText) -> { diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java b/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java new file mode 100644 index 00000000..6b94c4a4 --- /dev/null +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java @@ -0,0 +1,16 @@ +package cn.axzo.msg.center.api.mq; + +import cn.axzo.msg.center.api.request.GeneralMessageReq; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.io.Serializable; + +/** + * @author yanglin + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class SendMessageRecordMessage extends MqMessage implements Serializable { + private GeneralMessageReq request; +} \ No newline at end of file diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/GeneralMessageReq.java b/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/GeneralMessageReq.java index 351d818b..430554af 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/GeneralMessageReq.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/GeneralMessageReq.java @@ -2,7 +2,6 @@ package cn.axzo.msg.center.api.request; import cn.axzo.msg.center.api.enums.MsgRecordTerminalTypeEnum; import cn.axzo.msg.center.api.enums.ReceiveTypeEnum; -import com.alibaba.fastjson.JSONObject; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/service/dto/IdentityDTO.java b/msg-center-api/src/main/java/cn/axzo/msg/center/service/dto/IdentityDTO.java index 3eb6ec55..372aff7d 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/service/dto/IdentityDTO.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/service/dto/IdentityDTO.java @@ -40,6 +40,19 @@ public class IdentityDTO implements Serializable { return Objects.nonNull(id) && Objects.nonNull(type); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IdentityDTO that = (IdentityDTO) o; + return Objects.equals(id, that.id) && type == that.type; + } + + @Override + public int hashCode() { + return Objects.hash(id, type); + } + @Override public String toString() { return JSON.toJSONString(this); From 6cd89df8f3f1be5b5d5172f66686592998181115 Mon Sep 17 00:00:00 2001 From: yanglin Date: Wed, 24 Apr 2024 17:58:14 +0800 Subject: [PATCH 02/33] =?UTF-8?q?REQ-2405:=20=E9=80=9A=E8=BF=87job?= =?UTF-8?q?=E6=AF=8F=E5=A4=A9=E6=99=9A=E4=B8=8A=E5=88=B7=E6=96=B0=E8=80=81?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E7=9A=84=E7=BB=9F=E8=AE=A1=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/PendingMessageBizConfig.java | 3 + .../controller/GeneralMessageController.java | 1 + .../controller/PrivateMessageController.java | 6 +- .../impl/GeneralMessageOldServiceImpl.java | 5 +- .../impl/GeneralMessageServiceImpl.java | 2 +- .../oldmsg/OldMsgCacheReloadJobParam.java | 38 +++++ .../service/impl/oldmsg/OldMsgStatCache.java | 134 +++++++++--------- .../oldmsg/OldMsgStatCacheRefreshJob.java | 26 ---- .../impl/oldmsg/OldMsgStatCacheReloadJob.java | 100 +++++++++++++ .../service/impl/person/PersonIdInfo.java | 25 ++++ .../service/impl/person/PersonService.java | 47 ++++++ .../cn/axzo/msg/center/mq/RocketMQConfig.java | 38 ++++- .../center/api/request/CmsMsgQueryReq.java | 1 + .../dal/mapper/MessageRecordMapper.java | 5 +- .../axzo/msg/center/util/IterableMapper.java | 23 +++ .../axzo/msg/center/util/RecordIterable.java | 105 ++++++++++++++ .../oldmsg/OldMsgStatCacheReloadJobTest.java | 24 ++++ 17 files changed, 483 insertions(+), 100 deletions(-) create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgCacheReloadJobParam.java delete mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheRefreshJob.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/person/PersonIdInfo.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/person/PersonService.java create mode 100644 msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java create mode 100644 msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java create mode 100644 start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java index 53074767..b0912365 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java @@ -106,6 +106,9 @@ public class PendingMessageBizConfig { @Getter private int oldMsgStateDataBackendUpdateBatchSize = 100; + @Getter + private long msgCenterMqSelfConsumeMaxExecMs = 20000L; + public boolean hasMessageDetailStyle(String code) { return name2DetailStyle != null && name2DetailStyle.containsKey(code); } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/GeneralMessageController.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/GeneralMessageController.java index 764f1013..874f6098 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/GeneralMessageController.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/GeneralMessageController.java @@ -50,6 +50,7 @@ public class GeneralMessageController implements GeneralMessageClient { @Override public CommonResponse> pageQueryOldMessage(CmsMsgQueryReq request) { + request.setLogRequest(true); return CommonResponse.success(generalMessageOldService.pageMsgInfo(request)); } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/PrivateMessageController.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/PrivateMessageController.java index 43a6085e..e321f8d6 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/PrivateMessageController.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/controller/PrivateMessageController.java @@ -73,9 +73,9 @@ public class PrivateMessageController { return groupTemplateService.collectTemplateCodes(groupNodeCode); } - @PostMapping("/refreshOldMsgStat") - public Object refreshOldMsgStat(@RequestParam("personId") Long personId) throws Exception { - return oldMsgStatCache.refreshBackend(Sets.newHashSet(personId)); + @PostMapping("/reloadOldMsgStat") + public Object reloadOldMsgStat(@RequestParam("personId") Long personId) throws Exception { + return oldMsgStatCache.reloadBackground(Sets.newHashSet(personId)); } @PostMapping("/getPersonIdByPhone") diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageOldServiceImpl.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageOldServiceImpl.java index d478dd35..3ac53023 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageOldServiceImpl.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageOldServiceImpl.java @@ -64,7 +64,7 @@ public class GeneralMessageOldServiceImpl implements GeneralMessageOldService { GeneralMessageOldDataStatisticRequest request = new GeneralMessageOldDataStatisticRequest(); request.setPersonId(personId); request.setIdentities(identities); - GeneralMessageOldDataStatisticResponse statResp = oldMsgStatCache.getResponseOrRefresh(request); + GeneralMessageOldDataStatisticResponse statResp = oldMsgStatCache.getCacheResponseOrReload(request); return statResp == null ? 0 : statResp.getUnreadCount(); } @@ -102,7 +102,8 @@ public class GeneralMessageOldServiceImpl implements GeneralMessageOldService { @Override public Page pageMsgInfo(CmsMsgQueryReq request) { - log.info("GeneralMessageOldServiceImpl#pageMsgInfo. request:{}", request); + if (request.isLogRequest()) + log.info("GeneralMessageOldServiceImpl#pageMsgInfo. request:{}", request); if (CollectionUtils.isEmpty(request.determineIdentities())) { return Page.zero(); } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java index fb0b6a07..409a9055 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java @@ -87,7 +87,7 @@ public class GeneralMessageServiceImpl implements GeneralMessageService { @Override public GeneralMessageOldDataStatisticResponse statisticOldData(GeneralMessageOldDataStatisticRequest request) { - return oldMsgStatCache.getResponseOrRefresh(request); + return oldMsgStatCache.getCacheResponseOrReload(request); } private List buildMessageRecord(GeneralMessageSendRequest request, MessageTemplateDTO template) { diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgCacheReloadJobParam.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgCacheReloadJobParam.java new file mode 100644 index 00000000..af5ac67f --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgCacheReloadJobParam.java @@ -0,0 +1,38 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import com.alibaba.fastjson.JSON; +import lombok.Data; + +/** + * @author yanglin + */ +@Data +public class OldMsgCacheReloadJobParam { + + private static final int DEFAULT_SYNC_DAY = 60; + private static final int DEFAULT_BATCH_SIZE = 1000; + + private int syncDay; + private int batchSize; + + static OldMsgCacheReloadJobParam defaultParam() { + OldMsgCacheReloadJobParam defaultParam = new OldMsgCacheReloadJobParam(); + defaultParam.setSyncDay(DEFAULT_SYNC_DAY); + defaultParam.setBatchSize(DEFAULT_BATCH_SIZE); + return defaultParam; + } + + int determineSyncDay() { + return syncDay <= 0 ? DEFAULT_SYNC_DAY : syncDay; + } + + int determineBatchSize() { + return batchSize <= 0 ? DEFAULT_BATCH_SIZE : batchSize; + } + + @Override + public String toString() { + return JSON.toJSONString(this); + } + +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index bdc835f7..a53ae917 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -1,9 +1,5 @@ package cn.axzo.msg.center.message.service.impl.oldmsg; -import cn.axzo.basics.profiles.api.UserProfileServiceApi; -import cn.axzo.basics.profiles.common.enums.IdentityType; -import cn.axzo.basics.profiles.dto.basic.BasicDto; -import cn.axzo.basics.profiles.dto.basic.IdentityProfileDto; import cn.axzo.framework.rocketmq.Event; import cn.axzo.framework.rocketmq.EventConsumer; import cn.axzo.framework.rocketmq.EventHandler; @@ -15,6 +11,8 @@ import cn.axzo.msg.center.inside.notices.config.PendingMessageBizConfig; import cn.axzo.msg.center.message.service.MessageSendTwiceRecordService; import cn.axzo.msg.center.message.service.impl.GeneralMessageOldServiceImpl; import cn.axzo.msg.center.message.service.impl.oldmsg.CacheValue.IdentityResponse; +import cn.axzo.msg.center.message.service.impl.person.PersonIdInfo; +import cn.axzo.msg.center.message.service.impl.person.PersonService; import cn.axzo.msg.center.mq.MqMessageType; import cn.axzo.msg.center.notices.common.enums.ReturnCodeEnum; import cn.axzo.msg.center.notices.common.exception.BizException; @@ -22,14 +20,13 @@ import cn.axzo.msg.center.service.dto.IdentityDTO; import cn.axzo.msg.center.service.enums.MessageCategoryEnum; import cn.axzo.msg.center.service.general.request.GeneralMessageOldDataStatisticRequest; import cn.axzo.msg.center.service.general.response.GeneralMessageOldDataStatisticResponse; -import cn.azxo.framework.common.model.CommonResponse; import cn.azxo.framework.common.model.Page; +import cn.hutool.core.date.StopWatch; import com.alibaba.fastjson.JSON; import com.google.common.collect.Lists; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.InitializingBean; import org.springframework.context.ApplicationContext; @@ -50,8 +47,6 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import static java.util.stream.Collectors.toSet; - /** * @author yanglin */ @@ -66,19 +61,19 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { private final StringRedisTemplate stringRedisTemplate; private final PendingMessageBizConfig cfg; private final EventConsumer eventConsumer; - private final UserProfileServiceApi userProfileServiceApi; + private final PersonService personService; private final ForkJoinPool foregroundExecutor = new ForkJoinPool(20); private final ForkJoinPool backgroundExecutor = new ForkJoinPool(20); - public GeneralMessageOldDataStatisticResponse getResponseOrRefresh(GeneralMessageOldDataStatisticRequest request) { + public GeneralMessageOldDataStatisticResponse getCacheResponseOrReload(GeneralMessageOldDataStatisticRequest request) { GeneralMessageOldDataStatisticResponse resp = getCacheResponse(request); - return resp != null ? resp : refreshFrontend(request); + return resp != null ? resp : reloadForeground(request); } private GeneralMessageOldDataStatisticResponse getCacheResponse(GeneralMessageOldDataStatisticRequest request) { CacheValue cacheValue = getCacheValue(request.getPersonId()); if (cacheValue == null) { - log.info("oldMsgStat: Can't fnd stat data in cache, try to refresh. request={}", request); + log.info("oldMsgStat: cache miss, need to reload in foreground. request={}", request); return null; } GeneralMessageOldDataStatisticResponse response = cacheValue @@ -87,18 +82,24 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { if (response != null) return response; // 身份发生了变化, 而且异步刷新不及时 - log.info("oldMsgStat: cached data may be expired. Need to refresh. request={}", request); + log.info("oldMsgStat: identities cache miss, need to reload in foreground. request={}", request); return null; } - private GeneralMessageOldDataStatisticResponse refreshFrontend(GeneralMessageOldDataStatisticRequest request) { - log.info("oldMsgStat: refresh cache for request. request={}", request); - CacheValue cacheValue = initPersonCacheValueFrontend(request.getPersonId()); + private GeneralMessageOldDataStatisticResponse reloadForeground(GeneralMessageOldDataStatisticRequest request) { + log.info("oldMsgStat: reload cache for request. request={}", request); + StopWatch stopWatch = new StopWatch(String.format( + "%s-reloadForeground", getClass().getSimpleName())); + stopWatch.start("initPersonCacheValueForeground"); + CacheValue cacheValue = initPersonCacheValueForeground(request.getPersonId()); + stopWatch.stop(); + log.info(stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); GeneralMessageOldDataStatisticResponse response = cacheValue .findResponse(request.getIdentities()) .orElse(null); // 边界 if (response == null) { + log.info("oldMsgStat: identities cache missing? reload stat for request identities. request={}", request); IdentityResponse newIdentityResponse = getIdentityResponse(request.getPersonId(), request.getIdentities()); cacheValue.addIdentityResponse(newIdentityResponse); } @@ -109,7 +110,36 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { .orElse(null); } - private CacheValue initPersonCacheValueFrontend(Long personId) { + public Map reloadBackground(Set personIds) throws Exception { + log.info("oldMsgStat: reload cache for person in background. personIds={}", JSON.toJSONString(personIds)); + if (CollectionUtils.isEmpty(personIds)) { + log.info("oldMsgStat: personIds is empty"); + return Collections.emptyMap(); + } + HashMap personId2CacheValue = new HashMap<>(); + // 温柔点, 别一次性全丢到线程池 + int bachSize = cfg.getOldMsgStateDataBackendUpdateBatchSize(); + List> batches = Lists.partition(new ArrayList<>(personIds), bachSize); + for (List batch : batches) { + ArrayList> futures = new ArrayList<>(); + for (Long personId : batch) { + futures.add(CompletableFuture.supplyAsync(() -> { + CacheValue cacheValue = initPersonCacheValueBackground(personId); + setCacheValue(personId, cacheValue); + return new PersonAndCacheValue(personId, cacheValue); + }, backgroundExecutor)); + } + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + for (CompletableFuture future : futures) { + PersonAndCacheValue tmp = future.get(); + personId2CacheValue.put(tmp.getPersonId(), tmp.getCacheValue()); + } + } + return personId2CacheValue; + } + + private CacheValue initPersonCacheValueForeground(Long personId) { + log.info("initPersonCacheValueForeground. personId={}", personId); Function buildResponseFun = asManager -> { OldMsgIdentifyInfo identifyInfo = oldMsgIdentifyFilter.getIdentifyInfo(personId, asManager); return getIdentityResponse(personId, identifyInfo.getIdentities()); @@ -130,34 +160,8 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { return cacheValue; } - public Map refreshBackend(Set personIds) throws Exception { - log.info("oldMsgStat: refresh cache for person in background. personIds={}", personIds); - if (CollectionUtils.isEmpty(personIds)) { - log.info("oldMsgStat: personIds is empty"); - return Collections.emptyMap(); - } - HashMap personId2CacheValue = new HashMap<>(); - // 温柔点, 别一次性全丢到线程池 - int bachSize = cfg.getOldMsgStateDataBackendUpdateBatchSize(); - for (List batch : Lists.partition(new ArrayList<>(personIds), bachSize)) { - ArrayList> futures = new ArrayList<>(); - for (Long personId : batch) { - futures.add(CompletableFuture.supplyAsync(() -> { - CacheValue cacheValue = initPersonCacheValueBackend(personId); - setCacheValue(personId, cacheValue); - return new PersonAndCacheValue(personId, cacheValue); - }, backgroundExecutor)); - } - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); - for (CompletableFuture future : futures) { - PersonAndCacheValue tmp = future.get(); - personId2CacheValue.put(tmp.getPersonId(), tmp.getCacheValue()); - } - } - return personId2CacheValue; - } - - private CacheValue initPersonCacheValueBackend(Long personId) { + private CacheValue initPersonCacheValueBackground(Long personId) { + log.info("initPersonCacheValueBackground. personId={}", personId); Function buildResponseFun = asManager -> { OldMsgIdentifyInfo identifyInfo = oldMsgIdentifyFilter.getIdentifyInfo(personId, asManager); return getIdentityResponse(personId, identifyInfo.getIdentities()); @@ -182,6 +186,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { // !! cache helper private void setCacheValue(Long personId, CacheValue cacheValue) { + log.info("set cache value for person. personId={}", personId); long secondsDelta = TimeUnit.HOURS.toSeconds(cfg.getOldMsgStatDataExpireHours()); // with random seconds delta, 0-600 seconds secondsDelta += new SecureRandom().nextInt(10 * 60); @@ -240,9 +245,12 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { req.setPersonId(request.getPersonId()); req.setIdentities(request.determineIdentities()); req.setExcludeMsgIds(excludeMsgIds); + req.setLogRequest(false); return req; } + // !! 处理mq消息, 异步更新缓存 + @Override public void afterPropertiesSet() { eventConsumer.registerHandler(MqMessageType.OLD_MSG_SEND.getEventCode(), this); @@ -264,29 +272,27 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { private void handleMqMessage(Event event) throws Exception { SendMessageRecordMessage mqMessage = event.normalizedData(SendMessageRecordMessage.class); GeneralMessageReq request = mqMessage.getRequest(); - Map identityId2PersonId = request.getToldIdPersonIdMap(); - if (identityId2PersonId == null) - identityId2PersonId = new HashMap<>(); - HashSet personIds = new HashSet<>(identityId2PersonId.values()); - Set identityIds = request.getToId(); - if (identityIds == null) - identityIds = Collections.emptySet(); + Map knownIdentityId2PersonId = request.getToldIdPersonIdMap(); + if (knownIdentityId2PersonId == null) + knownIdentityId2PersonId = new HashMap<>(); + HashSet personIds = new HashSet<>(knownIdentityId2PersonId.values()); + Set receiverIdentityIds = request.getToId(); + if (receiverIdentityIds == null) + receiverIdentityIds = Collections.emptySet(); HashSet toQueryPersonIdIdentityIds = new HashSet<>(); - for (Long identityId : identityIds) { - if (!identityId2PersonId.containsKey(identityId)) - toQueryPersonIdIdentityIds.add(identityId2PersonId.get(identityId)); + for (Long identityId : receiverIdentityIds) { + if (!knownIdentityId2PersonId.containsKey(identityId)) + toQueryPersonIdIdentityIds.add(identityId); } if (!toQueryPersonIdIdentityIds.isEmpty()) { - CommonResponse> identitiesByIdSet = userProfileServiceApi.getIdentitiesByIdSet( - new ArrayList<>(toQueryPersonIdIdentityIds), IdentityType.NOT_SUPPORT.getCode()); - log.info("oldMsgStat: query person ids by identityIds. toQueryPersonIdIdentityIds={}, response={}", - JSON.toJSONString(toQueryPersonIdIdentityIds), JSON.toJSONString(identitiesByIdSet)); - Set queriedPersonIds = ListUtils.emptyIfNull(identitiesByIdSet.getData()).stream() - .map(BasicDto::getId) - .collect(toSet()); - personIds.addAll(queriedPersonIds); + PersonIdInfo personIdInfo = personService.getPersonIdsByIdentities(toQueryPersonIdIdentityIds); + personIds.addAll(personIdInfo.getPersonIds()); } - refreshBackend(personIds); + StopWatch stopWatch = new StopWatch("reloadBackground driven by mq event"); + stopWatch.start("reloadBackground"); + reloadBackground(personIds); + stopWatch.stop(); + log.info(stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); } } \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheRefreshJob.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheRefreshJob.java deleted file mode 100644 index 3973c1a4..00000000 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheRefreshJob.java +++ /dev/null @@ -1,26 +0,0 @@ -package cn.axzo.msg.center.message.service.impl.oldmsg; - -import com.xxl.job.core.biz.model.ReturnT; -import com.xxl.job.core.handler.IJobHandler; -import com.xxl.job.core.handler.annotation.XxlJob; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -/** - * @author yanglin - */ -@Slf4j -@Component -@RequiredArgsConstructor -public class OldMsgStatCacheRefreshJob extends IJobHandler { - - private final OldMsgStatCache oldMsgStatCache; - - @Override - @XxlJob("oldMsgStatCacheRefreshJob") - public ReturnT execute(String param) throws Exception { - return null; - } - -} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java new file mode 100644 index 00000000..9b7ddd05 --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java @@ -0,0 +1,100 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import cn.axzo.msg.center.dal.mapper.MessageRecordMapper; +import cn.axzo.msg.center.domain.entity.MessageRecord; +import cn.axzo.msg.center.message.service.impl.person.PersonService; +import cn.hutool.core.date.StopWatch; +import com.alibaba.fastjson.JSON; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.xxl.job.core.biz.model.ReturnT; +import com.xxl.job.core.handler.IJobHandler; +import com.xxl.job.core.handler.annotation.XxlJob; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.joda.time.DateTime; +import org.springframework.stereotype.Component; + +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static cn.axzo.msg.center.inside.notices.utils.Queries.query; +import static java.util.stream.Collectors.toSet; + +/** + * @author yanglin + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class OldMsgStatCacheReloadJob extends IJobHandler { + + private final MessageRecordMapper messageRecordMapper; + private final PersonService personService; + private final OldMsgStatCache oldMsgStatCache; + + @Override + @XxlJob("oldMsgStatCacheReloadJob") + public ReturnT execute(String param) throws Exception { + try { + log.info("start - running job using param: {}", param); + executeImpl(param); + log.info("end - running job using param: {}", param); + return ReturnT.SUCCESS; + } catch (Exception e) { + log.warn("run job error, param={}", param, e); + return ReturnT.FAIL; + } + } + + private void executeImpl(String param) throws Exception { + OldMsgCacheReloadJobParam jobParam = StringUtils.isBlank(param) + ? OldMsgCacheReloadJobParam.defaultParam() + : JSON.parseObject(param, OldMsgCacheReloadJobParam.class); + Date sinceDate = DateTime.now().minusDays(jobParam.determineSyncDay()).toDate(); + LambdaQueryWrapper query = query(MessageRecord.class) + .ge(MessageRecord::getCreateAt, sinceDate) + // 选刷最近的 + .orderByDesc(MessageRecord::getId); + StopWatch stopWatch = new StopWatch("reloadBackground driven by job"); + Set reloadedPersonIds = new HashSet<>(); + for (Page page : messageRecordMapper.iterateBath(jobParam.determineBatchSize(), query)) { + stopWatch.start("reload stat, records size=" + page.getRecords().size()); + reloadStat(reloadedPersonIds, page); + stopWatch.stop(); + } + log.info("end sync old msg stat. reloaded personIds size={}", reloadedPersonIds.size()); + log.info(stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); + } + + private void reloadStat(Set reloadedPersonIds, Page page) throws Exception { + log.info("start sync old msg stat. page size={}, page current={}, total pages={}", + page.getSize(), page.getCurrent(), page.getPages()); + List records = page.getRecords(); + Set maybeReloadPersonIds = new HashSet<>(); + // 传了personId的 + records.stream() + .map(MessageRecord::getPersonId) + .filter(personId -> personId != 0L) + .forEach(maybeReloadPersonIds::add); + // 没有传personId的 + Set identityIds = records.stream() + .filter(record -> record.getPersonId() == 0L) + .map(MessageRecord::getToId) + .collect(toSet()); + if (!identityIds.isEmpty()) { + List fetchedPersonIds = personService.getPersonIdsByIdentities(identityIds).getPersonIds(); + maybeReloadPersonIds.addAll(fetchedPersonIds); + } + maybeReloadPersonIds.removeAll(reloadedPersonIds); + if (!maybeReloadPersonIds.isEmpty()) { + oldMsgStatCache.reloadBackground(maybeReloadPersonIds); + reloadedPersonIds.addAll(maybeReloadPersonIds); + } + } + +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/person/PersonIdInfo.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/person/PersonIdInfo.java new file mode 100644 index 00000000..d07e702d --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/person/PersonIdInfo.java @@ -0,0 +1,25 @@ +package cn.axzo.msg.center.message.service.impl.person; + +import cn.axzo.basics.profiles.dto.basic.IdentityProfileDto; +import lombok.RequiredArgsConstructor; + +import java.util.List; + +import static java.util.stream.Collectors.toList; + +/** + * @author yanglin + */ +@RequiredArgsConstructor +public class PersonIdInfo { + + private final List identityProfiles; + + public List getPersonIds() { + return identityProfiles.stream() + .map(i -> i.getPersonProfile().getId()) + .distinct() + .collect(toList()); + + } +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/person/PersonService.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/person/PersonService.java new file mode 100644 index 00000000..fb8c9fd7 --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/person/PersonService.java @@ -0,0 +1,47 @@ +package cn.axzo.msg.center.message.service.impl.person; + +import cn.axzo.basics.profiles.api.UserProfileServiceApi; +import cn.axzo.basics.profiles.common.enums.IdentityType; +import cn.axzo.basics.profiles.dto.basic.IdentityProfileDto; +import cn.axzo.basics.profiles.dto.basic.PersonProfileDto; +import cn.axzo.msg.center.common.utils.BizAssertions; +import cn.azxo.framework.common.model.CommonResponse; +import com.alibaba.fastjson.JSON; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; + +/** + * @author yanglin + */ +@Service +@Slf4j +@RequiredArgsConstructor +public class PersonService { + + private final UserProfileServiceApi userProfileServiceApi; + + public PersonIdInfo getPersonIdsByIdentities(Collection identityIds) { + log.info("start - querying person ids by identity ids: {}", JSON.toJSONString(identityIds)); + ArrayList uniqueIdentityIds = new ArrayList<>(new HashSet<>(identityIds)); + CommonResponse> resp = userProfileServiceApi.getIdentitiesByIdSet( + uniqueIdentityIds, IdentityType.NOT_SUPPORT.getCode()); + log.info("end - querying person ids by identity ids, request={}, response={}", + JSON.toJSONString(identityIds), JSON.toJSONString(resp)); + BizAssertions.assertResponse(resp); + return new PersonIdInfo(resp.getData()); + } + + public Long getPersonIdByPhone(String phone) { + CommonResponse profileResp = userProfileServiceApi + .getUnionPersonProfile(null, phone); + PersonProfileDto personProfile = BizAssertions.assertResponse(profileResp, "未找根据手机找到人员"); + BizAssertions.assertNotNull(personProfile, "未找根据手机找到人员"); + return personProfile.getId(); + } +} diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/mq/RocketMQConfig.java b/inside-notices/src/main/java/cn/axzo/msg/center/mq/RocketMQConfig.java index 3e4622a9..0d62229c 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/mq/RocketMQConfig.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/mq/RocketMQConfig.java @@ -8,20 +8,28 @@ import cn.axzo.framework.rocketmq.EventProducer; import cn.axzo.framework.rocketmq.EventProducer.Context; import cn.axzo.framework.rocketmq.RocketMQEventProducer; import cn.axzo.framework.rocketmq.RocketMQEventProducer.RocketMQMessageMeta; +import cn.axzo.framework.rocketmq.utils.TraceUtils; import cn.axzo.msg.center.api.mq.MqMessage; +import cn.axzo.msg.center.inside.notices.config.PendingMessageBizConfig; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.spring.annotation.ConsumeMode; import org.apache.rocketmq.spring.annotation.RocketMQMessageListener; import org.apache.rocketmq.spring.core.RocketMQListener; import org.apache.rocketmq.spring.core.RocketMQTemplate; +import org.slf4j.MDC; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Component; +import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; /** @@ -89,10 +97,38 @@ public class RocketMQConfig { public static class MsgCenterListener extends BaseListener implements RocketMQListener { private final EventConsumer eventConsumer; + private final PendingMessageBizConfig cfg; @Override public void onMessage(MessageExt message) { - super.onEvent(message, eventConsumer); + onEvent(message, eventConsumer); + } + + /** + * 拷贝下来改一下, 设置最大处理时间 + */ + public void onEvent(MessageExt message, EventConsumer eventConsumer) { + String topic = message.getTopic(); + String value = new String(message.getBody()); + Map headers = message.getProperties(); + MDC.put(TraceUtils.CTX_LOG_ID, headers.get(TraceUtils.TRACE_ID)); + MDC.put(TraceUtils.TRACE_ID, headers.get(TraceUtils.TRACE_ID)); + MDC.put(TraceUtils.TRACE_ID_IN_MDC, headers.get(TraceUtils.TRACE_ID)); + if (log.isDebugEnabled()) { + log.debug("received message, topic={}, headers={}, value={}", topic, headers, value); + } + + //当前消息所在分区的lag. 而不是整个topic + Long partitionLag = Optional.ofNullable(message.getProperties().get(MessageConst.PROPERTY_MAX_OFFSET)) + .map(e -> Long.parseLong(e) - message.getQueueOffset()).orElse(0L); + + eventConsumer.onEvent(value, EventConsumer.Context.builder() + .msgId(message.getMsgId()) + .ext(ImmutableMap.of("topic", topic)) + .headers(Maps.transformValues(headers, header -> Optional.ofNullable(header).map(String::getBytes).orElse(new byte[] {}))) + .lagSupplier(() -> partitionLag) + .maxAllowElapsedMillis(cfg.getMsgCenterMqSelfConsumeMaxExecMs()) + .build()); } } diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/CmsMsgQueryReq.java b/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/CmsMsgQueryReq.java index 08d7c0f7..9fddcde4 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/CmsMsgQueryReq.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/CmsMsgQueryReq.java @@ -107,4 +107,5 @@ public class CmsMsgQueryReq extends PageRequest { @Deprecated private Long identityId; + private boolean logRequest = true; } diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageRecordMapper.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageRecordMapper.java index a98b16c9..8835d2d0 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageRecordMapper.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageRecordMapper.java @@ -9,8 +9,7 @@ import cn.axzo.msg.center.domain.dto.MsgStatisticsDTO; import cn.axzo.msg.center.domain.dto.UpdateReadDTO; import cn.axzo.msg.center.domain.entity.MessageRecord; import cn.axzo.msg.center.service.dto.IdentifyAndReceiveType; -import cn.axzo.msg.center.service.dto.IdentityDTO; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import cn.axzo.msg.center.util.IterableMapper; import com.baomidou.mybatisplus.core.metadata.IPage; import org.apache.ibatis.annotations.Param; @@ -23,7 +22,7 @@ import java.util.List; * @since 2022-03-28 14:59:16 */ /*@Mapper*/ -public interface MessageRecordMapper extends BaseMapper{ +public interface MessageRecordMapper extends IterableMapper { List statisticsMsg(@Param("bizType") Integer bizType, diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java new file mode 100644 index 00000000..45642f7e --- /dev/null +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java @@ -0,0 +1,23 @@ +package cn.axzo.msg.center.util; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +/** + * @author yanglin + */ +public interface IterableMapper extends BaseMapper { + + default Iterable> iterateBath(long pageSize, LambdaQueryWrapper query) { + return RecordIterable.iterateBath(this, pageSize, query); + } + + /** + * 还没有严格验证, 小心使用 + */ + default Iterable iterateElement(long pageSize, LambdaQueryWrapper query) { + return RecordIterable.iterateElement(this, pageSize, query); + } + +} \ No newline at end of file diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java new file mode 100644 index 00000000..cdcb8fbe --- /dev/null +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java @@ -0,0 +1,105 @@ +package cn.axzo.msg.center.util; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * @author yanglin + */ +class RecordIterable { + + static > Iterable> iterateBath( + M mapper, long pageSize, LambdaQueryWrapper query) { + return () -> new BatchIterator<>(mapper, pageSize, query); + } + + static > Iterable iterateElement( + M mapper, long pageSize, LambdaQueryWrapper query) { + return () -> { + BatchIterator batchIterator = new BatchIterator<>(mapper, pageSize, query); + return new ElementIterator<>(batchIterator); + }; + } + + private static class ElementIterator> implements Iterator { + + private final BatchIterator batch; + private Iterator current; + + ElementIterator(BatchIterator batchIterator) { + this.batch = batchIterator; + } + + @Override + public boolean hasNext() { + maybeAdvance(); + return current.hasNext() || batch.hasNext(); + } + + @Override + public T next() { + if (!hasNext()) + throw new NoSuchElementException(); + return current.next(); + } + + private void maybeAdvance() { + if (current == null || (!current.hasNext() && batch.hasNext())) { + current = batch.next().getRecords().iterator(); + } + } + + } + + private static class BatchIterator> implements Iterator> { + + private final M mapper; + private final LambdaQueryWrapper query; + private final Page page; + private boolean pageConsumed = false; + + private BatchIterator(M mapper, long pageSize, LambdaQueryWrapper query) { + this.mapper = mapper; + this.query = query; + this.page = new Page<>(); + this.page.setSize(pageSize); + this.page.setCurrent(0); + } + + @Override + public boolean hasNext() { + maybeFetchNextPage(); + // 1. 很明显的还有下一页 + // 2. 看当前页的数据消费情况 (第一页和最后一页) + return page.getCurrent() < page.getPages() || (!pageConsumed && !page.getRecords().isEmpty()); + } + + @Override + public Page next() { + if (!hasNext()) + throw new NoSuchElementException(); + pageConsumed = true; + Page copy = new Page<>(); + copy.setRecords(page.getRecords()); + copy.setTotal(page.getTotal()); + copy.setSize(page.getSize()); + copy.setCurrent(page.getCurrent()); + copy.setOrders(page.orders()); + return copy; + } + + private void maybeFetchNextPage() { + // 还没有获取过数据或者当前页数据已经消费, 就获取新page数据 + if (page.getCurrent() == 0 || (pageConsumed && page.hasNext())) { + page.setCurrent(page.getCurrent() + 1); + mapper.selectPage(page, query); + pageConsumed = false; + } + } + } + +} \ No newline at end of file diff --git a/start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java b/start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java new file mode 100644 index 00000000..9914197d --- /dev/null +++ b/start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java @@ -0,0 +1,24 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import cn.axzo.msg.center.MsgCenterApplication; +import lombok.RequiredArgsConstructor; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author yanglin + */ +@SpringBootTest(classes = MsgCenterApplication.class) +@RequiredArgsConstructor(onConstructor_ = @Autowired) +class OldMsgStatCacheReloadJobTest { + private final OldMsgStatCacheReloadJob oldMsgStatCacheReloadJob; + + @Test + void foo() throws Exception { + oldMsgStatCacheReloadJob.execute(null); + } + +} \ No newline at end of file From a05e7e4112dbba106fd4bb334419493957f04d39 Mon Sep 17 00:00:00 2001 From: yanglin Date: Wed, 24 Apr 2024 18:09:38 +0800 Subject: [PATCH 03/33] =?UTF-8?q?REQ-2405:=20=E6=B7=BB=E5=8A=A0=E5=8F=8B?= =?UTF-8?q?=E7=9B=9Fapi=E6=96=87=E6=A1=A3=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msg/center/inside/notices/service/umeng/PushClient.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/umeng/PushClient.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/umeng/PushClient.java index e2bcadbc..f0923708 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/umeng/PushClient.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/umeng/PushClient.java @@ -4,6 +4,9 @@ import cn.hutool.http.HttpUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; +/** + * api文档: api文档 + */ @Slf4j public class PushClient { From 418304115d9d8f4e6bf7afbe24ee108dd5033354 Mon Sep 17 00:00:00 2001 From: yanglin Date: Wed, 24 Apr 2024 18:14:50 +0800 Subject: [PATCH 04/33] =?UTF-8?q?REQ-2405:=20=E6=B7=BB=E5=8A=A0todo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/cn/axzo/msg/center/util/IterableMapper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java index 45642f7e..cedfc180 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java @@ -20,4 +20,6 @@ public interface IterableMapper extends BaseMapper { return RecordIterable.iterateElement(this, pageSize, query); } + // TODO(yl): new methods with dynamic total + } \ No newline at end of file From 744c868cda87180a76279b6358d8513dfc487fb8 Mon Sep 17 00:00:00 2001 From: yanglin Date: Mon, 22 Apr 2024 10:15:43 +0800 Subject: [PATCH 05/33] =?UTF-8?q?REQ-2303:=20=E7=BB=9F=E8=AE=A1=E5=88=86?= =?UTF-8?q?=E7=B1=BB=E6=95=B0=E6=8D=AE=E6=97=B6,=20=E5=8F=AA=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1=E8=BF=94=E5=9B=9E=E5=88=86=E7=B1=BB=E5=AD=90=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E7=9A=84=E6=95=B0=E6=8D=AE=EF=BC=8C=E5=89=8D=E7=AB=AF?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E4=B8=8A=E4=B8=8D=E5=85=B3=E6=B3=A8=E5=88=86?= =?UTF-8?q?=E7=B1=BB=E7=9A=84=E5=B1=82=E7=BA=A7=E5=85=B3=E7=B3=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/group/GroupTemplateService.java | 14 +++++++------- .../service/todo/TodoRangeQueryService.java | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/group/GroupTemplateService.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/group/GroupTemplateService.java index 5e29b215..bbebacf8 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/group/GroupTemplateService.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/group/GroupTemplateService.java @@ -52,10 +52,8 @@ public class GroupTemplateService { return new RootNodeWrapper(TreeBuilder.build(nodes, true)); } - public List> getTodoGroups( - AppTerminalTypeEnum terminal, boolean leafOnly) { - Set configuredIds = new HashSet<>( - cfg.fetchMessageGroupTreeNodeIds(terminal)); + public List> getTodoGroups(AppTerminalTypeEnum terminal) { + Set configuredIds = new HashSet<>(cfg.fetchMessageGroupTreeNodeIds(terminal)); ArrayList> configuredNodes = new ArrayList<>(); getGroupRoot().unwrap().walkDown(node -> { //noinspection unchecked @@ -69,10 +67,12 @@ public class GroupTemplateService { configuredNodes.add(valueNode); return true; }); - if (!leafOnly) - return configuredNodes; + return configuredNodes; + } + + public List> collectLeafNodes(List> nodes) { List> leafNodes = new ArrayList<>(); - for (ValueNode configuredNode : configuredNodes) { + for (ValueNode configuredNode : nodes) { configuredNode.walkDown(node -> { //noinspection unchecked ValueNode valueNode = node.tryCast(ValueNode.class); diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java index 27200d30..30ef31f4 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java @@ -217,8 +217,7 @@ public class TodoRangeQueryService { if (!request.determineGroupNodeCodes().isEmpty()) return groupTemplateService.collectTemplateCodes(request.determineGroupNodeCodes()); // 获取可见的模版时, 不用定位到叶子节点 - List> nodes = groupTemplateService - .getTodoGroups(request.getAppTerminalType(), false); + List> nodes = groupTemplateService.getTodoGroups(request.getAppTerminalType()); return groupTemplateService.collectTemplateCodes(nodes); } @@ -277,10 +276,11 @@ public class TodoRangeQueryService { } public List> determineStatNodes(MessageGroupNodeStatisticParam request) { - if (CollectionUtils.isNotEmpty(request.getGroupNodeCodes())) - return groupTemplateService.getGroupRoot().getNodes(request.getGroupNodeCodes()); + List> nodes = CollectionUtils.isNotEmpty(request.getGroupNodeCodes()) + ? groupTemplateService.getGroupRoot().getNodes(request.getGroupNodeCodes()) + : groupTemplateService.getTodoGroups(request.getTerminalType()); // 只返回给前端叶子节点的信息, 前端不需要层级信息 - return groupTemplateService.getTodoGroups(request.getTerminalType(), true); + return groupTemplateService.collectLeafNodes(nodes); } // helper From 77fcf015e016182eb71979270f284b4a372d1a63 Mon Sep 17 00:00:00 2001 From: yanglin Date: Thu, 25 Apr 2024 13:40:30 +0800 Subject: [PATCH 06/33] =?UTF-8?q?REQ-2405:=20=E5=AE=8C=E6=88=90IterableMap?= =?UTF-8?q?per=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/oldmsg/OldMsgIdentifyInfo.java | 6 +++ .../service/impl/oldmsg/OldMsgStatCache.java | 6 ++- .../impl/oldmsg/OldMsgStatCacheReloadJob.java | 12 ++--- .../impl/oldmsg/PersonAndCacheValue.java | 8 +++- .../axzo/msg/center/util/IterableMapper.java | 45 ++++++++++++++++++- .../axzo/msg/center/util/RecordIterable.java | 39 +++++++++++----- .../oldmsg/OldMsgStatCacheReloadJobTest.java | 24 ---------- 7 files changed, 95 insertions(+), 45 deletions(-) delete mode 100644 start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyInfo.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyInfo.java index dddcddff..04fbea57 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyInfo.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgIdentifyInfo.java @@ -1,6 +1,7 @@ package cn.axzo.msg.center.message.service.impl.oldmsg; import cn.axzo.msg.center.service.dto.IdentityDTO; +import com.alibaba.fastjson.JSON; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -14,4 +15,9 @@ import java.util.List; public class OldMsgIdentifyInfo { private final Long personId; private final List identities; + + @Override + public String toString() { + return JSON.toJSONString(this); + } } \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index a53ae917..604dff9b 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -63,7 +63,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { private final EventConsumer eventConsumer; private final PersonService personService; private final ForkJoinPool foregroundExecutor = new ForkJoinPool(20); - private final ForkJoinPool backgroundExecutor = new ForkJoinPool(20); + private final ForkJoinPool backgroundExecutor = new ForkJoinPool(10); public GeneralMessageOldDataStatisticResponse getCacheResponseOrReload(GeneralMessageOldDataStatisticRequest request) { GeneralMessageOldDataStatisticResponse resp = getCacheResponse(request); @@ -93,17 +93,19 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { stopWatch.start("initPersonCacheValueForeground"); CacheValue cacheValue = initPersonCacheValueForeground(request.getPersonId()); stopWatch.stop(); - log.info(stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); GeneralMessageOldDataStatisticResponse response = cacheValue .findResponse(request.getIdentities()) .orElse(null); // 边界 if (response == null) { log.info("oldMsgStat: identities cache missing? reload stat for request identities. request={}", request); + stopWatch.start("initPersonCacheValueForeground - miss again?"); IdentityResponse newIdentityResponse = getIdentityResponse(request.getPersonId(), request.getIdentities()); + stopWatch.stop(); cacheValue.addIdentityResponse(newIdentityResponse); } setCacheValue(request.getPersonId(), cacheValue); + log.info(stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); return cacheValue .findResponse(request.getIdentities()) // should never happen diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java index 9b7ddd05..15c5a788 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java @@ -5,7 +5,6 @@ import cn.axzo.msg.center.domain.entity.MessageRecord; import cn.axzo.msg.center.message.service.impl.person.PersonService; import cn.hutool.core.date.StopWatch; import com.alibaba.fastjson.JSON; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.IJobHandler; @@ -56,13 +55,14 @@ public class OldMsgStatCacheReloadJob extends IJobHandler { ? OldMsgCacheReloadJobParam.defaultParam() : JSON.parseObject(param, OldMsgCacheReloadJobParam.class); Date sinceDate = DateTime.now().minusDays(jobParam.determineSyncDay()).toDate(); - LambdaQueryWrapper query = query(MessageRecord.class) - .ge(MessageRecord::getCreateAt, sinceDate) - // 选刷最近的 - .orderByDesc(MessageRecord::getId); StopWatch stopWatch = new StopWatch("reloadBackground driven by job"); Set reloadedPersonIds = new HashSet<>(); - for (Page page : messageRecordMapper.iterateBath(jobParam.determineBatchSize(), query)) { + Iterable> pages = messageRecordMapper.iterateBath( + jobParam.determineBatchSize(), query(MessageRecord.class) + .ge(MessageRecord::getCreateAt, sinceDate) + // 选刷最近的 + .orderByDesc(MessageRecord::getId)); + for (Page page : pages) { stopWatch.start("reload stat, records size=" + page.getRecords().size()); reloadStat(reloadedPersonIds, page); stopWatch.stop(); diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java index 7ea644b0..58a37244 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java @@ -1,5 +1,6 @@ package cn.axzo.msg.center.message.service.impl.oldmsg; +import com.alibaba.fastjson.JSON; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -11,4 +12,9 @@ import lombok.RequiredArgsConstructor; public class PersonAndCacheValue { private final Long personId; private final CacheValue cacheValue; -} + + @Override + public String toString() { + return JSON.toJSONString(this); + } +} \ No newline at end of file diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java index cedfc180..60519c09 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java @@ -9,17 +9,58 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; */ public interface IterableMapper extends BaseMapper { + /** + * 遍历数据 + *

对查询出的数据的操作肯定不会更新query中查询条件包含的字段 + */ default Iterable> iterateBath(long pageSize, LambdaQueryWrapper query) { return RecordIterable.iterateBath(this, pageSize, query); } /** - * 还没有严格验证, 小心使用 + * 遍历数据 + *

对查询出的数据的操作肯定不会更新query中查询条件包含的字段 */ default Iterable iterateElement(long pageSize, LambdaQueryWrapper query) { return RecordIterable.iterateElement(this, pageSize, query); } - // TODO(yl): new methods with dynamic total + /** + * 遍历数据 + *

对查询出的数据的操作肯定会更新query中查询条件包含的字段 + *

+ * LambdaQueryWrapper query = Queries.query(MessageRecord.class) + * .eq(MessageRecord::getWorkspaceId, workspaceId) + * // 通过状态查询 + * .eq(MessageRecord::getState, MsgStateEnum.UNSENT); + * for(Page page : iterateBathDoUpdateQueryFields(1000, query)) { + * for(T record : page.getRecords()) { + * // 更新状态 + * record.setState(MsgStateEnum.HAS_BEEN_SENT); + * messageRecordMapper.updateById(record); + * } + * } + */ + default Iterable> iterateBathDoUpdateQueryFields(long pageSize, LambdaQueryWrapper query) { + return RecordIterable.iterateBathDoUpdateQueryFields(this, pageSize, query); + } + + /** + * 遍历数据 + *

对查询出的数据的操作肯定会更新query中查询条件包含的字段 + *

+ * LambdaQueryWrapper query = Queries.query(MessageRecord.class) + * .eq(MessageRecord::getWorkspaceId, workspaceId) + * // 通过状态查询 + * .eq(MessageRecord::getState, MsgStateEnum.UNSENT); + * for(T record : iterateElementDoUpdateQueryFields(1000, query)) { + * // 更新状态 + * record.setState(MsgStateEnum.HAS_BEEN_SENT); + * messageRecordMapper.updateById(record); + * } + */ + default Iterable iterateElementDoUpdateQueryFields(long pageSize, LambdaQueryWrapper query) { + return RecordIterable.iterateElementDoUpdateQueryFields(this, pageSize, query); + } } \ No newline at end of file diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java index cdcb8fbe..a72f5954 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import java.util.Collections; import java.util.Iterator; import java.util.NoSuchElementException; @@ -14,15 +15,24 @@ class RecordIterable { static > Iterable> iterateBath( M mapper, long pageSize, LambdaQueryWrapper query) { - return () -> new BatchIterator<>(mapper, pageSize, query); + return () -> new BatchIterator<>(mapper, pageSize, query, PageStrategy.ADVANCE_CURRENT_PAGE); } static > Iterable iterateElement( M mapper, long pageSize, LambdaQueryWrapper query) { - return () -> { - BatchIterator batchIterator = new BatchIterator<>(mapper, pageSize, query); - return new ElementIterator<>(batchIterator); - }; + return () -> new ElementIterator<>( + new BatchIterator<>(mapper, pageSize, query, PageStrategy.ADVANCE_CURRENT_PAGE)); + } + + static > Iterable> iterateBathDoUpdateQueryFields( + M mapper, long pageSize, LambdaQueryWrapper query) { + return () -> new BatchIterator<>(mapper, pageSize, query, PageStrategy.WILL_UPDATE_QUERY_FIELDS); + } + + static > Iterable iterateElementDoUpdateQueryFields( + M mapper, long pageSize, LambdaQueryWrapper query) { + return () -> new ElementIterator<>( + new BatchIterator<>(mapper, pageSize, query, PageStrategy.WILL_UPDATE_QUERY_FIELDS)); } private static class ElementIterator> implements Iterator { @@ -48,9 +58,10 @@ class RecordIterable { } private void maybeAdvance() { - if (current == null || (!current.hasNext() && batch.hasNext())) { - current = batch.next().getRecords().iterator(); - } + if (current == null || !current.hasNext() && batch.hasNext()) + current = batch.hasNext() + ? batch.next().getRecords().iterator() + : Collections.emptyIterator(); } } @@ -59,12 +70,14 @@ class RecordIterable { private final M mapper; private final LambdaQueryWrapper query; + private final PageStrategy pageStrategy; private final Page page; private boolean pageConsumed = false; - private BatchIterator(M mapper, long pageSize, LambdaQueryWrapper query) { + BatchIterator(M mapper, long pageSize, LambdaQueryWrapper query, PageStrategy pageStrategy) { this.mapper = mapper; this.query = query; + this.pageStrategy = pageStrategy; this.page = new Page<>(); this.page.setSize(pageSize); this.page.setCurrent(0); @@ -95,11 +108,17 @@ class RecordIterable { private void maybeFetchNextPage() { // 还没有获取过数据或者当前页数据已经消费, 就获取新page数据 if (page.getCurrent() == 0 || (pageConsumed && page.hasNext())) { - page.setCurrent(page.getCurrent() + 1); + if (pageStrategy == PageStrategy.ADVANCE_CURRENT_PAGE) + page.setCurrent(page.getCurrent() + 1); + else + page.setCurrent(1); mapper.selectPage(page, query); pageConsumed = false; } } } + private enum PageStrategy { + ADVANCE_CURRENT_PAGE, WILL_UPDATE_QUERY_FIELDS + } } \ No newline at end of file diff --git a/start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java b/start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java deleted file mode 100644 index 9914197d..00000000 --- a/start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java +++ /dev/null @@ -1,24 +0,0 @@ -package cn.axzo.msg.center.message.service.impl.oldmsg; - -import cn.axzo.msg.center.MsgCenterApplication; -import lombok.RequiredArgsConstructor; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author yanglin - */ -@SpringBootTest(classes = MsgCenterApplication.class) -@RequiredArgsConstructor(onConstructor_ = @Autowired) -class OldMsgStatCacheReloadJobTest { - private final OldMsgStatCacheReloadJob oldMsgStatCacheReloadJob; - - @Test - void foo() throws Exception { - oldMsgStatCacheReloadJob.execute(null); - } - -} \ No newline at end of file From 789b755c5ca73d7c736688066d5aaabdf9d3a1d5 Mon Sep 17 00:00:00 2001 From: yanglin Date: Thu, 25 Apr 2024 18:19:42 +0800 Subject: [PATCH 07/33] REQ-2405: almost there --- .../config/PendingMessageBizConfig.java | 8 +- .../service/impl/MessageCoreServiceImpl.java | 23 +++-- .../impl/MessageRecordServiceImpl.java | 2 +- .../param/MessageGroupNodeStatisticParam.java | 4 + .../oldmsg/OldMsgCacheReloadJobParam.java | 8 +- .../service/impl/oldmsg/OldMsgStatCache.java | 63 ++++++++---- .../impl/oldmsg/OldMsgStatCacheReloadJob.java | 6 +- ...heValue.java => OldMsgStatCacheValue.java} | 2 +- .../impl/oldmsg/PersonAndCacheValue.java | 2 +- .../service/todo/TodoRangeQueryService.java | 48 ++++++--- .../service/todo/cache/NodeStatCache.java | 99 +++++++++++++++++++ .../todo/cache/NodeStatCacheValue.java | 24 +++++ .../api/mq/SendMessageRecordMessage.java | 4 +- .../dal/mapper/MessageBaseTemplateMapper.java | 7 ++ .../msg/center/dal/mapper/TodoMapper.java | 6 +- .../axzo/msg/center/util/IterableMapper.java | 92 ++++++++++++++--- .../axzo/msg/center/util/RecordIterable.java | 21 ++-- .../mapper/MessageBaseTemplateMapper.xml | 9 ++ .../src/main/resources/mapper/Todo.xml | 4 + .../oldmsg/OldMsgStatCacheReloadJobTest.java | 24 +++++ .../msg/center/util/RecordIterableTest.java | 34 +++++++ 21 files changed, 412 insertions(+), 78 deletions(-) rename inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/{CacheValue.java => OldMsgStatCacheValue.java} (97%) create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCacheValue.java create mode 100644 msg-center-dal/src/main/resources/mapper/MessageBaseTemplateMapper.xml create mode 100644 start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java create mode 100644 start/src/test/java/cn/axzo/msg/center/util/RecordIterableTest.java diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java index b0912365..99eb9d74 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java @@ -101,7 +101,7 @@ public class PendingMessageBizConfig { private boolean formatExecSQL; @Getter - private int oldMsgStatDataExpireHours = 6; + private int oldMsgStatCacheDataExpireHours = 8; @Getter private int oldMsgStateDataBackendUpdateBatchSize = 100; @@ -109,6 +109,12 @@ public class PendingMessageBizConfig { @Getter private long msgCenterMqSelfConsumeMaxExecMs = 20000L; + @Getter + private int nodeStatCacheMaxRequestNodeCodeSize = 10; + + @Getter + private int nodeStatCacheDataExpireHours = 8; + public boolean hasMessageDetailStyle(String code) { return name2DetailStyle != null && name2DetailStyle.containsKey(code); } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageCoreServiceImpl.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageCoreServiceImpl.java index 51e75c4a..edfdef5f 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageCoreServiceImpl.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageCoreServiceImpl.java @@ -3,6 +3,7 @@ package cn.axzo.msg.center.inside.notices.service.impl; import cn.axzo.core.domain.PageResult; import cn.axzo.msg.center.api.enums.MsgStateEnum; import cn.axzo.msg.center.api.enums.MsgTypeEnum; +import cn.axzo.msg.center.api.mq.SendMessageRecordMessage; import cn.axzo.msg.center.api.request.CmsMsgQueryReq; import cn.axzo.msg.center.api.request.MessageModuleStatisticReq; import cn.axzo.msg.center.api.request.MessagePageQueryReq; @@ -34,6 +35,9 @@ import cn.axzo.msg.center.inside.notices.service.MessageRouterService; import cn.axzo.msg.center.inside.notices.service.RawMessageRecordService; import cn.axzo.msg.center.message.domain.param.MessageGroupNodeStatisticParam; import cn.axzo.msg.center.message.service.PendingMessageNewService; +import cn.axzo.msg.center.mq.MqMessageRecord; +import cn.axzo.msg.center.mq.MqMessageType; +import cn.axzo.msg.center.mq.MqProducer; import cn.axzo.msg.center.service.dto.PersonDTO; import cn.axzo.msg.center.service.enums.AppTerminalTypeEnum; import cn.azxo.framework.common.model.Page; @@ -99,7 +103,7 @@ public class MessageCoreServiceImpl implements MessageCoreService { @Resource private RawMessageRecordService rawMessageRecordService; @Resource - private PendingMessageNewService pendingMessageNewService; + private MqProducer mqProducer; public MsgRouteTypeEnum getSystemType(String systemType) { /*String systemType = ContextInfoHolder.get().getSystemAndDeviceInfo().getSystemType();*/ @@ -355,13 +359,18 @@ public class MessageCoreServiceImpl implements MessageCoreService { // 阅读单条数据 rawMessageRecordService.updatePersonMessageState(request.getPersonId(), srcStates, MsgStateEnum.HAVE_READ, request.getMsgId()); - return; + } else { + List moduleIds = messageModuleService.listByModuleName(request.getModuleName()).stream() + .map(MessageModule::getId) + .collect(Collectors.toList()); + rawMessageRecordService.updatePersonMessageState(request.getPersonId(), srcStates, MsgStateEnum.HAVE_READ, + MsgTypeEnum.GENERAL_MESSAGE, null, moduleIds); } - List moduleIds = messageModuleService.listByModuleName(request.getModuleName()).stream() - .map(MessageModule::getId) - .collect(Collectors.toList()); - rawMessageRecordService.updatePersonMessageState(request.getPersonId(), srcStates, MsgStateEnum.HAVE_READ, - MsgTypeEnum.GENERAL_MESSAGE, null, moduleIds); + SendMessageRecordMessage mqMessage = new SendMessageRecordMessage(); + mqMessage.setReadRequest(request); + mqProducer.send(MqMessageRecord + .builder(MqMessageType.OLD_MSG_SEND, mqMessage) + .build()); } @Override diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java index 7e2996ae..aad42973 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java @@ -186,7 +186,7 @@ public class MessageRecordServiceImpl implements MessageRecordService { pushMessages.addAll(messageRecords); }); SendMessageRecordMessage mqMessage = new SendMessageRecordMessage(); - mqMessage.setRequest(message); + mqMessage.setSendRequest(message); mqProducer.send(MqMessageRecord .builder(MqMessageType.OLD_MSG_SEND, mqMessage) .build()); diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/domain/param/MessageGroupNodeStatisticParam.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/domain/param/MessageGroupNodeStatisticParam.java index b758ee96..17d2be37 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/domain/param/MessageGroupNodeStatisticParam.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/domain/param/MessageGroupNodeStatisticParam.java @@ -49,6 +49,10 @@ public class MessageGroupNodeStatisticParam implements Serializable { */ private Boolean withIdentify; + public Long getPersonId() { + return getOperator().getId(); + } + public static MessageGroupNodeStatisticParam from(PendingMessageStatisticRequest request) { MessageGroupNodeStatisticParam param = BeanConverter.convert(request, MessageGroupNodeStatisticParam.class); IdentityDTO identity = IdentityDTO.builder() diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgCacheReloadJobParam.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgCacheReloadJobParam.java index af5ac67f..0f49006c 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgCacheReloadJobParam.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgCacheReloadJobParam.java @@ -9,8 +9,7 @@ import lombok.Data; @Data public class OldMsgCacheReloadJobParam { - private static final int DEFAULT_SYNC_DAY = 60; - private static final int DEFAULT_BATCH_SIZE = 1000; + private static final int DEFAULT_SYNC_DAY = 30; private int syncDay; private int batchSize; @@ -18,7 +17,6 @@ public class OldMsgCacheReloadJobParam { static OldMsgCacheReloadJobParam defaultParam() { OldMsgCacheReloadJobParam defaultParam = new OldMsgCacheReloadJobParam(); defaultParam.setSyncDay(DEFAULT_SYNC_DAY); - defaultParam.setBatchSize(DEFAULT_BATCH_SIZE); return defaultParam; } @@ -26,10 +24,6 @@ public class OldMsgCacheReloadJobParam { return syncDay <= 0 ? DEFAULT_SYNC_DAY : syncDay; } - int determineBatchSize() { - return batchSize <= 0 ? DEFAULT_BATCH_SIZE : batchSize; - } - @Override public String toString() { return JSON.toJSONString(this); diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index 604dff9b..0a83bd5b 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -6,11 +6,12 @@ import cn.axzo.framework.rocketmq.EventHandler; import cn.axzo.msg.center.api.mq.SendMessageRecordMessage; import cn.axzo.msg.center.api.request.CmsMsgQueryReq; import cn.axzo.msg.center.api.request.GeneralMessageReq; +import cn.axzo.msg.center.api.request.MessageReadAllReq; import cn.axzo.msg.center.api.response.MessageNewRes; import cn.axzo.msg.center.inside.notices.config.PendingMessageBizConfig; import cn.axzo.msg.center.message.service.MessageSendTwiceRecordService; import cn.axzo.msg.center.message.service.impl.GeneralMessageOldServiceImpl; -import cn.axzo.msg.center.message.service.impl.oldmsg.CacheValue.IdentityResponse; +import cn.axzo.msg.center.message.service.impl.oldmsg.OldMsgStatCacheValue.IdentityResponse; import cn.axzo.msg.center.message.service.impl.person.PersonIdInfo; import cn.axzo.msg.center.message.service.impl.person.PersonService; import cn.axzo.msg.center.mq.MqMessageType; @@ -24,6 +25,7 @@ import cn.azxo.framework.common.model.Page; import cn.hutool.core.date.StopWatch; import com.alibaba.fastjson.JSON; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -62,7 +64,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { private final PendingMessageBizConfig cfg; private final EventConsumer eventConsumer; private final PersonService personService; - private final ForkJoinPool foregroundExecutor = new ForkJoinPool(20); + private final ForkJoinPool foregroundExecutor = new ForkJoinPool(15); private final ForkJoinPool backgroundExecutor = new ForkJoinPool(10); public GeneralMessageOldDataStatisticResponse getCacheResponseOrReload(GeneralMessageOldDataStatisticRequest request) { @@ -71,7 +73,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { } private GeneralMessageOldDataStatisticResponse getCacheResponse(GeneralMessageOldDataStatisticRequest request) { - CacheValue cacheValue = getCacheValue(request.getPersonId()); + OldMsgStatCacheValue cacheValue = getCacheValue(request.getPersonId()); if (cacheValue == null) { log.info("oldMsgStat: cache miss, need to reload in foreground. request={}", request); return null; @@ -91,7 +93,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { StopWatch stopWatch = new StopWatch(String.format( "%s-reloadForeground", getClass().getSimpleName())); stopWatch.start("initPersonCacheValueForeground"); - CacheValue cacheValue = initPersonCacheValueForeground(request.getPersonId()); + OldMsgStatCacheValue cacheValue = initPersonCacheValueForeground(request.getPersonId()); stopWatch.stop(); GeneralMessageOldDataStatisticResponse response = cacheValue .findResponse(request.getIdentities()) @@ -112,13 +114,13 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { .orElse(null); } - public Map reloadBackground(Set personIds) throws Exception { + public Map reloadBackground(Set personIds) throws Exception { log.info("oldMsgStat: reload cache for person in background. personIds={}", JSON.toJSONString(personIds)); if (CollectionUtils.isEmpty(personIds)) { log.info("oldMsgStat: personIds is empty"); return Collections.emptyMap(); } - HashMap personId2CacheValue = new HashMap<>(); + HashMap personId2CacheValue = new HashMap<>(); // 温柔点, 别一次性全丢到线程池 int bachSize = cfg.getOldMsgStateDataBackendUpdateBatchSize(); List> batches = Lists.partition(new ArrayList<>(personIds), bachSize); @@ -126,7 +128,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { ArrayList> futures = new ArrayList<>(); for (Long personId : batch) { futures.add(CompletableFuture.supplyAsync(() -> { - CacheValue cacheValue = initPersonCacheValueBackground(personId); + OldMsgStatCacheValue cacheValue = initPersonCacheValueBackground(personId); setCacheValue(personId, cacheValue); return new PersonAndCacheValue(personId, cacheValue); }, backgroundExecutor)); @@ -140,13 +142,13 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { return personId2CacheValue; } - private CacheValue initPersonCacheValueForeground(Long personId) { + private OldMsgStatCacheValue initPersonCacheValueForeground(Long personId) { log.info("initPersonCacheValueForeground. personId={}", personId); Function buildResponseFun = asManager -> { OldMsgIdentifyInfo identifyInfo = oldMsgIdentifyFilter.getIdentifyInfo(personId, asManager); return getIdentityResponse(personId, identifyInfo.getIdentities()); }; - CacheValue cacheValue = new CacheValue(); + OldMsgStatCacheValue cacheValue = new OldMsgStatCacheValue(); CompletableFuture asManagerFuture = CompletableFuture .supplyAsync(() -> buildResponseFun.apply(true), foregroundExecutor); CompletableFuture asWorkerFuture = CompletableFuture @@ -162,13 +164,13 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { return cacheValue; } - private CacheValue initPersonCacheValueBackground(Long personId) { + private OldMsgStatCacheValue initPersonCacheValueBackground(Long personId) { log.info("initPersonCacheValueBackground. personId={}", personId); Function buildResponseFun = asManager -> { OldMsgIdentifyInfo identifyInfo = oldMsgIdentifyFilter.getIdentifyInfo(personId, asManager); return getIdentityResponse(personId, identifyInfo.getIdentities()); }; - CacheValue cacheValue = new CacheValue(); + OldMsgStatCacheValue cacheValue = new OldMsgStatCacheValue(); cacheValue.addIdentityResponse(buildResponseFun.apply(true)); cacheValue.addIdentityResponse(buildResponseFun.apply(false)); return cacheValue; @@ -187,10 +189,10 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { // !! cache helper - private void setCacheValue(Long personId, CacheValue cacheValue) { + private void setCacheValue(Long personId, OldMsgStatCacheValue cacheValue) { log.info("set cache value for person. personId={}", personId); - long secondsDelta = TimeUnit.HOURS.toSeconds(cfg.getOldMsgStatDataExpireHours()); - // with random seconds delta, 0-600 seconds + long secondsDelta = TimeUnit.HOURS.toSeconds(cfg.getOldMsgStatCacheDataExpireHours()); + // add some random delta seconds, range: 0-600 seconds secondsDelta += new SecureRandom().nextInt(10 * 60); String redisKey = buildCacheKey(personId); @@ -198,10 +200,10 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { stringRedisTemplate.opsForValue().set(redisKey, redisValue, secondsDelta, TimeUnit.SECONDS); } - private CacheValue getCacheValue(Long personId) { + private OldMsgStatCacheValue getCacheValue(Long personId) { String redisKey = buildCacheKey(personId); String json = stringRedisTemplate.opsForValue().get(redisKey); - return StringUtils.isBlank(json) ? null : JSON.parseObject(json, CacheValue.class); + return StringUtils.isBlank(json) ? null : JSON.parseObject(json, OldMsgStatCacheValue.class); } private static String buildCacheKey(Long personId) { @@ -273,12 +275,23 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { private void handleMqMessage(Event event) throws Exception { SendMessageRecordMessage mqMessage = event.normalizedData(SendMessageRecordMessage.class); - GeneralMessageReq request = mqMessage.getRequest(); - Map knownIdentityId2PersonId = request.getToldIdPersonIdMap(); + GeneralMessageReq sendRequest = mqMessage.getSendRequest(); + MessageReadAllReq readRequest = mqMessage.getReadRequest(); + if (sendRequest != null) { + log.info("handle mq event - driven by sendRequest={}", sendRequest); + handleSendRequestEvent(sendRequest); + } else if (readRequest != null) { + log.info("handle mq event - driven by readRequest={}", readRequest); + handleReadRequest(readRequest); + } + } + + private void handleSendRequestEvent(GeneralMessageReq sendRequest) throws Exception { + Map knownIdentityId2PersonId = sendRequest.getToldIdPersonIdMap(); if (knownIdentityId2PersonId == null) knownIdentityId2PersonId = new HashMap<>(); HashSet personIds = new HashSet<>(knownIdentityId2PersonId.values()); - Set receiverIdentityIds = request.getToId(); + Set receiverIdentityIds = sendRequest.getToId(); if (receiverIdentityIds == null) receiverIdentityIds = Collections.emptySet(); HashSet toQueryPersonIdIdentityIds = new HashSet<>(); @@ -290,11 +303,19 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { PersonIdInfo personIdInfo = personService.getPersonIdsByIdentities(toQueryPersonIdIdentityIds); personIds.addAll(personIdInfo.getPersonIds()); } - StopWatch stopWatch = new StopWatch("reloadBackground driven by mq event"); - stopWatch.start("reloadBackground"); + StopWatch stopWatch = new StopWatch("reloadBackground"); + stopWatch.start("handleSendRequestEvent"); reloadBackground(personIds); stopWatch.stop(); log.info(stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); } + private void handleReadRequest(MessageReadAllReq readRequest) throws Exception { + StopWatch stopWatch = new StopWatch("reloadBackground"); + stopWatch.start("handleReadRequest"); + reloadBackground(Sets.newHashSet(readRequest.getPersonId())); + stopWatch.stop(); + log.info(stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); + } + } \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java index 15c5a788..2c05989e 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJob.java @@ -57,10 +57,10 @@ public class OldMsgStatCacheReloadJob extends IJobHandler { Date sinceDate = DateTime.now().minusDays(jobParam.determineSyncDay()).toDate(); StopWatch stopWatch = new StopWatch("reloadBackground driven by job"); Set reloadedPersonIds = new HashSet<>(); - Iterable> pages = messageRecordMapper.iterateBath( - jobParam.determineBatchSize(), query(MessageRecord.class) + Iterable> pages = messageRecordMapper + .iterateBath(query(MessageRecord.class) .ge(MessageRecord::getCreateAt, sinceDate) - // 选刷最近的 + // 先刷最近的 .orderByDesc(MessageRecord::getId)); for (Page page : pages) { stopWatch.start("reload stat, records size=" + page.getRecords().size()); diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/CacheValue.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java similarity index 97% rename from inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/CacheValue.java rename to inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java index 0894aab3..5900173d 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/CacheValue.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java @@ -16,7 +16,7 @@ import java.util.Set; * @author yanglin */ @Data -public class CacheValue { +public class OldMsgStatCacheValue { private List identityResponses; diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java index 58a37244..5868a3d5 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/PersonAndCacheValue.java @@ -11,7 +11,7 @@ import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public class PersonAndCacheValue { private final Long personId; - private final CacheValue cacheValue; + private final OldMsgStatCacheValue cacheValue; @Override public String toString() { diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java index 30ef31f4..5ff148bf 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java @@ -1,5 +1,6 @@ package cn.axzo.msg.center.message.service.todo; +import cn.axzo.basics.common.exception.ServiceException; import cn.axzo.maokai.api.util.Ref; import cn.axzo.maokai.api.vo.response.tree.ValueNode; import cn.axzo.msg.center.common.enums.TableIsDeleteEnum; @@ -14,6 +15,7 @@ import cn.axzo.msg.center.message.domain.param.MessageGroupNodeStatisticParam; import cn.axzo.msg.center.message.service.group.GroupTemplateService; import cn.axzo.msg.center.message.service.group.NodeWrapper; import cn.axzo.msg.center.message.service.impl.PendingMessageNewServiceImpl; +import cn.axzo.msg.center.message.service.todo.cache.NodeStatCache; import cn.axzo.msg.center.message.service.todo.mybatis.CollectSQLInterceptor; import cn.axzo.msg.center.message.service.todo.pagequery.PageQuerySort; import cn.axzo.msg.center.message.service.todo.queryanalyze.SimpleAnalyzer; @@ -44,6 +46,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.concurrent.ForkJoinPool; import java.util.function.Function; import java.util.function.Supplier; @@ -67,6 +70,8 @@ public class TodoRangeQueryService { private final TodoRespBuilder todoRespBuilder; private final AnalysisConfig analysisConfig; private final GroupTemplateService groupTemplateService; + private final NodeStatCache nodeStatCache; + private final ForkJoinPool statExecutor = new ForkJoinPool(20); // !! page query @@ -224,6 +229,10 @@ public class TodoRangeQueryService { // !! stat public PendingMessageStatisticResponseV2 countStatGrouped(MessageGroupNodeStatisticParam request) { + return nodeStatCache.getCacheResponseOrReload(request, () -> countStatGroupedImpl(request)); + } + + private PendingMessageStatisticResponseV2 countStatGroupedImpl(MessageGroupNodeStatisticParam request) { List> nodes = determineStatNodes(request); Function, GroupStat> nodeStatFun = valueNode -> { int executableCount = countStatByNode(request, valueNode, TodoType.EXECUTABLE); @@ -236,20 +245,30 @@ public class TodoRangeQueryService { groupStat.setStat(new Stat(executableCount, copiedToMeCount)); return groupStat; }; - PendingMessageStatisticResponseV2 resp = new PendingMessageStatisticResponseV2(); - List stats = nodes.parallelStream().map(nodeStatFun).collect(toList()); - for (GroupStat stat : stats) - resp.addGroupStat(stat); - resp.setTotalStat(); - return resp; + try { + PendingMessageStatisticResponseV2 resp = new PendingMessageStatisticResponseV2(); + // 用一个偏大的线程池, IO密集型任务, commonPool不够用 + List stats = statExecutor + .submit(() -> nodes.parallelStream().map(nodeStatFun).collect(toList())) + .get(); + for (GroupStat stat : stats) + resp.addGroupStat(stat); + resp.setTotalStat(); + return resp; + } catch (Exception e) { + log.warn("分类统计异常, request={}", request, e); + throw new ServiceException("分类统计, 请稍后重试"); + } } /** * app首页 */ public Integer countStat(MessageGroupNodeStatisticParam request) { - List> nodes = determineStatNodes(request); - return countStatByNodes(request, nodes, TodoType.EXECUTABLE); + PendingMessageStatisticResponseV2 nodeStat = nodeStatCache + .getCacheResponseOrReload(request, () -> countStatGroupedImpl(request)); + Stat totalStat = nodeStat.getTotalStat(); + return totalStat == null ? 0 : totalStat.getPendingCount(); } private int countStatByNode(MessageGroupNodeStatisticParam request, @@ -265,7 +284,7 @@ public class TodoRangeQueryService { OuInfo ouInfo = OuInfo.create(request.getOuId(), request.getTerminalType(), todoType); LambdaQueryWrapper query = todoQuery(ouInfo) .in(Todo::getTemplateCode, templateCodes) - .eq(Todo::getExecutorPersonId, request.getOperator().getId()) + .eq(Todo::getExecutorPersonId, request.getPersonId()) .and(todoType == TodoType.EXECUTABLE, nested -> nested .eq(Todo::getType, TodoType.EXECUTABLE) .eq(Todo::getState, PendingMessageStateEnum.HAS_BEEN_SENT)) @@ -276,8 +295,15 @@ public class TodoRangeQueryService { } public List> determineStatNodes(MessageGroupNodeStatisticParam request) { - List> nodes = CollectionUtils.isNotEmpty(request.getGroupNodeCodes()) - ? groupTemplateService.getGroupRoot().getNodes(request.getGroupNodeCodes()) + Collection nodeCodes = request.getGroupNodeCodes(); + if (nodeCodes == null) { + nodeCodes = Collections.emptyList(); + } + nodeCodes = nodeCodes.stream() + .filter(StringUtils::isNotBlank) + .collect(toList()); + List> nodes = CollectionUtils.isNotEmpty(nodeCodes) + ? groupTemplateService.getGroupRoot().getNodes(nodeCodes) : groupTemplateService.getTodoGroups(request.getTerminalType()); // 只返回给前端叶子节点的信息, 前端不需要层级信息 return groupTemplateService.collectLeafNodes(nodes); diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java new file mode 100644 index 00000000..94ef1ced --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java @@ -0,0 +1,99 @@ +package cn.axzo.msg.center.message.service.todo.cache; + +import cn.axzo.msg.center.dal.mapper.MessageBaseTemplateMapper; +import cn.axzo.msg.center.dal.mapper.TodoMapper; +import cn.axzo.msg.center.inside.notices.config.PendingMessageBizConfig; +import cn.axzo.msg.center.message.domain.param.MessageGroupNodeStatisticParam; +import cn.axzo.msg.center.service.pending.response.PendingMessageStatisticResponseV2; +import com.alibaba.fastjson.JSON; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import java.security.SecureRandom; +import java.util.Collection; +import java.util.Date; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * @author yanglin + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class NodeStatCache { + + private final MessageBaseTemplateMapper messageBaseTemplateMapper; + private final TodoMapper todoMapper; + private final StringRedisTemplate stringRedisTemplate; + private final PendingMessageBizConfig cfg; + + public PendingMessageStatisticResponseV2 getCacheResponseOrReload( + MessageGroupNodeStatisticParam request, Supplier loader) { + Collection nodeCodes = request.getGroupNodeCodes(); + if (CollectionUtils.isNotEmpty(nodeCodes) && nodeCodes.size() > cfg.getNodeStatCacheMaxRequestNodeCodeSize()) { + log.info("超过了允许缓存的最大请求节点编码数量，不使用缓存, 直接从数据库获取. request={}", request); + return loader.get(); + } + Date templateLatestUpdateTime = messageBaseTemplateMapper.getTemplateLatestUpdateTime(); + Date todoLatestUpdateTime = todoMapper.getTodoLatestUpdateTime(request.getPersonId()); + NodeStatCacheValue cacheValue = getCacheValue(request); + if (cacheValue == null || cacheValue.shouldReload(templateLatestUpdateTime, todoLatestUpdateTime)) { + log.info("reloading node stat data, request={}", request); + PendingMessageStatisticResponseV2 response = loader.get(); + cacheValue = new NodeStatCacheValue(); + cacheValue.setResponse(response); + cacheValue.setTemplateLatestUpdateTime(templateLatestUpdateTime); + cacheValue.setTodoLatestUpdateTime(todoLatestUpdateTime); + setCacheValue(request, cacheValue); + } + return cacheValue.getResponse(); + } + + // !! cache helper + + private NodeStatCacheValue getCacheValue(MessageGroupNodeStatisticParam request) { + String redisKey = buildCacheKey(request); + String json = stringRedisTemplate.opsForValue().get(redisKey); + return StringUtils.isBlank(json) ? null : JSON.parseObject(json, NodeStatCacheValue.class); + } + + private void setCacheValue(MessageGroupNodeStatisticParam request, NodeStatCacheValue cacheValue) { + log.info("set cache value for request. request={}", request); + long secondsDelta = TimeUnit.HOURS.toSeconds(cfg.getNodeStatCacheDataExpireHours()); + // add some random delta seconds, range: 0-600 seconds + secondsDelta += new SecureRandom().nextInt(10 * 60); + + String redisKey = buildCacheKey(request); + String redisValue = JSON.toJSONString(cacheValue); + stringRedisTemplate.opsForValue().set(redisKey, redisValue, secondsDelta, TimeUnit.SECONDS); + } + + private static String buildCacheKey(MessageGroupNodeStatisticParam request) { + StringBuffer buf = new StringBuffer(); + buf.append("msg_center:node_stat:v1"); + Consumer appender = value -> { + if (value != null) + buf.append(":").append(value); + }; + appender.accept(request.getPersonId()); + appender.accept(request.getOuId()); + appender.accept(request.getTerminalType()); + Collection nodeCodes = request.getGroupNodeCodes(); + if (CollectionUtils.isNotEmpty(nodeCodes)) { + // 排序避免编码顺序和重复编码的影响 + nodeCodes.stream() + .filter(StringUtils::isNotBlank) + .distinct() + .sorted() + .forEach(appender); + } + return buf.toString(); + } + +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCacheValue.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCacheValue.java new file mode 100644 index 00000000..8d72daa2 --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCacheValue.java @@ -0,0 +1,24 @@ +package cn.axzo.msg.center.message.service.todo.cache; + +import cn.axzo.msg.center.service.pending.response.PendingMessageStatisticResponseV2; +import lombok.Data; + +import java.util.Date; +import java.util.Objects; + +/** + * @author yanglin + */ +@Data +class NodeStatCacheValue { + + private Date templateLatestUpdateTime; + private Date todoLatestUpdateTime; + private PendingMessageStatisticResponseV2 response; + + boolean shouldReload(Date templateLatestUpdateTime, Date todoLatestUpdateTime) { + return !Objects.equals(this.templateLatestUpdateTime, templateLatestUpdateTime) + || !Objects.equals(this.todoLatestUpdateTime, todoLatestUpdateTime); + } + +} \ No newline at end of file diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java b/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java index 6b94c4a4..c507168d 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java @@ -1,6 +1,7 @@ package cn.axzo.msg.center.api.mq; import cn.axzo.msg.center.api.request.GeneralMessageReq; +import cn.axzo.msg.center.api.request.MessageReadAllReq; import lombok.Data; import lombok.EqualsAndHashCode; @@ -12,5 +13,6 @@ import java.io.Serializable; @Data @EqualsAndHashCode(callSuper = true) public class SendMessageRecordMessage extends MqMessage implements Serializable { - private GeneralMessageReq request; + private GeneralMessageReq sendRequest; + private MessageReadAllReq readRequest; } \ No newline at end of file diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageBaseTemplateMapper.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageBaseTemplateMapper.java index 5937e5bb..34ce72d0 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageBaseTemplateMapper.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageBaseTemplateMapper.java @@ -4,6 +4,9 @@ import cn.axzo.msg.center.domain.entity.MessageBaseTemplate; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.ibatis.annotations.Mapper; +import javax.annotation.Nullable; +import java.util.Date; + /** * @description * 消息基础模板Mapper @@ -13,4 +16,8 @@ import org.apache.ibatis.annotations.Mapper; */ @Mapper public interface MessageBaseTemplateMapper extends BaseMapper { + + @Nullable + Date getTemplateLatestUpdateTime(); + } diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/TodoMapper.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/TodoMapper.java index 37dfe89f..5b2bc7eb 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/TodoMapper.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/TodoMapper.java @@ -5,6 +5,7 @@ import cn.axzo.msg.center.domain.entity.Todo; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.ibatis.annotations.Param; +import javax.annotation.Nullable; import java.util.Date; import java.util.List; @@ -21,4 +22,7 @@ public interface TodoMapper extends BaseMapper { List getMigratedBusinessId(); -} + @Nullable + Date getTodoLatestUpdateTime(@Param("executorPersonId") Long executorPersonId); + +} \ No newline at end of file diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java index 60519c09..d9f7c043 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java @@ -9,55 +9,121 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; */ public interface IterableMapper extends BaseMapper { + int DEFAULT_BATCH_SIZE = 2000; + /** - * 遍历数据 - *

对查询出的数据的操作肯定不会更新query中查询条件包含的字段 + * foreach遍历数据. 默认批量大小 + * + *

对查询出的数据的操作肯定不会更新query查询条件包含的字段 + */ + default Iterable> iterateBath(LambdaQueryWrapper query) { + return RecordIterable.iterateBath(this, DEFAULT_BATCH_SIZE, query); + } + + /** + * foreach遍历数据 + * + *

对查询出的数据的操作肯定不会更新query查询条件包含的字段 */ default Iterable> iterateBath(long pageSize, LambdaQueryWrapper query) { return RecordIterable.iterateBath(this, pageSize, query); } /** - * 遍历数据 - *

对查询出的数据的操作肯定不会更新query中查询条件包含的字段 + * foreach遍历数据. 默认批量大小 + *

对查询出的数据的操作肯定不会更新query查询条件包含的字段 + */ + default Iterable iterateElement(LambdaQueryWrapper query) { + return RecordIterable.iterateElement(this, DEFAULT_BATCH_SIZE, query); + } + + /** + * foreach遍历数据 + *

对查询出的数据的操作肯定不会更新query查询条件包含的字段 */ default Iterable iterateElement(long pageSize, LambdaQueryWrapper query) { return RecordIterable.iterateElement(this, pageSize, query); } /** - * 遍历数据 - *

对查询出的数据的操作肯定会更新query中查询条件包含的字段 - *

+ * foreach遍历数据. 默认批量大小 + * + *

对查询出的数据的操作肯定会更新query查询条件包含的字段 + * 

      * LambdaQueryWrapper query = Queries.query(MessageRecord.class)
      *        .eq(MessageRecord::getWorkspaceId, workspaceId)
      *        // 通过状态查询
      *        .eq(MessageRecord::getState, MsgStateEnum.UNSENT);
-     * for(Page page : iterateBathDoUpdateQueryFields(1000, query)) {
-     *     for(T record : page.getRecords()) {
+     * for(Page page : iterateBathDoUpdateQueryFields(1000, query)) {
+     *     for(MessageRecord record : page.getRecords()) {
      *         // 更新状态
      *         record.setState(MsgStateEnum.HAS_BEEN_SENT);
      *         messageRecordMapper.updateById(record);
      *     }
      * }
+     *
+ */ + default Iterable> iterateBathDoUpdateQueryFields(LambdaQueryWrapper query) { + return RecordIterable.iterateBathDoUpdateQueryFields(this, DEFAULT_BATCH_SIZE, query); + } + + /** + * foreach遍历数据 + * + *

对查询出的数据的操作肯定会更新query查询条件包含的字段 + * 

+     * LambdaQueryWrapper query = Queries.query(MessageRecord.class)
+     *        .eq(MessageRecord::getWorkspaceId, workspaceId)
+     *        // 通过状态查询
+     *        .eq(MessageRecord::getState, MsgStateEnum.UNSENT);
+     * for(Page page : iterateBathDoUpdateQueryFields(1000, query)) {
+     *     for(MessageRecord record : page.getRecords()) {
+     *         // 更新状态
+     *         record.setState(MsgStateEnum.HAS_BEEN_SENT);
+     *         messageRecordMapper.updateById(record);
+     *     }
+     * }
+     *
*/ default Iterable> iterateBathDoUpdateQueryFields(long pageSize, LambdaQueryWrapper query) { return RecordIterable.iterateBathDoUpdateQueryFields(this, pageSize, query); } /** - * 遍历数据 - *

对查询出的数据的操作肯定会更新query中查询条件包含的字段 - *

+ * foreach遍历数据. 默认批量大小 + * + *

对查询出的数据的操作肯定会更新query查询条件包含的字段 + * 

      * LambdaQueryWrapper query = Queries.query(MessageRecord.class)
      *        .eq(MessageRecord::getWorkspaceId, workspaceId)
      *        // 通过状态查询
      *        .eq(MessageRecord::getState, MsgStateEnum.UNSENT);
-     * for(T record : iterateElementDoUpdateQueryFields(1000, query)) {
+     * for(MessageRecord record : iterateElementDoUpdateQueryFields(1000, query)) {
      *     // 更新状态
      *     record.setState(MsgStateEnum.HAS_BEEN_SENT);
      *     messageRecordMapper.updateById(record);
      * }
+     *
+ */ + default Iterable iterateElementDoUpdateQueryFields(LambdaQueryWrapper query) { + return RecordIterable.iterateElementDoUpdateQueryFields(this, DEFAULT_BATCH_SIZE, query); + } + + /** + * foreach遍历数据 + * + *

对查询出的数据的操作肯定会更新query查询条件包含的字段 + * 

+     * LambdaQueryWrapper query = Queries.query(MessageRecord.class)
+     *        .eq(MessageRecord::getWorkspaceId, workspaceId)
+     *        // 通过状态查询
+     *        .eq(MessageRecord::getState, MsgStateEnum.UNSENT);
+     * for(MessageRecord record : iterateElementDoUpdateQueryFields(1000, query)) {
+     *     // 更新状态
+     *     record.setState(MsgStateEnum.HAS_BEEN_SENT);
+     *     messageRecordMapper.updateById(record);
+     * }
+     *
*/ default Iterable iterateElementDoUpdateQueryFields(long pageSize, LambdaQueryWrapper query) { return RecordIterable.iterateElementDoUpdateQueryFields(this, pageSize, query); diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java index a72f5954..ddb95e60 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java @@ -3,12 +3,15 @@ package cn.axzo.msg.center.util; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.springframework.beans.BeanUtils; import java.util.Collections; import java.util.Iterator; import java.util.NoSuchElementException; /** + * 不要直接使用这个类, 用 {@link IterableMapper} + * * @author yanglin */ class RecordIterable { @@ -26,13 +29,13 @@ class RecordIterable { static > Iterable> iterateBathDoUpdateQueryFields( M mapper, long pageSize, LambdaQueryWrapper query) { - return () -> new BatchIterator<>(mapper, pageSize, query, PageStrategy.WILL_UPDATE_QUERY_FIELDS); + return () -> new BatchIterator<>(mapper, pageSize, query, PageStrategy.DO_UPDATE_QUERY_FIELDS); } static > Iterable iterateElementDoUpdateQueryFields( M mapper, long pageSize, LambdaQueryWrapper query) { return () -> new ElementIterator<>( - new BatchIterator<>(mapper, pageSize, query, PageStrategy.WILL_UPDATE_QUERY_FIELDS)); + new BatchIterator<>(mapper, pageSize, query, PageStrategy.DO_UPDATE_QUERY_FIELDS)); } private static class ElementIterator> implements Iterator { @@ -58,10 +61,11 @@ class RecordIterable { } private void maybeAdvance() { - if (current == null || !current.hasNext() && batch.hasNext()) + if (current == null || !current.hasNext() && batch.hasNext()) { current = batch.hasNext() ? batch.next().getRecords().iterator() : Collections.emptyIterator(); + } } } @@ -87,7 +91,7 @@ class RecordIterable { public boolean hasNext() { maybeFetchNextPage(); // 1. 很明显的还有下一页 - // 2. 看当前页的数据消费情况 (第一页和最后一页) + // 2. 看当前页的数据以及当前页的消费情况 (第一页和最后一页) return page.getCurrent() < page.getPages() || (!pageConsumed && !page.getRecords().isEmpty()); } @@ -97,11 +101,8 @@ class RecordIterable { throw new NoSuchElementException(); pageConsumed = true; Page copy = new Page<>(); - copy.setRecords(page.getRecords()); - copy.setTotal(page.getTotal()); - copy.setSize(page.getSize()); - copy.setCurrent(page.getCurrent()); - copy.setOrders(page.orders()); + // 不能暴露内部状态. 如果外部改动了内部的page, 会影响到下次迭代 + BeanUtils.copyProperties(page, copy); return copy; } @@ -119,6 +120,6 @@ class RecordIterable { } private enum PageStrategy { - ADVANCE_CURRENT_PAGE, WILL_UPDATE_QUERY_FIELDS + ADVANCE_CURRENT_PAGE, DO_UPDATE_QUERY_FIELDS } } \ No newline at end of file diff --git a/msg-center-dal/src/main/resources/mapper/MessageBaseTemplateMapper.xml b/msg-center-dal/src/main/resources/mapper/MessageBaseTemplateMapper.xml new file mode 100644 index 00000000..c1fec104 --- /dev/null +++ b/msg-center-dal/src/main/resources/mapper/MessageBaseTemplateMapper.xml @@ -0,0 +1,9 @@ + + + + + + + \ No newline at end of file diff --git a/msg-center-dal/src/main/resources/mapper/Todo.xml b/msg-center-dal/src/main/resources/mapper/Todo.xml index 923bfd6e..15f69d7e 100644 --- a/msg-center-dal/src/main/resources/mapper/Todo.xml +++ b/msg-center-dal/src/main/resources/mapper/Todo.xml @@ -16,5 +16,9 @@ AND record_ext -> '$.isMigratedFromPendingMessage' = true + + diff --git a/start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java b/start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java new file mode 100644 index 00000000..9914197d --- /dev/null +++ b/start/src/test/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheReloadJobTest.java @@ -0,0 +1,24 @@ +package cn.axzo.msg.center.message.service.impl.oldmsg; + +import cn.axzo.msg.center.MsgCenterApplication; +import lombok.RequiredArgsConstructor; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author yanglin + */ +@SpringBootTest(classes = MsgCenterApplication.class) +@RequiredArgsConstructor(onConstructor_ = @Autowired) +class OldMsgStatCacheReloadJobTest { + private final OldMsgStatCacheReloadJob oldMsgStatCacheReloadJob; + + @Test + void foo() throws Exception { + oldMsgStatCacheReloadJob.execute(null); + } + +} \ No newline at end of file diff --git a/start/src/test/java/cn/axzo/msg/center/util/RecordIterableTest.java b/start/src/test/java/cn/axzo/msg/center/util/RecordIterableTest.java new file mode 100644 index 00000000..3b5423e4 --- /dev/null +++ b/start/src/test/java/cn/axzo/msg/center/util/RecordIterableTest.java @@ -0,0 +1,34 @@ +package cn.axzo.msg.center.util; + +import cn.axzo.msg.center.MsgCenterApplication; +import cn.axzo.msg.center.dal.mapper.MessageRecordMapper; +import cn.axzo.msg.center.domain.entity.MessageRecord; +import cn.axzo.msg.center.inside.notices.utils.Queries; +import lombok.RequiredArgsConstructor; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import static cn.axzo.msg.center.inside.notices.utils.Queries.query; +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author yanglin + */ +@SpringBootTest(classes = MsgCenterApplication.class) +@RequiredArgsConstructor(onConstructor_ = @Autowired) +class RecordIterableTest { + + private final MessageRecordMapper messageRecordMapper; + + @Test + void foo() { + Iterable records = messageRecordMapper + .iterateElement(1, query(MessageRecord.class) + .eq(MessageRecord::getId, 3)); + for (MessageRecord record : records) { + System.out.println("record=" + record); + } + } + +} \ No newline at end of file From 14882e994e11da6e76f5f3ed69699aaf4715a35a Mon Sep 17 00:00:00 2001 From: yanglin Date: Thu, 25 Apr 2024 18:34:53 +0800 Subject: [PATCH 08/33] REQ-2405: fix imports --- .../center/message/service/impl/GeneralMessageServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java index ccef3c32..a7bfc431 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/GeneralMessageServiceImpl.java @@ -15,6 +15,7 @@ import cn.axzo.msg.center.message.domain.dto.MessageTemplateRouterDTO; import cn.axzo.msg.center.message.domain.dto.SendImMessageDTO; import cn.axzo.msg.center.message.domain.vo.GeneralMessagePushVO; import cn.axzo.msg.center.message.service.GeneralMessageService; +import cn.axzo.msg.center.message.service.MessageSendTwiceRecordService; import cn.axzo.msg.center.message.service.MessageTemplateNewService; import cn.axzo.msg.center.message.service.impl.oldmsg.OldMsgStatCache; import cn.axzo.msg.center.service.dto.IdentityDTO; From 4405dee676a2272e676b4f87d17d97bbc8ba694e Mon Sep 17 00:00:00 2001 From: yanglin Date: Fri, 26 Apr 2024 09:21:10 +0800 Subject: [PATCH 09/33] =?UTF-8?q?REQ-2405:=20=E5=A4=87=E6=B3=A8=E4=BF=A1?= =?UTF-8?q?=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/cn/axzo/msg/center/util/IterableMapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java index d9f7c043..9e8bd78b 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/IterableMapper.java @@ -9,6 +9,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; */ public interface IterableMapper extends BaseMapper { + // 注意自己的分页插件有没有限制分页的最大条数 int DEFAULT_BATCH_SIZE = 2000; /** From 16081d590e2472357d73703b6251d884503d1923 Mon Sep 17 00:00:00 2001 From: yanglin Date: Fri, 26 Apr 2024 09:31:55 +0800 Subject: [PATCH 10/33] =?UTF-8?q?REQ-2405:=20=E6=B7=BB=E5=8A=A0=E4=B8=80?= =?UTF-8?q?=E4=BA=9B=E6=A0=87=E8=AF=86=E6=9D=A5=E6=8E=92=E6=9F=A5=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../message/service/impl/oldmsg/OldMsgStatCache.java | 9 ++++++++- .../service/impl/oldmsg/OldMsgStatCacheValue.java | 10 +++++++++- .../message/service/todo/cache/NodeStatCache.java | 9 ++++++++- .../GeneralMessageOldDataStatisticResponse.java | 2 ++ .../response/PendingMessageStatisticResponseV2.java | 2 ++ 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index 0a83bd5b..90936c1e 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -69,7 +69,13 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { public GeneralMessageOldDataStatisticResponse getCacheResponseOrReload(GeneralMessageOldDataStatisticRequest request) { GeneralMessageOldDataStatisticResponse resp = getCacheResponse(request); - return resp != null ? resp : reloadForeground(request); + if (resp != null) { + resp.setReloadForeground(false); + return resp; + } + GeneralMessageOldDataStatisticResponse response = reloadForeground(request); + response.setReloadForeground(true); + return response; } private GeneralMessageOldDataStatisticResponse getCacheResponse(GeneralMessageOldDataStatisticRequest request) { @@ -190,6 +196,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { // !! cache helper private void setCacheValue(Long personId, OldMsgStatCacheValue cacheValue) { + cacheValue.clear(); log.info("set cache value for person. personId={}", personId); long secondsDelta = TimeUnit.HOURS.toSeconds(cfg.getOldMsgStatCacheDataExpireHours()); // add some random delta seconds, range: 0-600 seconds diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java index 5900173d..d1b3030d 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java @@ -16,7 +16,7 @@ import java.util.Set; * @author yanglin */ @Data -public class OldMsgStatCacheValue { +class OldMsgStatCacheValue { private List identityResponses; @@ -35,6 +35,14 @@ public class OldMsgStatCacheValue { identityResponses.add(identityResponse); } + void clear() { + if (identityResponses == null) return; + for (IdentityResponse identityResponse : identityResponses) { + if (identityResponse.response != null) + identityResponse.response.setReloadForeground(null); + } + } + @Data public static class IdentityResponse { private GeneralMessageOldDataStatisticResponse response; diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java index 94ef1ced..4e056028 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java @@ -10,6 +10,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Component; @@ -43,6 +44,7 @@ public class NodeStatCache { Date templateLatestUpdateTime = messageBaseTemplateMapper.getTemplateLatestUpdateTime(); Date todoLatestUpdateTime = todoMapper.getTodoLatestUpdateTime(request.getPersonId()); NodeStatCacheValue cacheValue = getCacheValue(request); + boolean reloaded = false; if (cacheValue == null || cacheValue.shouldReload(templateLatestUpdateTime, todoLatestUpdateTime)) { log.info("reloading node stat data, request={}", request); PendingMessageStatisticResponseV2 response = loader.get(); @@ -51,8 +53,12 @@ public class NodeStatCache { cacheValue.setTemplateLatestUpdateTime(templateLatestUpdateTime); cacheValue.setTodoLatestUpdateTime(todoLatestUpdateTime); setCacheValue(request, cacheValue); + reloaded = true; } - return cacheValue.getResponse(); + PendingMessageStatisticResponseV2 copy = new PendingMessageStatisticResponseV2(); + BeanUtils.copyProperties(cacheValue.getResponse(), copy); + copy.setCacheReloaded(reloaded); + return copy; } // !! cache helper @@ -65,6 +71,7 @@ public class NodeStatCache { private void setCacheValue(MessageGroupNodeStatisticParam request, NodeStatCacheValue cacheValue) { log.info("set cache value for request. request={}", request); + cacheValue.getResponse().setCacheReloaded(null); long secondsDelta = TimeUnit.HOURS.toSeconds(cfg.getNodeStatCacheDataExpireHours()); // add some random delta seconds, range: 0-600 seconds secondsDelta += new SecureRandom().nextInt(10 * 60); diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/service/general/response/GeneralMessageOldDataStatisticResponse.java b/msg-center-api/src/main/java/cn/axzo/msg/center/service/general/response/GeneralMessageOldDataStatisticResponse.java index f864a589..5e4d6316 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/service/general/response/GeneralMessageOldDataStatisticResponse.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/service/general/response/GeneralMessageOldDataStatisticResponse.java @@ -38,6 +38,8 @@ public class GeneralMessageOldDataStatisticResponse implements Serializable { */ private String latestMsgContent; + private Boolean reloadForeground; + @Override public String toString() { return JSON.toJSONString(this); diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java b/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java index bacd1bb7..0c20ae40 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java @@ -37,6 +37,8 @@ public class PendingMessageStatisticResponseV2 implements Serializable { */ private List groupStats = Collections.synchronizedList(new ArrayList<>()); + private Boolean cacheReloaded; + public void addGroupStat(GroupStat groupStat) { this.groupStats.add(groupStat); } From 73cb855fead69834a438a49066d7983e81d92768 Mon Sep 17 00:00:00 2001 From: yanglin Date: Fri, 26 Apr 2024 10:03:19 +0800 Subject: [PATCH 11/33] =?UTF-8?q?REQ-2405:=20=E6=B7=BB=E5=8A=A0=E4=B8=80?= =?UTF-8?q?=E4=BA=9B=E6=A0=87=E8=AF=86=E6=9D=A5=E6=8E=92=E6=9F=A5=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/todo/cache/NodeStatCacheValue.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCacheValue.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCacheValue.java index 8d72daa2..c4e61858 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCacheValue.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCacheValue.java @@ -1,6 +1,7 @@ package cn.axzo.msg.center.message.service.todo.cache; import cn.axzo.msg.center.service.pending.response.PendingMessageStatisticResponseV2; +import cn.axzo.msg.center.utils.DateFormatUtil; import lombok.Data; import java.util.Date; @@ -14,6 +15,8 @@ class NodeStatCacheValue { private Date templateLatestUpdateTime; private Date todoLatestUpdateTime; + private String templateLatestUpdateTimeStr; + private String todoLatestUpdateTimeStr; private PendingMessageStatisticResponseV2 response; boolean shouldReload(Date templateLatestUpdateTime, Date todoLatestUpdateTime) { @@ -21,4 +24,14 @@ class NodeStatCacheValue { || !Objects.equals(this.todoLatestUpdateTime, todoLatestUpdateTime); } + public void setTemplateLatestUpdateTime(Date templateLatestUpdateTime) { + this.templateLatestUpdateTime = templateLatestUpdateTime; + this.templateLatestUpdateTimeStr = DateFormatUtil.toReadableString(templateLatestUpdateTime); + } + + public void setTodoLatestUpdateTime(Date todoLatestUpdateTime) { + this.todoLatestUpdateTime = todoLatestUpdateTime; + this.todoLatestUpdateTimeStr = DateFormatUtil.toReadableString(todoLatestUpdateTime); + } + } \ No newline at end of file From bc20d2e4d629604c6eaec1ad6134df2d39152c11 Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 13:51:47 +0800 Subject: [PATCH 12/33] =?UTF-8?q?REQ-2405:=20=E8=B0=83=E6=95=B4=E7=BC=93?= =?UTF-8?q?=E5=AD=98key=E5=89=8D=E7=BC=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msg/center/message/service/impl/oldmsg/OldMsgStatCache.java | 2 +- .../msg/center/message/service/todo/cache/NodeStatCache.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index 90936c1e..5eebace0 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -214,7 +214,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { } private static String buildCacheKey(Long personId) { - return String.format("msg_center:old_msg_stat:v1:%d", personId); + return String.format("msg-center:old_msg_stat:v1:%d", personId); } // !! 以下为原有的统计逻辑 diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java index 4e056028..042b320e 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java @@ -83,7 +83,7 @@ public class NodeStatCache { private static String buildCacheKey(MessageGroupNodeStatisticParam request) { StringBuffer buf = new StringBuffer(); - buf.append("msg_center:node_stat:v1"); + buf.append("msg-center:node_stat:v1"); Consumer appender = value -> { if (value != null) buf.append(":").append(value); From e8f8275f6c19f10d76971efd23f5238901a33876 Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 13:56:35 +0800 Subject: [PATCH 13/33] =?UTF-8?q?REQ-2405:=20=E8=B0=83=E6=95=B4=E7=BC=93?= =?UTF-8?q?=E5=AD=98key=E5=89=8D=E7=BC=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msg/center/message/service/impl/oldmsg/OldMsgStatCache.java | 2 +- .../msg/center/message/service/todo/cache/NodeStatCache.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index 5eebace0..a539bf4c 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -214,7 +214,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { } private static String buildCacheKey(Long personId) { - return String.format("msg-center:old_msg_stat:v1:%d", personId); + return String.format("msg-center:old-msg-stat:v1:%d", personId); } // !! 以下为原有的统计逻辑 diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java index 042b320e..37226b54 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java @@ -83,7 +83,7 @@ public class NodeStatCache { private static String buildCacheKey(MessageGroupNodeStatisticParam request) { StringBuffer buf = new StringBuffer(); - buf.append("msg-center:node_stat:v1"); + buf.append("msg-center:node-stat:v1"); Consumer appender = value -> { if (value != null) buf.append(":").append(value); From 0954805520a009fb3b2e14c145276885f8ee834c Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 14:45:08 +0800 Subject: [PATCH 14/33] =?UTF-8?q?REQ-2405:=20=E6=B7=BB=E5=8A=A0=E5=BC=80?= =?UTF-8?q?=E5=85=B3=E6=8E=A7=E5=88=B6=E6=98=AF=E5=90=A6=E8=B5=B0=E7=BC=93?= =?UTF-8?q?=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/PendingMessageBizConfig.java | 20 ++++++++++++++++++- .../service/impl/oldmsg/OldMsgStatCache.java | 3 +++ .../service/todo/cache/NodeStatCache.java | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java index 0fedeea5..5396344e 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java @@ -110,14 +110,32 @@ public class PendingMessageBizConfig { return dontConcatRouterParamsTemplateCodes == null || !dontConcatRouterParamsTemplateCodes.contains(templateCode); } + // !! msg-center MQ消息自产自消 + + @Getter + private long msgCenterMqSelfConsumeMaxExecMs = 20000L; + + // !! 老消息缓存 + + /** + * 是否开启老消息走否走缓存 + */ + @Getter + private boolean oldMsgStatCacheOn = true; + @Getter private int oldMsgStatCacheDataExpireHours = 8; @Getter private int oldMsgStateDataBackendUpdateBatchSize = 100; + // !! 待办分类统计缓存 + + /** + * 待办分类统计是否走缓存 + */ @Getter - private long msgCenterMqSelfConsumeMaxExecMs = 20000L; + private boolean nodeStatCacheOn = true; @Getter private int nodeStatCacheMaxRequestNodeCodeSize = 10; diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index a539bf4c..af957e17 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -68,6 +68,9 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { private final ForkJoinPool backgroundExecutor = new ForkJoinPool(10); public GeneralMessageOldDataStatisticResponse getCacheResponseOrReload(GeneralMessageOldDataStatisticRequest request) { + if (!cfg.isOldMsgStatCacheOn()) { + return statisticOldDataImpl(request); + } GeneralMessageOldDataStatisticResponse resp = getCacheResponse(request); if (resp != null) { resp.setReloadForeground(false); diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java index 37226b54..a3fbbecc 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java @@ -36,6 +36,9 @@ public class NodeStatCache { public PendingMessageStatisticResponseV2 getCacheResponseOrReload( MessageGroupNodeStatisticParam request, Supplier loader) { + if (!cfg.isNodeStatCacheOn()) { + return loader.get(); + } Collection nodeCodes = request.getGroupNodeCodes(); if (CollectionUtils.isNotEmpty(nodeCodes) && nodeCodes.size() > cfg.getNodeStatCacheMaxRequestNodeCodeSize()) { log.info("超过了允许缓存的最大请求节点编码数量，不使用缓存, 直接从数据库获取. request={}", request); From 465a5b0f802f6289ad6fc872658d7a8b180dc088 Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 15:40:27 +0800 Subject: [PATCH 15/33] =?UTF-8?q?REQ-2405:=20=E6=B7=BB=E5=8A=A0=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E5=85=BC=E5=AE=B9=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../message/service/impl/oldmsg/OldMsgStatCache.java | 11 ++++++++++- .../message/service/todo/cache/NodeStatCache.java | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index af957e17..d6af9748 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -213,7 +213,16 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { private OldMsgStatCacheValue getCacheValue(Long personId) { String redisKey = buildCacheKey(personId); String json = stringRedisTemplate.opsForValue().get(redisKey); - return StringUtils.isBlank(json) ? null : JSON.parseObject(json, OldMsgStatCacheValue.class); + if (StringUtils.isBlank(json)) { + return null; + } + try { + return JSON.parseObject(json, OldMsgStatCacheValue.class); + } catch (Exception e) { + // 人为改了缓存, 而且没有改对 + log.warn("fail to parse cache value, will reload. personId={}, cacheValue={}", personId, json, e); + return null; + } } private static String buildCacheKey(Long personId) { diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java index a3fbbecc..c36b79f3 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java @@ -69,7 +69,16 @@ public class NodeStatCache { private NodeStatCacheValue getCacheValue(MessageGroupNodeStatisticParam request) { String redisKey = buildCacheKey(request); String json = stringRedisTemplate.opsForValue().get(redisKey); - return StringUtils.isBlank(json) ? null : JSON.parseObject(json, NodeStatCacheValue.class); + if (StringUtils.isBlank(json)) { + return null; + } + try { + return JSON.parseObject(json, NodeStatCacheValue.class); + } catch (Exception e) { + // 人为改了缓存, 而且没有改对 + log.warn("fail to parse cache value, will reload. request={}, cacheValue={}", request, json, e); + return null; + } } private void setCacheValue(MessageGroupNodeStatisticParam request, NodeStatCacheValue cacheValue) { From f9e28b38df273f2e21e55f2e023a2fd142c649f7 Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 16:10:50 +0800 Subject: [PATCH 16/33] =?UTF-8?q?REQ-2405:=20=E4=BF=AE=E6=AD=A3=E8=AE=BF?= =?UTF-8?q?=E9=97=AE=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../message/service/impl/oldmsg/OldMsgStatCacheValue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java index d1b3030d..8384fc9d 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCacheValue.java @@ -16,7 +16,7 @@ import java.util.Set; * @author yanglin */ @Data -class OldMsgStatCacheValue { +public class OldMsgStatCacheValue { private List identityResponses; From b4162f80ae95210662fb1c39e1bff136ecfacf1c Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 16:36:18 +0800 Subject: [PATCH 17/33] =?UTF-8?q?REQ-2405:=20=E8=B0=83=E6=95=B4=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=9A=84=E8=BF=87=E6=9C=9F=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../center/inside/notices/config/PendingMessageBizConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java index 5396344e..b5efae54 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java @@ -124,7 +124,7 @@ public class PendingMessageBizConfig { private boolean oldMsgStatCacheOn = true; @Getter - private int oldMsgStatCacheDataExpireHours = 8; + private int oldMsgStatCacheDataExpireHours = 16; @Getter private int oldMsgStateDataBackendUpdateBatchSize = 100; From 46184636b91e486e082e6b352e594e7deba1cd6e Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 17:30:43 +0800 Subject: [PATCH 18/33] =?UTF-8?q?REQ-2405:=20=E5=A4=84=E7=90=86cms?= =?UTF-8?q?=E7=9A=84=E8=80=81=E6=B6=88=E6=81=AF=E8=AF=BB=E4=BA=8B=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/MessageRecordServiceImpl.java | 5 +++++ .../service/impl/oldmsg/OldMsgStatCache.java | 13 +++++++++++++ .../msg/center/api/mq/SendMessageRecordMessage.java | 2 ++ .../axzo/msg/center/api/request/CmsReadMsgReq.java | 5 +++++ .../msg/center/api/request/GeneralMessageReq.java | 6 ++++++ 5 files changed, 31 insertions(+) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java index aad42973..1a558bd6 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/service/impl/MessageRecordServiceImpl.java @@ -443,6 +443,11 @@ public class MessageRecordServiceImpl implements MessageRecordService { } else { messageRecordDao.readMsg(request, req.getPersonId(), req.getIdentityId()); } + SendMessageRecordMessage mqMessage = new SendMessageRecordMessage(); + mqMessage.setCmsReadRequest(req); + mqProducer.send(MqMessageRecord + .builder(MqMessageType.OLD_MSG_SEND, mqMessage) + .build()); } /** diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index d6af9748..f5eb007e 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -5,6 +5,7 @@ import cn.axzo.framework.rocketmq.EventConsumer; import cn.axzo.framework.rocketmq.EventHandler; import cn.axzo.msg.center.api.mq.SendMessageRecordMessage; import cn.axzo.msg.center.api.request.CmsMsgQueryReq; +import cn.axzo.msg.center.api.request.CmsReadMsgReq; import cn.axzo.msg.center.api.request.GeneralMessageReq; import cn.axzo.msg.center.api.request.MessageReadAllReq; import cn.axzo.msg.center.api.response.MessageNewRes; @@ -296,12 +297,16 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { SendMessageRecordMessage mqMessage = event.normalizedData(SendMessageRecordMessage.class); GeneralMessageReq sendRequest = mqMessage.getSendRequest(); MessageReadAllReq readRequest = mqMessage.getReadRequest(); + CmsReadMsgReq cmsReadRequest = mqMessage.getCmsReadRequest(); if (sendRequest != null) { log.info("handle mq event - driven by sendRequest={}", sendRequest); handleSendRequestEvent(sendRequest); } else if (readRequest != null) { log.info("handle mq event - driven by readRequest={}", readRequest); handleReadRequest(readRequest); + } else if (cmsReadRequest != null) { + log.info("handle mq event - driven by cmsReadRequest={}", cmsReadRequest); + handleCmsReadRequest(cmsReadRequest); } } @@ -337,4 +342,12 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { log.info(stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); } + private void handleCmsReadRequest(CmsReadMsgReq cmsReadRequest) throws Exception { + StopWatch stopWatch = new StopWatch("reloadBackground"); + stopWatch.start("handleCmsReadRequest"); + reloadBackground(Sets.newHashSet(cmsReadRequest.getPersonId())); + stopWatch.stop(); + log.info(stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); + } + } \ No newline at end of file diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java b/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java index c507168d..835e8746 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/api/mq/SendMessageRecordMessage.java @@ -1,5 +1,6 @@ package cn.axzo.msg.center.api.mq; +import cn.axzo.msg.center.api.request.CmsReadMsgReq; import cn.axzo.msg.center.api.request.GeneralMessageReq; import cn.axzo.msg.center.api.request.MessageReadAllReq; import lombok.Data; @@ -15,4 +16,5 @@ import java.io.Serializable; public class SendMessageRecordMessage extends MqMessage implements Serializable { private GeneralMessageReq sendRequest; private MessageReadAllReq readRequest; + private CmsReadMsgReq cmsReadRequest; } \ No newline at end of file diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/CmsReadMsgReq.java b/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/CmsReadMsgReq.java index 37cdd9b9..35dc962b 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/CmsReadMsgReq.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/CmsReadMsgReq.java @@ -1,6 +1,7 @@ package cn.axzo.msg.center.api.request; import cn.axzo.msg.center.api.enums.BizTypeEnum; +import com.alibaba.fastjson.JSON; import lombok.Data; import javax.validation.constraints.NotNull; @@ -29,4 +30,8 @@ public class CmsReadMsgReq { */ private transient BizTypeEnum bizTypeEnum = BizTypeEnum.CONSTRUCTION; + @Override + public String toString() { + return JSON.toJSONString(this); + } } diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/GeneralMessageReq.java b/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/GeneralMessageReq.java index 430554af..fc3ed358 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/GeneralMessageReq.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/api/request/GeneralMessageReq.java @@ -2,6 +2,7 @@ package cn.axzo.msg.center.api.request; import cn.axzo.msg.center.api.enums.MsgRecordTerminalTypeEnum; import cn.axzo.msg.center.api.enums.ReceiveTypeEnum; +import com.alibaba.fastjson.JSON; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -77,5 +78,10 @@ public class GeneralMessageReq extends AbstractMessage implements Serializable { * appClient: e.g. cmp cm */ public String appClient; + + @Override + public String toString() { + return JSON.toJSONString(this); + } } From d2ca950d274fb7455cc0f8755f6ab5c346233153 Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 18:39:18 +0800 Subject: [PATCH 19/33] =?UTF-8?q?REQ-2405:=20=E5=A4=84=E7=90=86=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E5=88=86=E7=B1=BB=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../center/message/service/todo/TodoRangeQueryService.java | 7 ++++++- .../center/message/service/todo/cache/NodeStatCache.java | 2 +- .../response/PendingMessageStatisticResponseV2.java | 7 ++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java index 5ff148bf..cb3c57c5 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java @@ -253,7 +253,12 @@ public class TodoRangeQueryService { .get(); for (GroupStat stat : stats) resp.addGroupStat(stat); + // 一个端可能会有重复分类 resp.setTotalStat(); + // 一个端不包含重复分类 + int uniqueExecutableCount = countStatByNodes(request, nodes, TodoType.EXECUTABLE); + int uniqueCopiedToMeCount = countStatByNodes(request, nodes, TodoType.COPIED_TO_ME); + resp.setUniqueTotalStat(new Stat(uniqueExecutableCount, uniqueCopiedToMeCount)); return resp; } catch (Exception e) { log.warn("分类统计异常, request={}", request, e); @@ -267,7 +272,7 @@ public class TodoRangeQueryService { public Integer countStat(MessageGroupNodeStatisticParam request) { PendingMessageStatisticResponseV2 nodeStat = nodeStatCache .getCacheResponseOrReload(request, () -> countStatGroupedImpl(request)); - Stat totalStat = nodeStat.getTotalStat(); + Stat totalStat = nodeStat.getUniqueTotalStat(); return totalStat == null ? 0 : totalStat.getPendingCount(); } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java index c36b79f3..c13baac9 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/cache/NodeStatCache.java @@ -95,7 +95,7 @@ public class NodeStatCache { private static String buildCacheKey(MessageGroupNodeStatisticParam request) { StringBuffer buf = new StringBuffer(); - buf.append("msg-center:node-stat:v1"); + buf.append("msg-center:node-stat:v2"); Consumer appender = value -> { if (value != null) buf.append(":").append(value); diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java b/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java index 0c20ae40..1c3d7f5e 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java @@ -28,10 +28,15 @@ public class PendingMessageStatisticResponseV2 implements Serializable { private static final long serialVersionUID = 972200410809186003L; /** - * 统计情况总数 (各分类相加) + * 统计总数 (各分类相加) */ private Stat totalStat; + /** + * 统计总数 (一个端分类配置重复了也没有关系) + */ + private Stat uniqueTotalStat; + /** * 分类情况 */ From 2d97bb4b3686c35aef5cc5f7ed467f6fc11460f9 Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 18:47:37 +0800 Subject: [PATCH 20/33] =?UTF-8?q?REQ-2405:=20=E5=A4=84=E7=90=86=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E5=88=86=E7=B1=BB=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../message/service/todo/TodoRangeQueryService.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java index cb3c57c5..522e7b32 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java @@ -46,6 +46,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ForkJoinPool; import java.util.function.Function; import java.util.function.Supplier; @@ -247,7 +248,13 @@ public class TodoRangeQueryService { }; try { PendingMessageStatisticResponseV2 resp = new PendingMessageStatisticResponseV2(); + // 先提交任务 + CompletableFuture uniqueExecutableCountFuture = CompletableFuture.supplyAsync( + () -> countStatByNodes(request, nodes, TodoType.EXECUTABLE)); + CompletableFuture uniqueCopiedToMeCountFuture = CompletableFuture.supplyAsync( + () -> countStatByNodes(request, nodes, TodoType.COPIED_TO_ME)); // 用一个偏大的线程池, IO密集型任务, commonPool不够用 + // 提交任务并获取结果 List stats = statExecutor .submit(() -> nodes.parallelStream().map(nodeStatFun).collect(toList())) .get(); @@ -255,9 +262,10 @@ public class TodoRangeQueryService { resp.addGroupStat(stat); // 一个端可能会有重复分类 resp.setTotalStat(); + CompletableFuture.allOf(uniqueExecutableCountFuture, uniqueCopiedToMeCountFuture).join(); // 一个端不包含重复分类 - int uniqueExecutableCount = countStatByNodes(request, nodes, TodoType.EXECUTABLE); - int uniqueCopiedToMeCount = countStatByNodes(request, nodes, TodoType.COPIED_TO_ME); + int uniqueExecutableCount = uniqueExecutableCountFuture.get(); + int uniqueCopiedToMeCount = uniqueCopiedToMeCountFuture.get(); resp.setUniqueTotalStat(new Stat(uniqueExecutableCount, uniqueCopiedToMeCount)); return resp; } catch (Exception e) { From 3c71b4f75154214f3d58e52b76f4ec0147a294d6 Mon Sep 17 00:00:00 2001 From: yanglin Date: Sun, 28 Apr 2024 18:53:33 +0800 Subject: [PATCH 21/33] =?UTF-8?q?REQ-2405:=20=E5=A4=84=E7=90=86=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E5=88=86=E7=B1=BB=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../center/message/service/todo/TodoRangeQueryService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java index 522e7b32..1439ddbf 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java @@ -250,9 +250,9 @@ public class TodoRangeQueryService { PendingMessageStatisticResponseV2 resp = new PendingMessageStatisticResponseV2(); // 先提交任务 CompletableFuture uniqueExecutableCountFuture = CompletableFuture.supplyAsync( - () -> countStatByNodes(request, nodes, TodoType.EXECUTABLE)); + () -> countStatByNodes(request, nodes, TodoType.EXECUTABLE), statExecutor); CompletableFuture uniqueCopiedToMeCountFuture = CompletableFuture.supplyAsync( - () -> countStatByNodes(request, nodes, TodoType.COPIED_TO_ME)); + () -> countStatByNodes(request, nodes, TodoType.COPIED_TO_ME), statExecutor); // 用一个偏大的线程池, IO密集型任务, commonPool不够用 // 提交任务并获取结果 List stats = statExecutor From 37496f51388498fe6229ff0a0d12fc5cf287e4db Mon Sep 17 00:00:00 2001 From: yanglin Date: Mon, 29 Apr 2024 08:48:42 +0800 Subject: [PATCH 22/33] =?UTF-8?q?REQ-2405:=20=E5=A4=84=E7=90=86=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E5=88=86=E7=B1=BB=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../message/service/todo/TodoRangeQueryService.java | 8 +++++--- .../response/PendingMessageStatisticResponseV2.java | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java index 1439ddbf..65f41226 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/TodoRangeQueryService.java @@ -256,10 +256,12 @@ public class TodoRangeQueryService { // 用一个偏大的线程池, IO密集型任务, commonPool不够用 // 提交任务并获取结果 List stats = statExecutor - .submit(() -> nodes.parallelStream().map(nodeStatFun).collect(toList())) + .submit(() -> nodes + .parallelStream() + .map(nodeStatFun) + .collect(toList())) .get(); - for (GroupStat stat : stats) - resp.addGroupStat(stat); + resp.addGroupStats(stats); // 一个端可能会有重复分类 resp.setTotalStat(); CompletableFuture.allOf(uniqueExecutableCountFuture, uniqueCopiedToMeCountFuture).join(); diff --git a/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java b/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java index 1c3d7f5e..55f6d19b 100644 --- a/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java +++ b/msg-center-api/src/main/java/cn/axzo/msg/center/service/pending/response/PendingMessageStatisticResponseV2.java @@ -44,6 +44,10 @@ public class PendingMessageStatisticResponseV2 implements Serializable { private Boolean cacheReloaded; + public void addGroupStats(List groupStats) { + this.groupStats.addAll(groupStats); + } + public void addGroupStat(GroupStat groupStat) { this.groupStats.add(groupStat); } From df68d95fad7557703cb84e41e3f2763fe1447638 Mon Sep 17 00:00:00 2001 From: yanglin Date: Mon, 29 Apr 2024 15:18:59 +0800 Subject: [PATCH 23/33] =?UTF-8?q?REQ-2405:=20=E8=B0=83=E6=95=B4=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=9A=84=E8=BF=87=E6=9C=9F=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../center/inside/notices/config/PendingMessageBizConfig.java | 4 ++++ .../center/message/service/impl/oldmsg/OldMsgStatCache.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java index b5efae54..41df2dd1 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/inside/notices/config/PendingMessageBizConfig.java @@ -143,6 +143,10 @@ public class PendingMessageBizConfig { @Getter private int nodeStatCacheDataExpireHours = 8; + public boolean determineOldMsgStatCacheOn() { + return isOldMsgStatCacheOn(); + } + public boolean hasMessageDetailStyle(String code) { return name2DetailStyle != null && name2DetailStyle.containsKey(code); } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java index f5eb007e..1f1cfc6d 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/impl/oldmsg/OldMsgStatCache.java @@ -69,7 +69,7 @@ public class OldMsgStatCache implements EventHandler, InitializingBean { private final ForkJoinPool backgroundExecutor = new ForkJoinPool(10); public GeneralMessageOldDataStatisticResponse getCacheResponseOrReload(GeneralMessageOldDataStatisticRequest request) { - if (!cfg.isOldMsgStatCacheOn()) { + if (!cfg.determineOldMsgStatCacheOn()) { return statisticOldDataImpl(request); } GeneralMessageOldDataStatisticResponse resp = getCacheResponse(request); From ffd688c7ad5573c7db25b014f6ff7612bb57e76b Mon Sep 17 00:00:00 2001 From: yanglin Date: Tue, 30 Apr 2024 15:15:30 +0800 Subject: [PATCH 24/33] =?UTF-8?q?REQ-2405:=20=E5=8A=A8=E6=80=81=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E8=A1=A8=E5=90=8D=E7=9A=84mybatis=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=20-=2070%?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inside-notices/pom.xml | 5 + .../todo/mybatis/ReplaceInterceptor.java | 181 ++++++++++++++++++ .../service/todo/mybatis/Replacement.java | 22 +++ .../todo/mybatis/SqlInterceptorConfig.java | 5 + .../axzo/msg/center/util/RecordIterable.java | 18 +- 5 files changed, 222 insertions(+), 9 deletions(-) create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/ReplaceInterceptor.java create mode 100644 inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/Replacement.java diff --git a/inside-notices/pom.xml b/inside-notices/pom.xml index c5063d08..27c331f0 100644 --- a/inside-notices/pom.xml +++ b/inside-notices/pom.xml @@ -21,6 +21,11 @@ + + org.mybatis + mybatis + 3.5.6 + com.alibaba druid diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/ReplaceInterceptor.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/ReplaceInterceptor.java new file mode 100644 index 00000000..1f107b85 --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/ReplaceInterceptor.java @@ -0,0 +1,181 @@ +package cn.axzo.msg.center.message.service.todo.mybatis; + +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; +import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement; +import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; +import com.alibaba.druid.sql.visitor.SQLASTVisitor; +import lombok.RequiredArgsConstructor; +import org.apache.ibatis.cache.CacheKey; +import org.apache.ibatis.executor.Executor; +import org.apache.ibatis.mapping.BoundSql; +import org.apache.ibatis.mapping.MappedStatement; +import org.apache.ibatis.mapping.ParameterMapping; +import org.apache.ibatis.mapping.SqlCommandType; +import org.apache.ibatis.mapping.SqlSource; +import org.apache.ibatis.plugin.Interceptor; +import org.apache.ibatis.plugin.Intercepts; +import org.apache.ibatis.plugin.Invocation; +import org.apache.ibatis.plugin.Signature; +import org.apache.ibatis.session.Configuration; +import org.apache.ibatis.session.ResultHandler; +import org.apache.ibatis.session.RowBounds; +import org.springframework.util.ReflectionUtils; + +import java.lang.reflect.Field; +import java.util.List; + +/** + * @author yanglin + */ +@Intercepts({ + @Signature(type = Executor.class, method = "update", + args = {MappedStatement.class, Object.class}), + @Signature(type = Executor.class, method = "query", + args = {MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class}), + @Signature(type = Executor.class, method = "query", + args = {MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class, CacheKey.class, BoundSql.class}), +}) +public class ReplaceInterceptor implements Interceptor { + + private static final ThreadLocal LOCAL = new ThreadLocal<>(); + + public static void enableReplacement(Replacement replacement) { + LOCAL.set(replacement); + } + + public static void disableReplacement() { + LOCAL.remove(); + } + + @Override + public Object intercept(Invocation invocation) throws Throwable { + Replacement replacement = LOCAL.get(); + if (replacement == null) + return invocation.proceed(); + final Object[] args = invocation.getArgs(); + MappedStatement ms = (MappedStatement) args[0]; + SqlCommandType commandType = ms.getSqlCommandType(); + BoundSql boundSql = ms.getBoundSql(args[1]); + SQLStatement stmt; + MySqlStatementParser parser = new MySqlStatementParser(boundSql.getSql()); + if (commandType == SqlCommandType.UPDATE) { + stmt = parser.parseUpdateStatement(); + } else if (commandType == SqlCommandType.SELECT) { + stmt = parser.parseSelect(); + } else if (commandType == SqlCommandType.INSERT) { + stmt = parser.parseInsert(); + } else { + return invocation.proceed(); + } + if (stmt instanceof MySqlInsertStatement && replacement.isInsertPreserveId()) { + maybeInsertPreserveId((MySqlInsertStatement) stmt, boundSql.getParameterObject()); + } + stmt.accept(new ReplaceVisitor(replacement)); + String newSql = stmt.toString(); + args[0] = copyMappedStatement(ms, parameterObject -> new CustomBoundSql( + ms.getConfiguration(), newSql, boundSql.getParameterMappings(), parameterObject, boundSql)); + return invocation.proceed(); + } + + private void maybeInsertPreserveId(MySqlInsertStatement stmt, Object parameterObject) { + if (stmt.getColumns().isEmpty() || stmt.getValues() == null) { + return; + } + List columns = stmt.getColumns(); + String idColumn = columns.stream() + .filter(column -> column instanceof SQLIdentifierExpr) + .map(SQLIdentifierExpr.class::cast) + .map(SQLIdentifierExpr::getName) + .filter(name -> name.equals("id")) + .findFirst() + .orElse(null); + if (idColumn != null) return; + Number idValue = getIdValue(parameterObject); + if (idValue == null) return; + columns.add(new SQLIdentifierExpr("id")); + stmt.getValues().addValue(idValue.longValue()); + } + + private Number getIdValue(Object parameterObject) { + Field field = ReflectionUtils.findField(parameterObject.getClass(), "id"); + if (field == null) return null; + field.setAccessible(true); + Object value = ReflectionUtils.getField(field, parameterObject); + return value instanceof Number ? (Number) value : null; + } + + @RequiredArgsConstructor + private static class ReplaceVisitor implements SQLASTVisitor { + + final Replacement replacement; + + @Override + public boolean visit(SQLExprTableSource x) { + SQLExpr expr = x.getExpr(); + if (expr instanceof SQLIdentifierExpr) { + String tableName = ((SQLIdentifierExpr) expr).getName(); + if (tableName.equals(replacement.getSrcTable())) { + x.replace(expr, new SQLIdentifierExpr(replacement.getDestTable())); + } + } + return false; + } + + } + + private static class CustomBoundSql extends BoundSql { + + final BoundSql delegate; + + CustomBoundSql(Configuration configuration, String sql, List parameterMappings, + Object parameterObject, BoundSql delegate) { + super(configuration, sql, parameterMappings, parameterObject); + this.delegate = delegate; + } + + @Override + public boolean hasAdditionalParameter(String name) { + if (super.hasAdditionalParameter(name)) + return true; + return delegate.hasAdditionalParameter(name); + } + + @Override + public Object getAdditionalParameter(String name) { + Object parameter = super.getAdditionalParameter(name); + if (parameter != null) + return parameter; + return delegate.getAdditionalParameter(name); + } + + } + + private MappedStatement copyMappedStatement(MappedStatement ms, SqlSource sqlSource) { + MappedStatement.Builder builder = + new MappedStatement.Builder(ms.getConfiguration(), ms.getId(), sqlSource, ms.getSqlCommandType()); + builder.resource(ms.getResource()); + builder.fetchSize(ms.getFetchSize()); + builder.statementType(ms.getStatementType()); + builder.keyGenerator(ms.getKeyGenerator()); + if (ms.getKeyProperties() != null && ms.getKeyProperties().length != 0) { + StringBuilder keyProperties = new StringBuilder(); + for (String keyProperty : ms.getKeyProperties()) { + keyProperties.append(keyProperty).append(","); + } + keyProperties.delete(keyProperties.length() - 1, keyProperties.length()); + builder.keyProperty(keyProperties.toString()); + } + builder.timeout(ms.getTimeout()); + builder.parameterMap(ms.getParameterMap()); + builder.resultMaps(ms.getResultMaps()); + builder.resultSetType(ms.getResultSetType()); + builder.cache(ms.getCache()); + builder.flushCacheRequired(ms.isFlushCacheRequired()); + builder.useCache(ms.isUseCache()); + return builder.build(); + } + +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/Replacement.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/Replacement.java new file mode 100644 index 00000000..856d68ae --- /dev/null +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/Replacement.java @@ -0,0 +1,22 @@ +package cn.axzo.msg.center.message.service.todo.mybatis; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * @author yanglin + */ +@Getter +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public enum Replacement { + + TO_MESSAGE_RECORD_COLD("message_record", "message_record_cold", true) + + ; + + private final String srcTable; + private final String destTable; + private final boolean insertPreserveId; + +} \ No newline at end of file diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/SqlInterceptorConfig.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/SqlInterceptorConfig.java index c4d840c2..bd6e4cd3 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/SqlInterceptorConfig.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/SqlInterceptorConfig.java @@ -12,6 +12,11 @@ import org.springframework.context.annotation.Configuration; @Configuration public class SqlInterceptorConfig { + @Bean + ReplaceInterceptor replaceTableInterceptor() { + return new ReplaceInterceptor(); + } + @Bean CollectSQLInterceptor sqlCollectInterceptor() { return new CollectSQLInterceptor(); diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java index ddb95e60..5ee4b65c 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/util/RecordIterable.java @@ -40,30 +40,30 @@ class RecordIterable { private static class ElementIterator> implements Iterator { - private final BatchIterator batch; - private Iterator current; + private final BatchIterator pageIterator; + private Iterator elementIterator; - ElementIterator(BatchIterator batchIterator) { - this.batch = batchIterator; + ElementIterator(BatchIterator pageIterator) { + this.pageIterator = pageIterator; } @Override public boolean hasNext() { maybeAdvance(); - return current.hasNext() || batch.hasNext(); + return elementIterator.hasNext() || pageIterator.hasNext(); } @Override public T next() { if (!hasNext()) throw new NoSuchElementException(); - return current.next(); + return elementIterator.next(); } private void maybeAdvance() { - if (current == null || !current.hasNext() && batch.hasNext()) { - current = batch.hasNext() - ? batch.next().getRecords().iterator() + if (elementIterator == null || !elementIterator.hasNext() && pageIterator.hasNext()) { + elementIterator = pageIterator.hasNext() + ? pageIterator.next().getRecords().iterator() : Collections.emptyIterator(); } } From 8a452b84399ac9dd386a9525186aa61419168e76 Mon Sep 17 00:00:00 2001 From: yanglin Date: Tue, 30 Apr 2024 17:24:45 +0800 Subject: [PATCH 25/33] REQ-2405: fix bugs --- .../todo/mybatis/ReplaceInterceptor.java | 6 ++- .../service/todo/mybatis/Replacement.java | 8 +++ .../dal/mapper/MessageRecordMapper.java | 1 + .../resources/mapper/MessageRecordMapper.xml | 8 +++ .../dal/mapper/MessageRecordMapperTest.java | 50 +++++++++++++++++++ 5 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 start/src/test/java/cn/axzo/msg/center/dal/mapper/MessageRecordMapperTest.java diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/ReplaceInterceptor.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/ReplaceInterceptor.java index 1f107b85..85a39e75 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/ReplaceInterceptor.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/ReplaceInterceptor.java @@ -42,11 +42,11 @@ public class ReplaceInterceptor implements Interceptor { private static final ThreadLocal LOCAL = new ThreadLocal<>(); - public static void enableReplacement(Replacement replacement) { + public static void enable(Replacement replacement) { LOCAL.set(replacement); } - public static void disableReplacement() { + public static void disable() { LOCAL.remove(); } @@ -67,6 +67,8 @@ public class ReplaceInterceptor implements Interceptor { stmt = parser.parseSelect(); } else if (commandType == SqlCommandType.INSERT) { stmt = parser.parseInsert(); + } else if (commandType == SqlCommandType.DELETE) { + stmt = parser.parseDeleteStatement(); } else { return invocation.proceed(); } diff --git a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/Replacement.java b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/Replacement.java index 856d68ae..bfc066bd 100644 --- a/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/Replacement.java +++ b/inside-notices/src/main/java/cn/axzo/msg/center/message/service/todo/mybatis/Replacement.java @@ -19,4 +19,12 @@ public enum Replacement { private final String destTable; private final boolean insertPreserveId; + public void run(Runnable runnable) { + ReplaceInterceptor.enable(this); + try { + runnable.run(); + } finally { + ReplaceInterceptor.disable(); + } + } } \ No newline at end of file diff --git a/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageRecordMapper.java b/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageRecordMapper.java index 8835d2d0..f132d8e6 100644 --- a/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageRecordMapper.java +++ b/msg-center-dal/src/main/java/cn/axzo/msg/center/dal/mapper/MessageRecordMapper.java @@ -24,6 +24,7 @@ import java.util.List; /*@Mapper*/ public interface MessageRecordMapper extends IterableMapper { + void deleteRecordsPhysically(@Param("ids") List ids); List statisticsMsg(@Param("bizType") Integer bizType, @Param("personId") Long personId, diff --git a/msg-center-dal/src/main/resources/mapper/MessageRecordMapper.xml b/msg-center-dal/src/main/resources/mapper/MessageRecordMapper.xml index 9f580563..96b89925 100644 --- a/msg-center-dal/src/main/resources/mapper/MessageRecordMapper.xml +++ b/msg-center-dal/src/main/resources/mapper/MessageRecordMapper.xml @@ -2,6 +2,14 @@ + + DELETE FROM message_record + WHERE id IN + + #{id} + + +