From a3b0141f75dffbcb8e8c75ecdb080cd3c0d65b29 Mon Sep 17 00:00:00 2001 From: pepsi Date: Wed, 28 Sep 2022 19:52:55 +0800 Subject: [PATCH] operate log modify --- .../controller/api/OperateLogController.java | 28 +++++-- .../server/dto/OperateLogQueryReqDTO.java | 7 +- .../xlog/server/dto/OperateLogReqDTO.java | 1 + .../xlog/server/service/BaseEsService.java | 54 ++++++++++++- .../service/impl/OperateLogServiceImpl.java | 77 +++++++++++++++---- .../xlog/server/XlogApplicationTestBase.java | 2 +- .../config/ElasticClientConfigTest.java | 57 ++++++++++++++ 7 files changed, 201 insertions(+), 25 deletions(-) diff --git a/xlog-server/src/main/java/cn/axzo/xlog/server/controller/api/OperateLogController.java b/xlog-server/src/main/java/cn/axzo/xlog/server/controller/api/OperateLogController.java index 507add3..fb5a465 100644 --- a/xlog-server/src/main/java/cn/axzo/xlog/server/controller/api/OperateLogController.java +++ b/xlog-server/src/main/java/cn/axzo/xlog/server/controller/api/OperateLogController.java @@ -3,15 +3,16 @@ package cn.axzo.xlog.server.controller.api; import cn.axzo.xlog.server.dto.OperateLogQueryReqDTO; import cn.axzo.xlog.server.dto.OperateLogQueryRespDTO; import cn.axzo.xlog.server.dto.OperateLogReqDTO; -import cn.axzo.xlog.server.exception.ServiceException; import cn.axzo.xlog.server.service.OperateLogService; import cn.azxo.framework.common.model.CommonPageResponse; import cn.azxo.framework.common.model.CommonResponse; +import cn.hutool.core.date.DateUtil; import com.github.xiaoymin.knife4j.annotations.ApiSupport; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.validation.BindingResult; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -38,21 +39,36 @@ public class OperateLogController { @RequestMapping(value = "/operateLog/create", method = RequestMethod.POST) @ApiOperation(value = "操作日志创建") - public CommonResponse operateLogCreate(@RequestBody @Valid OperateLogReqDTO req) { + public CommonResponse operateLogCreate(@RequestBody @Valid OperateLogReqDTO req, BindingResult bindingResult) { + //参数缺失等返回 + if (bindingResult.hasErrors()) { + return CommonResponse.fail(bindingResult.getAllErrors().get(0).getDefaultMessage()); + } try { return CommonResponse.success(operateLogService.insertOperaLog(req)); } catch (Exception e) { - logger.error("create operate log failed,", e); + logger.error("create operate log failed.", e); return CommonResponse.fail(e.getMessage()); } } @RequestMapping(value = "/operateLog/queryForPage", method = RequestMethod.POST) @ApiOperation(value = "操作日志查询") - public CommonResponse> operateLogsQuery(@RequestBody @Valid OperateLogQueryReqDTO req) { + public CommonResponse> operateLogsQuery(@RequestBody @Valid OperateLogQueryReqDTO req, + BindingResult bindingResult) { + //参数缺失等返回 + if (bindingResult.hasErrors()) { + return CommonResponse.fail(bindingResult.getAllErrors().get(0).getDefaultMessage()); + } + //时间跨度校验 + if (DateUtil.betweenDay(req.getStartTime(), req.getEndTime(), true) > 7) { + logger.error("start and end date interval greater than 7."); + return CommonResponse.fail("the time span is greater than 7"); + } try { - return CommonResponse.success(operateLogService.queryForPage(req)); - } catch (ServiceException e) { + CommonPageResponse resp = operateLogService.queryForPageFromEs(req); + return CommonResponse.success(resp); + } catch (Exception e) { logger.error("query operate logs failed,", e); return CommonResponse.fail(e.getMessage()); } diff --git a/xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogQueryReqDTO.java b/xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogQueryReqDTO.java index 3d8070d..ccc9a87 100644 --- a/xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogQueryReqDTO.java +++ b/xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogQueryReqDTO.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import javax.validation.constraints.NotNull; import java.util.Date; import java.util.List; @@ -45,15 +46,17 @@ public class OperateLogQueryReqDTO extends PageRequest { /** * 开始时间 */ - @ApiModelProperty(value = "开始时间", position = 1) + @ApiModelProperty(value = "操作开始时间", position = 1) @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @NotNull private Date startTime; /** * 结束时间 */ - @ApiModelProperty(value = "结束时间", position = 1) + @ApiModelProperty(value = "操作结束时间", position = 1) @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @NotNull private Date endTime; /** diff --git a/xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogReqDTO.java b/xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogReqDTO.java index bd69f35..0eb3d71 100644 --- a/xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogReqDTO.java +++ b/xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogReqDTO.java @@ -51,6 +51,7 @@ public class OperateLogReqDTO { */ @ApiModelProperty(value = "操作时间", required = true, position = 5) @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @NotNull(message = "操作时间不能为空") private Date operateTime; /** diff --git a/xlog-server/src/main/java/cn/axzo/xlog/server/service/BaseEsService.java b/xlog-server/src/main/java/cn/axzo/xlog/server/service/BaseEsService.java index 1dff36f..09bc7dd 100644 --- a/xlog-server/src/main/java/cn/axzo/xlog/server/service/BaseEsService.java +++ b/xlog-server/src/main/java/cn/axzo/xlog/server/service/BaseEsService.java @@ -1,6 +1,9 @@ package cn.axzo.xlog.server.service; import cn.axzo.xlog.server.config.SpringContextAware; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; @@ -16,7 +19,10 @@ import org.elasticsearch.rest.RestStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; /*** * @author: pepsi @@ -27,13 +33,55 @@ public class BaseEsService { private final Logger logger = LoggerFactory.getLogger(BaseEsService.class); + /** + * 索引缓存, 索引数量不大、新增频率不高等。可以做成内存缓存。 + */ + private LoadingCache> INDICES_CACHE = CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.MINUTES) + .build(new CacheLoader>() { + @Override + public List load(String key) throws Exception { + return loadIndices(key); + } + }); + + public List getIndices(String indexName) { + try { + return INDICES_CACHE.get(indexName); + } catch (Exception e) { + logger.error("get indices from cache exception,indexName=" + indexName, e); + return null; + } + + } + + /*** + * 根据索引前置获取到索引列表,并排序 + * @param key + * @return + */ + private List loadIndices(String key) { + ArrayList lists = new ArrayList(); + try { + String[] indices = queryIndices(key + "*"); + if (indices.length == 0) { + return lists; + } + lists.addAll(Arrays.asList(indices)); + return lists; + } catch (Exception e) { + logger.error("load indices exception", e); + return null; + } + } + /** * 获取索引 * * @param wildcardIndex * @return */ - public String[] getIndices(String wildcardIndex) throws Exception { + public String[] queryIndices(String wildcardIndex) throws Exception { GetIndexRequest request = new GetIndexRequest(wildcardIndex); try { RestHighLevelClient client = SpringContextAware.getBean(RestHighLevelClient.class); @@ -107,8 +155,8 @@ public class BaseEsService { RestHighLevelClient client = SpringContextAware.getBean(RestHighLevelClient.class); return client.search(request, RequestOptions.DEFAULT); } catch (Exception e) { - logger.error("query failed,", e); - throw new Exception(""); + logger.error("query occur exception,", e); + throw new Exception(e.getMessage()); } } } diff --git a/xlog-server/src/main/java/cn/axzo/xlog/server/service/impl/OperateLogServiceImpl.java b/xlog-server/src/main/java/cn/axzo/xlog/server/service/impl/OperateLogServiceImpl.java index 43bbf4a..cdee3d8 100644 --- a/xlog-server/src/main/java/cn/axzo/xlog/server/service/impl/OperateLogServiceImpl.java +++ b/xlog-server/src/main/java/cn/axzo/xlog/server/service/impl/OperateLogServiceImpl.java @@ -13,13 +13,14 @@ import cn.axzo.xlog.server.dto.OperateLogQueryReqDTO; import cn.axzo.xlog.server.dto.OperateLogQueryRespDTO; import cn.axzo.xlog.server.dto.OperateLogReqDTO; import cn.axzo.xlog.server.entity.OperateLogRecordEntity; -import cn.axzo.xlog.server.enums.IdentityType; import cn.axzo.xlog.server.repository.OperateLogRepository; import cn.axzo.xlog.server.service.BaseEsService; import cn.axzo.xlog.server.service.OperateLogService; import cn.axzo.xlog.server.service.converter.OperateLogConverter; import cn.azxo.framework.common.model.CommonPageResponse; import cn.azxo.framework.common.model.CommonResponse; +import cn.hutool.core.date.CalendarUtil; +import cn.hutool.core.date.DatePattern; import cn.hutool.core.date.DateUtil; import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -37,8 +38,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; +import java.util.*; /*** * @author: pepsi @@ -50,12 +50,21 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe private final Logger logger = LoggerFactory.getLogger(OperateLogServiceImpl.class); - @Value("${xlog.elastic.index.name:xlog_operatelog_}") + /** + * 索引前置名称,看后续是否要修改。 + */ + @Value("${xlog.elastic.index.name:axzo_operatelog_}") private String indexNamePrex; @Value("${xlog.es.storage:true}") private boolean esStorageFlag; + @Value("${oprlog.index.interval.days:7}") + private int intervalDays; + + @Value("${oprlog.index.interval.mills:604800000}") + private long intervalMills; + @Resource private OperateLogRepository operateLogRepository; @@ -80,12 +89,25 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe operateLogRepository.save(record); return true; } - //todo 落 ES,根据日期决定索引,目前索引规则还没定,暂定一个月一个。 - String indexDate = DateUtil.format(operateLogReq.getOperateTime(), "YYYYMM"); + //目前决定根据operateTime时间定,7天 (一般查询限制时间都是7天或者30天) 数据落一个索引中， + // 目前查看pre_new环境数据很少，上线前再查看一波线上数据决定索引生成的时间跨度,再做调整。 + String indexSuffixDate = findIndexSuffixDate(operateLogReq.getOperateTime()); String json = JSONObject.toJSONString(operateLogReq); - return insert(json, indexNamePrex + indexDate); + return insert(json, indexNamePrex + indexSuffixDate); } + /*** + * 根据数据中的日期信息找到所属的索引 + * 目前是按7天的规则。 + * @param date + * @return + */ + private String findIndexSuffixDate(Date date) { + long mills = date.getTime(); + long weekMills = mills % intervalMills; + Date currDate = new Date(mills - weekMills); + return DateUtil.format(currDate, DatePattern.PURE_DATE_PATTERN); + } private OperateLogRecordEntity fieldFill(OperateLogReqDTO req) { //补充 termimnal identityType featureName @@ -124,13 +146,17 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe * @return */ public CommonPageResponse queryForPageFromEs(OperateLogQueryReqDTO req) throws Exception { - //todo 根据时间段获取数据所在的索引s,目前索引规则还没定 - String[] indices = new String[]{"xlog_operatelog_202209"}; + //根据查询的时间段获取所属的索引集合 + Set indices = betweenIndices(req.getStartTime(), req.getEndTime()); + if (indices.isEmpty()) { + logger.warn("can not find index,startDate={}.", DateUtil.format(req.getStartTime(), DatePattern.PURE_DATE_PATTERN)); + return CommonPageResponse.zero(req.getPage(), req.getPageSize()); + } //组装查询条件. SearchSourceBuilder builder = assembleQueryBuilder(req); SearchRequest request = new SearchRequest(); request.source(builder); - SearchResponse response = search(request, indices); + SearchResponse response = search(request, indices.toArray(new String[]{})); //处理查询结果的数据 SearchHits hits = response.getHits(); SearchHit[] searchHits = hits.getHits(); @@ -146,6 +172,30 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe return CommonPageResponse.list(req.getPage(), req.getPageSize(), hits.getTotalHits().value, respList); } + /*** + * 获取2个时间段之间的索引。startTime + 7 < endTime 即可。 + * @param startDate + * @param endDate + * @return + */ + private Set betweenIndices(Date startDate, Date endDate) { + Set betweenIndices = new HashSet<>(); + long startMills = startDate.getTime(); + long startIntervalMills = startMills % intervalMills; + Date startWeekDate = new Date(startMills - startIntervalMills); + while (startWeekDate.before(endDate)) { + String index = indexNamePrex + DateUtil.format(startWeekDate, DatePattern.PURE_DATE_PATTERN); + betweenIndices.add(index); + Calendar calendar = CalendarUtil.calendar(startWeekDate); + calendar.add(Calendar.DAY_OF_MONTH, intervalDays); + startWeekDate = calendar.getTime(); + } + //需要判断下索引是否存在, + List indices = getIndices(indexNamePrex); + betweenIndices.removeIf(index -> !indices.contains(index)); + return betweenIndices; + } + /*** * 组装ES查询条件 * @param req @@ -192,9 +242,10 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe .lte(req.getEndTime().getTime()); boolQueryBuilder.filter(rangeQueryBuilder); searchSourceBuilder.sort("operateTime", SortOrder.DESC); - //todo 分页,先采用 from +size 模式，后续优化考虑scroll或者 after. - int offset = (req.getPage().intValue() - 1) * req.getPageSize().intValue(); - searchSourceBuilder.from(offset).size(req.getPageSize().intValue()); + //采用 from +size，条件限制多一点。不做深查询了，因为scroll 快照数据不准确问题。 + int page = req.getPage() == null ? 1 : req.getPage().intValue(); + int size = req.getPageSize() == null ? 20 : req.getPageSize().intValue(); + searchSourceBuilder.from((page - 1) * size).size(size); searchSourceBuilder.query(boolQueryBuilder); return searchSourceBuilder; } diff --git a/xlog-server/src/test/java/cn/axzo/xlog/server/XlogApplicationTestBase.java b/xlog-server/src/test/java/cn/axzo/xlog/server/XlogApplicationTestBase.java index cab822f..6bca3f7 100644 --- a/xlog-server/src/test/java/cn/axzo/xlog/server/XlogApplicationTestBase.java +++ b/xlog-server/src/test/java/cn/axzo/xlog/server/XlogApplicationTestBase.java @@ -12,5 +12,5 @@ import org.springframework.test.context.junit4.SpringRunner; @RunWith(SpringRunner.class) @SpringBootTest public class XlogApplicationTestBase { - + } diff --git a/xlog-server/src/test/java/cn/axzo/xlog/server/config/ElasticClientConfigTest.java b/xlog-server/src/test/java/cn/axzo/xlog/server/config/ElasticClientConfigTest.java index 3745544..b38aa36 100644 --- a/xlog-server/src/test/java/cn/axzo/xlog/server/config/ElasticClientConfigTest.java +++ b/xlog-server/src/test/java/cn/axzo/xlog/server/config/ElasticClientConfigTest.java @@ -2,6 +2,9 @@ package cn.axzo.xlog.server.config; import cn.axzo.xlog.server.entity.OperateLogRecordEntity; import cn.axzo.xlog.server.service.BaseEsService; +import cn.hutool.core.date.CalendarUtil; +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; import com.alibaba.fastjson.JSONObject; import org.elasticsearch.client.RestHighLevelClient; import org.junit.Assert; @@ -12,6 +15,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; import java.util.ArrayList; +import java.util.Calendar; import java.util.Date; import java.util.List; @@ -27,6 +31,8 @@ public class ElasticClientConfigTest extends BaseEsService { @Autowired private RestHighLevelClient client; + private long sevenDayMills = 7 * 24 * 60 * 60 * 1000; + @Test public void testInsert() throws Exception { OperateLogRecordEntity entity = new OperateLogRecordEntity(); @@ -58,4 +64,55 @@ public class ElasticClientConfigTest extends BaseEsService { Assert.assertTrue(flag); } + + @Test + public void testIndexInterval() { + long sevenDayMills = 7 * 24 * 60 * 60 * 1000; + Date now = new Date(); + String weekDate = interval(now); + System.out.println("now:" + weekDate); + + String weekDateThreeDaysAgo = interval(add(now, -3)); + System.out.println("ThreeDaysAgo:" + weekDateThreeDaysAgo); + + String weekDateTwoDaysAgo = interval(add(now, -2)); + System.out.println("now:" + weekDateTwoDaysAgo); + + String weekDateAfter1 = interval(add(now, 1)); + System.out.println("After1:" + weekDateAfter1); + + String weekDateAfter2 = interval(add(now, 2)); + System.out.println("After2:" + weekDateAfter2); + + String weekDateAfter3 = interval(add(now, 3)); + System.out.println("After3:" + weekDateAfter3); + + String weekDateAfter4 = interval(add(now, 4)); + System.out.println("After4:" + weekDateAfter4); + + String weekDateAfter5 = interval(add(now, 5)); + System.out.println("After5:" + weekDateAfter5); + + String weekDateAfter6 = interval(add(now, 6)); + System.out.println("After6:" + weekDateAfter6); + + String weekDateAfter7 = interval(add(now, 7)); + System.out.println("After7:" + weekDateAfter7); + + String weekDateAfter8 = interval(add(now, 8)); + System.out.println("After8:" + weekDateAfter8); + } + + private Date add(Date date, int interval) { + Calendar calendar = CalendarUtil.calendar(date); + calendar.add(Calendar.DAY_OF_MONTH, interval); + return calendar.getTime(); + } + + private String interval(Date date) { + long mills = date.getTime(); + long intervalMills = mills % sevenDayMills; + Date currDate = new Date(mills - intervalMills); + return DateUtil.format(currDate, DatePattern.PURE_DATE_PATTERN); + } }