wangli/axzo-log-plat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

operate log modify

Browse Source
This commit is contained in:
pepsi 2022-09-28 19:52:55 +08:00
parent b9f831027e
commit a3b0141f75
7 changed files with 201 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
xlog-server/src/main/java/cn/axzo/xlog/server/controller/api/OperateLogController.java
View File

@ -3,15 +3,16 @@ package cn.axzo.xlog.server.controller.api;
import cn.axzo.xlog.server.dto.OperateLogQueryReqDTO;
import cn.axzo.xlog.server.dto.OperateLogQueryRespDTO;
import cn.axzo.xlog.server.dto.OperateLogReqDTO;
import cn.axzo.xlog.server.exception.ServiceException;
import cn.axzo.xlog.server.service.OperateLogService;
import cn.azxo.framework.common.model.CommonPageResponse;
import cn.azxo.framework.common.model.CommonResponse;
import cn.hutool.core.date.DateUtil;
import com.github.xiaoymin.knife4j.annotations.ApiSupport;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.validation.BindingResult;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@ -38,21 +39,36 @@ public class OperateLogController {
@RequestMapping(value = "/operateLog/create", method = RequestMethod.POST)
@ApiOperation(value = "操作日志创建")
public CommonResponse<Boolean> operateLogCreate(@RequestBody @Valid OperateLogReqDTO req) {
public CommonResponse<Boolean> operateLogCreate(@RequestBody @Valid OperateLogReqDTO req, BindingResult bindingResult) {
//参数缺失等返回
if (bindingResult.hasErrors()) {
return CommonResponse.fail(bindingResult.getAllErrors().get(0).getDefaultMessage());
}
try {
return CommonResponse.success(operateLogService.insertOperaLog(req));
} catch (Exception e) {
logger.error("create operate log failed,", e);
logger.error("create operate log failed.", e);
return CommonResponse.fail(e.getMessage());
}
}
@RequestMapping(value = "/operateLog/queryForPage", method = RequestMethod.POST)
@ApiOperation(value = "操作日志查询")
public CommonResponse<CommonPageResponse<OperateLogQueryRespDTO>> operateLogsQuery(@RequestBody @Valid OperateLogQueryReqDTO req) {
public CommonResponse<CommonPageResponse<OperateLogQueryRespDTO>> operateLogsQuery(@RequestBody @Valid OperateLogQueryReqDTO req,
BindingResult bindingResult) {
//参数缺失等返回
if (bindingResult.hasErrors()) {
return CommonResponse.fail(bindingResult.getAllErrors().get(0).getDefaultMessage());
}
//时间跨度校验
if (DateUtil.betweenDay(req.getStartTime(), req.getEndTime(), true) > 7) {
logger.error("start and end date interval greater than 7.");
return CommonResponse.fail("the time span is greater than 7");
}
try {
return CommonResponse.success(operateLogService.queryForPage(req));
} catch (ServiceException e) {
CommonPageResponse<OperateLogQueryRespDTO> resp = operateLogService.queryForPageFromEs(req);
return CommonResponse.success(resp);
} catch (Exception e) {
logger.error("query operate logs failed,", e);
return CommonResponse.fail(e.getMessage());
}

7
xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogQueryReqDTO.java
View File

@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotNull;
import java.util.Date;
import java.util.List;
@ -45,15 +46,17 @@ public class OperateLogQueryReqDTO extends PageRequest {
/**
* 开始时间
*/
@ApiModelProperty(value = "开始时间", position = 1)
@ApiModelProperty(value = "操作开始时间", position = 1)
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
@NotNull
private Date startTime;
/**
* 结束时间
*/
@ApiModelProperty(value = "结束时间", position = 1)
@ApiModelProperty(value = "操作结束时间", position = 1)
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
@NotNull
private Date endTime;
/**

1
xlog-server/src/main/java/cn/axzo/xlog/server/dto/OperateLogReqDTO.java
View File

@ -51,6 +51,7 @@ public class OperateLogReqDTO {
*/
@ApiModelProperty(value = "操作时间", required = true, position = 5)
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
@NotNull(message = "操作时间不能为空")
private Date operateTime;
/**

54
xlog-server/src/main/java/cn/axzo/xlog/server/service/BaseEsService.java
View File

@ -1,6 +1,9 @@
package cn.axzo.xlog.server.service;
import cn.axzo.xlog.server.config.SpringContextAware;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
@ -16,7 +19,10 @@ import org.elasticsearch.rest.RestStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
/***
* @author: pepsi
@ -27,13 +33,55 @@ public class BaseEsService {
private final Logger logger = LoggerFactory.getLogger(BaseEsService.class);
/**
* 索引缓存, 索引数量不大、新增频率不高等。可以做成内存缓存。
*/
private LoadingCache<String, List<String>> INDICES_CACHE = CacheBuilder.newBuilder()
.expireAfterWrite(10, TimeUnit.MINUTES)
.build(new CacheLoader<String, List<String>>() {
@Override
public List<String> load(String key) throws Exception {
return loadIndices(key);
}
});
public List<String> getIndices(String indexName) {
try {
return INDICES_CACHE.get(indexName);
} catch (Exception e) {
logger.error("get indices from cache exception,indexName=" + indexName, e);
return null;
}
}
/***
* 根据索引前置获取到索引列表,并排序
* @param key
* @return
*/
private List<String> loadIndices(String key) {
ArrayList<String> lists = new ArrayList<String>();
try {
String[] indices = queryIndices(key + "*");
if (indices.length == 0) {
return lists;
}
lists.addAll(Arrays.asList(indices));
return lists;
} catch (Exception e) {
logger.error("load indices exception", e);
return null;
}
}
/**
* 获取索引
*
* @param wildcardIndex
* @return
*/
public String[] getIndices(String wildcardIndex) throws Exception {
public String[] queryIndices(String wildcardIndex) throws Exception {
GetIndexRequest request = new GetIndexRequest(wildcardIndex);
try {
RestHighLevelClient client = SpringContextAware.getBean(RestHighLevelClient.class);
@ -107,8 +155,8 @@ public class BaseEsService {
RestHighLevelClient client = SpringContextAware.getBean(RestHighLevelClient.class);
return client.search(request, RequestOptions.DEFAULT);
} catch (Exception e) {
logger.error("query failed,", e);
throw new Exception("");
logger.error("query occur exception,", e);
throw new Exception(e.getMessage());
}
}
}

77
xlog-server/src/main/java/cn/axzo/xlog/server/service/impl/OperateLogServiceImpl.java
View File

@ -13,13 +13,14 @@ import cn.axzo.xlog.server.dto.OperateLogQueryReqDTO;
import cn.axzo.xlog.server.dto.OperateLogQueryRespDTO;
import cn.axzo.xlog.server.dto.OperateLogReqDTO;
import cn.axzo.xlog.server.entity.OperateLogRecordEntity;
import cn.axzo.xlog.server.enums.IdentityType;
import cn.axzo.xlog.server.repository.OperateLogRepository;
import cn.axzo.xlog.server.service.BaseEsService;
import cn.axzo.xlog.server.service.OperateLogService;
import cn.axzo.xlog.server.service.converter.OperateLogConverter;
import cn.azxo.framework.common.model.CommonPageResponse;
import cn.azxo.framework.common.model.CommonResponse;
import cn.hutool.core.date.CalendarUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateUtil;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.metadata.IPage;
@ -37,8 +38,7 @@ import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.*;
/***
* @author: pepsi
@ -50,12 +50,21 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe
private final Logger logger = LoggerFactory.getLogger(OperateLogServiceImpl.class);
@Value("${xlog.elastic.index.name:xlog_operatelog_}")
/**
* 索引前置名称,看后续是否要修改。
*/
@Value("${xlog.elastic.index.name:axzo_operatelog_}")
private String indexNamePrex;
@Value("${xlog.es.storage:true}")
private boolean esStorageFlag;
@Value("${oprlog.index.interval.days:7}")
private int intervalDays;
@Value("${oprlog.index.interval.mills:604800000}")
private long intervalMills;
@Resource
private OperateLogRepository operateLogRepository;
@ -80,12 +89,25 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe
operateLogRepository.save(record);
return true;
}
//todo 落 ES,根据日期决定索引,目前索引规则还没定,暂定一个月一个。
String indexDate = DateUtil.format(operateLogReq.getOperateTime(), "YYYYMM");
//目前决定根据operateTime时间定,7天 (一般查询限制时间都是7天或者30天) 数据落一个索引中，
// 目前查看pre_new环境数据很少，上线前再查看一波线上数据决定索引生成的时间跨度,再做调整。
String indexSuffixDate = findIndexSuffixDate(operateLogReq.getOperateTime());
String json = JSONObject.toJSONString(operateLogReq);
return insert(json, indexNamePrex + indexDate);
return insert(json, indexNamePrex + indexSuffixDate);
}
/***
* 根据数据中的日期信息找到所属的索引
* 目前是按7天的规则。
* @param date
* @return
*/
private String findIndexSuffixDate(Date date) {
long mills = date.getTime();
long weekMills = mills % intervalMills;
Date currDate = new Date(mills - weekMills);
return DateUtil.format(currDate, DatePattern.PURE_DATE_PATTERN);
}
private OperateLogRecordEntity fieldFill(OperateLogReqDTO req) {
//补充 termimnal identityType featureName
@ -124,13 +146,17 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe
* @return
*/
public CommonPageResponse<OperateLogQueryRespDTO> queryForPageFromEs(OperateLogQueryReqDTO req) throws Exception {
//todo 根据时间段获取数据所在的索引s,目前索引规则还没定
String[] indices = new String[]{"xlog_operatelog_202209"};
//根据查询的时间段获取所属的索引集合
Set<String> indices = betweenIndices(req.getStartTime(), req.getEndTime());
if (indices.isEmpty()) {
logger.warn("can not find index,startDate={}.", DateUtil.format(req.getStartTime(), DatePattern.PURE_DATE_PATTERN));
return CommonPageResponse.zero(req.getPage(), req.getPageSize());
}
//组装查询条件.
SearchSourceBuilder builder = assembleQueryBuilder(req);
SearchRequest request = new SearchRequest();
request.source(builder);
SearchResponse response = search(request, indices);
SearchResponse response = search(request, indices.toArray(new String[]{}));
//处理查询结果的数据
SearchHits hits = response.getHits();
SearchHit[] searchHits = hits.getHits();
@ -146,6 +172,30 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe
return CommonPageResponse.list(req.getPage(), req.getPageSize(), hits.getTotalHits().value, respList);
}
/***
* 获取2个时间段之间的索引。startTime + 7 < endTime 即可。
* @param startDate
* @param endDate
* @return
*/
private Set<String> betweenIndices(Date startDate, Date endDate) {
Set<String> betweenIndices = new HashSet<>();
long startMills = startDate.getTime();
long startIntervalMills = startMills % intervalMills;
Date startWeekDate = new Date(startMills - startIntervalMills);
while (startWeekDate.before(endDate)) {
String index = indexNamePrex + DateUtil.format(startWeekDate, DatePattern.PURE_DATE_PATTERN);
betweenIndices.add(index);
Calendar calendar = CalendarUtil.calendar(startWeekDate);
calendar.add(Calendar.DAY_OF_MONTH, intervalDays);
startWeekDate = calendar.getTime();
}
//需要判断下索引是否存在,
List<String> indices = getIndices(indexNamePrex);
betweenIndices.removeIf(index -> !indices.contains(index));
return betweenIndices;
}
/***
* 组装ES查询条件
* @param req
@ -192,9 +242,10 @@ public class OperateLogServiceImpl extends BaseEsService implements OperateLogSe
.lte(req.getEndTime().getTime());
boolQueryBuilder.filter(rangeQueryBuilder);
searchSourceBuilder.sort("operateTime", SortOrder.DESC);
//todo 分页,先采用 from +size 模式，后续优化考虑scroll或者 after.
int offset = (req.getPage().intValue() - 1) * req.getPageSize().intValue();
searchSourceBuilder.from(offset).size(req.getPageSize().intValue());
//采用 from +size，条件限制多一点。不做深查询了，因为scroll 快照数据不准确问题。
int page = req.getPage() == null ? 1 : req.getPage().intValue();
int size = req.getPageSize() == null ? 20 : req.getPageSize().intValue();
searchSourceBuilder.from((page - 1) * size).size(size);
searchSourceBuilder.query(boolQueryBuilder);
return searchSourceBuilder;
}

2
xlog-server/src/test/java/cn/axzo/xlog/server/XlogApplicationTestBase.java
View File

@ -12,5 +12,5 @@ import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
@SpringBootTest
public class XlogApplicationTestBase {
}

57
xlog-server/src/test/java/cn/axzo/xlog/server/config/ElasticClientConfigTest.java
View File

@ -2,6 +2,9 @@ package cn.axzo.xlog.server.config;
import cn.axzo.xlog.server.entity.OperateLogRecordEntity;
import cn.axzo.xlog.server.service.BaseEsService;
import cn.hutool.core.date.CalendarUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateUtil;
import com.alibaba.fastjson.JSONObject;
import org.elasticsearch.client.RestHighLevelClient;
import org.junit.Assert;
@ -12,6 +15,7 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
@ -27,6 +31,8 @@ public class ElasticClientConfigTest extends BaseEsService {
@Autowired
private RestHighLevelClient client;
private long sevenDayMills = 7 * 24 * 60 * 60 * 1000;
@Test
public void testInsert() throws Exception {
OperateLogRecordEntity entity = new OperateLogRecordEntity();
@ -58,4 +64,55 @@ public class ElasticClientConfigTest extends BaseEsService {
Assert.assertTrue(flag);
}
@Test
public void testIndexInterval() {
long sevenDayMills = 7 * 24 * 60 * 60 * 1000;
Date now = new Date();
String weekDate = interval(now);
System.out.println("now:" + weekDate);
String weekDateThreeDaysAgo = interval(add(now, -3));
System.out.println("ThreeDaysAgo:" + weekDateThreeDaysAgo);
String weekDateTwoDaysAgo = interval(add(now, -2));
System.out.println("now:" + weekDateTwoDaysAgo);
String weekDateAfter1 = interval(add(now, 1));
System.out.println("After1:" + weekDateAfter1);
String weekDateAfter2 = interval(add(now, 2));
System.out.println("After2:" + weekDateAfter2);
String weekDateAfter3 = interval(add(now, 3));
System.out.println("After3:" + weekDateAfter3);
String weekDateAfter4 = interval(add(now, 4));
System.out.println("After4:" + weekDateAfter4);
String weekDateAfter5 = interval(add(now, 5));
System.out.println("After5:" + weekDateAfter5);
String weekDateAfter6 = interval(add(now, 6));
System.out.println("After6:" + weekDateAfter6);
String weekDateAfter7 = interval(add(now, 7));
System.out.println("After7:" + weekDateAfter7);
String weekDateAfter8 = interval(add(now, 8));
System.out.println("After8:" + weekDateAfter8);
}
private Date add(Date date, int interval) {
Calendar calendar = CalendarUtil.calendar(date);
calendar.add(Calendar.DAY_OF_MONTH, interval);
return calendar.getTime();
}
private String interval(Date date) {
long mills = date.getTime();
long intervalMills = mills % sevenDayMills;
Date currDate = new Date(mills - intervalMills);
return DateUtil.format(currDate, DatePattern.PURE_DATE_PATTERN);
}
}