wangli/axzo-log-plat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(REQ-3557): 简化日志删除定时任务

Browse Source
This commit is contained in:
chenwenjian 2025-01-10 14:09:06 +08:00
parent 65510516ff
commit b778e4a7ff
1 changed files with 12 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java
View File

@ -2,6 +2,7 @@ package cn.axzo.log.platform.server.job;
import cn.axzo.log.platform.server.entity.LogEntity; import cn.axzo.log.platform.server.entity.LogEntity;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.mongodb.client.result.DeleteResult;
import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.biz.model.ReturnT;
import com.xxl.job.core.handler.annotation.XxlJob; import com.xxl.job.core.handler.annotation.XxlJob;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -10,14 +11,13 @@ import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import java.util.List;
/** /**
* 清理日志 * 清理日志
@ -36,7 +36,7 @@ public class LogCleanupJob {
/** /**
* 清理日志 * 清理日志
* *
* @param param 示例：{\"batchSize\":1000,\"daysToKeep\":30,\"scene\":\"networkLog\"} * @param param 示例：{"batchSize":1000,"daysToKeep":15,"scene":"networkLog"}
* @return * @return
*/ */
@XxlJob("logCleanupJobHandler") @XxlJob("logCleanupJobHandler")
@ -48,7 +48,7 @@ public class LogCleanupJob {
try { try {
paramDTO = JSON.parseObject(param, ParamDTO.class); paramDTO = JSON.parseObject(param, ParamDTO.class);
} catch (Exception e) { } catch (Exception e) {
log.warn("Invalid parameter format, using default values."); log.warn("Invalid parameter format, using default values. error info: {}", e.getMessage());
} }
long deletedCount = 0; long deletedCount = 0;
try { try {
@ -57,22 +57,16 @@ public class LogCleanupJob {
Criteria criteria = Criteria.where("scene").is(paramDTO.getScene()) Criteria criteria = Criteria.where("scene").is(paramDTO.getScene())
.and("timestamp").lte(cutoffDate.getTime()); .and("timestamp").lte(cutoffDate.getTime());
Query query = new Query(criteria); Query query = new Query(criteria);
query.with(Sort.by(Sort.Direction.DESC, "_id"));
query.limit(paramDTO.getBatchSize()); query.limit(paramDTO.getBatchSize());
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
while (true) { while (true) {
List<LogEntity> logsToDelete = mongoTemplate.find(query, LogEntity.class); DeleteResult result = mongoTemplate.remove(query, LogEntity.class);
if (logsToDelete.isEmpty()) { if (result.getDeletedCount() == 0) {
break; break;
} }
deletedCount += result.getDeletedCount();
mongoTemplate.remove(query, LogEntity.class); log.info("Cleaned up {} logs for scene: {} before data: {}", result.getDeletedCount(), paramDTO.getScene(), format.format(cutoffDate));
deletedCount += logsToDelete.size();
log.info("Cleaned up {} logs for scene: {} before timestamp: {}", logsToDelete.size(), paramDTO.getScene(), cutoffDate);
// 更新查询条件，使用最后一个日志的ID作为游标
LogEntity lastLog = logsToDelete.get(logsToDelete.size() - 1);
query.addCriteria(Criteria.where("_id").lt(lastLog.getId()));
} }
} catch (Exception e) { } catch (Exception e) {
log.error("Error during log cleanup job", e); log.error("Error during log cleanup job", e);
@ -92,16 +86,19 @@ public class LogCleanupJob {
/** /**
* 每次删除的日志数量（页大小） * 每次删除的日志数量（页大小）
*/ */
@Builder.Default
private int batchSize = 1000; private int batchSize = 1000;
/** /**
* 保留天数 * 保留天数
*/ */
@Builder.Default
private int daysToKeep = 30; private int daysToKeep = 30;
/** /**
* 场景 * 场景
*/ */
@Builder.Default
private String scene = "networkLog"; private String scene = "networkLog";
} }
} }