From 2996ef2dc59f7702aa533033690aa8c87f8cbd66 Mon Sep 17 00:00:00 2001 From: chenwenjian Date: Thu, 9 Jan 2025 14:59:59 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(REQ-3557):=20=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1-=E5=AE=9A=E6=9C=9F?= =?UTF-8?q?=E6=B8=85=E7=90=86mogodb=E5=86=85=E6=97=A5=E5=BF=97=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- axzo-log-server/pom.xml | 4 + .../platform/server/job/LogCleanupJob.java | 107 ++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java diff --git a/axzo-log-server/pom.xml b/axzo-log-server/pom.xml index c597ffa..28c8395 100644 --- a/axzo-log-server/pom.xml +++ b/axzo-log-server/pom.xml @@ -173,6 +173,10 @@ 1.0.0-SNAPSHOT compile + + com.xuxueli + xxl-job-core + diff --git a/axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java b/axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java new file mode 100644 index 0000000..9ddd3ab --- /dev/null +++ b/axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java @@ -0,0 +1,107 @@ +package cn.axzo.log.platform.server.job; + +import cn.axzo.log.platform.server.entity.LogEntity; +import com.alibaba.fastjson.JSON; +import com.xxl.job.core.biz.model.ReturnT; +import com.xxl.job.core.handler.annotation.XxlJob; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Sort; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.stereotype.Component; + +import java.util.Date; +import java.util.List; + +/** + * 清理日志 + * + * @author chenwenjian + * @version 1.0 + * @date 2025/1/9 11:25 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class LogCleanupJob { + + private final MongoTemplate mongoTemplate; + + /** + * 清理日志 + * + * @param param 示例：{\"batchSize\":1000,\"daysToKeep\":30,\"scene\":\"networkLog\"} + * @return + */ + @XxlJob("logCleanupJobHandler") + public ReturnT cleanupLogs(String param) { + log.info("Starting log cleanup job with parameters: {}", param); + // 设置默认参数，默认清理scene为networkLog且时间超过30天的日志数据 + ParamDTO paramDTO = ParamDTO.builder().build(); + // 解析参数 + try { + paramDTO = JSON.parseObject(param, ParamDTO.class); + } catch (Exception e) { + log.warn("Invalid parameter format, using default values."); + } + long deletedCount = 0; + try { + // 构建查询条件，删除指定场景且时间超过指定天数的日志数据 + Date cutoffDate = new Date(System.currentTimeMillis() - (paramDTO.getDaysToKeep() * 24 * 60 * 60 * 1000L)); + Criteria criteria = Criteria.where("scene").is(paramDTO.getScene()) + .and("timestamp").lte(cutoffDate.getTime()); + Query query = new Query(criteria); + query.with(Sort.by(Sort.Direction.DESC, "_id")); + query.limit(paramDTO.getBatchSize()); + + while (true) { + List logsToDelete = mongoTemplate.find(query, LogEntity.class); + if (logsToDelete.isEmpty()) { + break; + } + + mongoTemplate.remove(query, LogEntity.class); + deletedCount += logsToDelete.size(); + log.info("Cleaned up {} logs for scene: {} before timestamp: {}", logsToDelete.size(), paramDTO.getScene(), cutoffDate); + + // 更新查询条件，使用最后一个日志的ID作为游标 + LogEntity lastLog = logsToDelete.get(logsToDelete.size() - 1); + query.addCriteria(Criteria.where("_id").lt(lastLog.getId())); + } + } catch (Exception e) { + log.error("Error during log cleanup job", e); + return ReturnT.FAIL; + } + + log.info("Ending log cleanup job, total cleaned up logs for scene {}: {}", paramDTO.getScene(), deletedCount); + return ReturnT.SUCCESS; + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + private static class ParamDTO { + + /** + * 每次删除的日志数量（页大小） + */ + private int batchSize = 1000; + + /** + * 保留天数 + */ + private int daysToKeep = 30; + + /** + * 场景 + */ + private String scene = "networkLog"; + } +} From 65510516fff832e3fb70a4aa59e0718adb2f30eb Mon Sep 17 00:00:00 2001 From: chenwenjian Date: Thu, 9 Jan 2025 15:39:06 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat(REQ-3557):=20=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/server/config/XxlJobConfig.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 axzo-log-server/src/main/java/cn/axzo/log/platform/server/config/XxlJobConfig.java diff --git a/axzo-log-server/src/main/java/cn/axzo/log/platform/server/config/XxlJobConfig.java b/axzo-log-server/src/main/java/cn/axzo/log/platform/server/config/XxlJobConfig.java new file mode 100644 index 0000000..71a4f67 --- /dev/null +++ b/axzo-log-server/src/main/java/cn/axzo/log/platform/server/config/XxlJobConfig.java @@ -0,0 +1,67 @@ +package cn.axzo.log.platform.server.config; + +import cn.azxo.framework.common.annotation.OnlyPodsEnvironment; +import com.xxl.job.core.executor.impl.XxlJobSpringExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * + * @author chenwenjian + * @version 1.0 + * @date 2025/1/9 15:35 + */ +@OnlyPodsEnvironment +@Configuration(value = "xxlJobConfig") +public class XxlJobConfig { + + Logger logger = LoggerFactory.getLogger(XxlJobConfig.class); + + /** + * //@Value("http://dev-xxl-job.axzo.cn/xxl-job-admin") + */ + @Value("${xxl.job.admin.addresses}") + private String adminAddresses; + + @Value("${xxl.job.executor.appname}") + private String appName; + + @Value("") + private String ip; + + @Value("${xxl.job.executor.port}") + private int port; + + + /** + * // @Value("${xxl.job.accessToken}") + */ + @Value("") + private String accessToken; + + @Value("") + private String logPath; + + @Value("-1") + private int logRetentionDays; + + @Bean + public XxlJobSpringExecutor xxlJobExecutor() { + logger.info(">>>>>>>>>>> xxl-job config init."); + XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor(); + xxlJobSpringExecutor.setAdminAddresses(adminAddresses); + xxlJobSpringExecutor.setAppname(appName); + xxlJobSpringExecutor.setIp(ip); + xxlJobSpringExecutor.setPort(port); + xxlJobSpringExecutor.setAccessToken(accessToken); + xxlJobSpringExecutor.setLogPath(logPath); + xxlJobSpringExecutor.setLogRetentionDays(logRetentionDays); + + return xxlJobSpringExecutor; + } + +} + From b778e4a7ff6a3ac9da1f49128855188798c8113a Mon Sep 17 00:00:00 2001 From: chenwenjian Date: Fri, 10 Jan 2025 14:09:06 +0800 Subject: [PATCH 3/3] =?UTF-8?q?feat(REQ-3557):=20=E7=AE=80=E5=8C=96?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=88=A0=E9=99=A4=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/server/job/LogCleanupJob.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java b/axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java index 9ddd3ab..e8ab4b6 100644 --- a/axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java +++ b/axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java @@ -2,6 +2,7 @@ package cn.axzo.log.platform.server.job; import cn.axzo.log.platform.server.entity.LogEntity; import com.alibaba.fastjson.JSON; +import com.mongodb.client.result.DeleteResult; import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.annotation.XxlJob; import lombok.AllArgsConstructor; @@ -10,14 +11,13 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.stereotype.Component; +import java.text.SimpleDateFormat; import java.util.Date; -import java.util.List; /** * 清理日志 @@ -36,7 +36,7 @@ public class LogCleanupJob { /** * 清理日志 * - * @param param 示例：{\"batchSize\":1000,\"daysToKeep\":30,\"scene\":\"networkLog\"} + * @param param 示例：{"batchSize":1000,"daysToKeep":15,"scene":"networkLog"} * @return */ @XxlJob("logCleanupJobHandler") @@ -48,7 +48,7 @@ public class LogCleanupJob { try { paramDTO = JSON.parseObject(param, ParamDTO.class); } catch (Exception e) { - log.warn("Invalid parameter format, using default values."); + log.warn("Invalid parameter format, using default values. error info: {}", e.getMessage()); } long deletedCount = 0; try { @@ -57,22 +57,16 @@ public class LogCleanupJob { Criteria criteria = Criteria.where("scene").is(paramDTO.getScene()) .and("timestamp").lte(cutoffDate.getTime()); Query query = new Query(criteria); - query.with(Sort.by(Sort.Direction.DESC, "_id")); query.limit(paramDTO.getBatchSize()); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); while (true) { - List logsToDelete = mongoTemplate.find(query, LogEntity.class); - if (logsToDelete.isEmpty()) { + DeleteResult result = mongoTemplate.remove(query, LogEntity.class); + if (result.getDeletedCount() == 0) { break; } - - mongoTemplate.remove(query, LogEntity.class); - deletedCount += logsToDelete.size(); - log.info("Cleaned up {} logs for scene: {} before timestamp: {}", logsToDelete.size(), paramDTO.getScene(), cutoffDate); - - // 更新查询条件，使用最后一个日志的ID作为游标 - LogEntity lastLog = logsToDelete.get(logsToDelete.size() - 1); - query.addCriteria(Criteria.where("_id").lt(lastLog.getId())); + deletedCount += result.getDeletedCount(); + log.info("Cleaned up {} logs for scene: {} before data: {}", result.getDeletedCount(), paramDTO.getScene(), format.format(cutoffDate)); } } catch (Exception e) { log.error("Error during log cleanup job", e); @@ -92,16 +86,19 @@ public class LogCleanupJob { /** * 每次删除的日志数量（页大小） */ + @Builder.Default private int batchSize = 1000; /** * 保留天数 */ + @Builder.Default private int daysToKeep = 30; /** * 场景 */ + @Builder.Default private String scene = "networkLog"; } }