feat(REQ-3557): 新增定时任务-定期清理mogodb内日志数据

axzo-log-server/pom.xml

@@ -173,6 +173,10 @@
             <version>1.0.0-SNAPSHOT</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>com.xuxueli</groupId>
+            <artifactId>xxl-job-core</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

axzo-log-server/src/main/java/cn/axzo/log/platform/server/job/LogCleanupJob.java

@@ -0,0 +1,107 @@
+package cn.axzo.log.platform.server.job;
+
+import cn.axzo.log.platform.server.entity.LogEntity;
+import com.alibaba.fastjson.JSON;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+
+/**
+ * 清理日志
+ *
+ * @author chenwenjian
+ * @version 1.0
+ * @date 2025/1/9 11:25
+ */
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class LogCleanupJob {
+
+    private final MongoTemplate mongoTemplate;
+
+    /**
+     * 清理日志
+     *
+     * @param param 示例：{\"batchSize\":1000,\"daysToKeep\":30,\"scene\":\"networkLog\"}
+     * @return
+     */
+    @XxlJob("logCleanupJobHandler")
+    public ReturnT<String> cleanupLogs(String param) {
+        log.info("Starting log cleanup job with parameters: {}", param);
+        // 设置默认参数，默认清理scene为networkLog且时间超过30天的日志数据
+        ParamDTO paramDTO = ParamDTO.builder().build();
+        // 解析参数
+        try {
+            paramDTO = JSON.parseObject(param, ParamDTO.class);
+        } catch (Exception e) {
+            log.warn("Invalid parameter format, using default values.");
+        }
+        long deletedCount = 0;
+        try {
+            // 构建查询条件，删除指定场景且时间超过指定天数的日志数据
+            Date cutoffDate = new Date(System.currentTimeMillis() - (paramDTO.getDaysToKeep() * 24 * 60 * 60 * 1000L));
+            Criteria criteria = Criteria.where("scene").is(paramDTO.getScene())
+                    .and("timestamp").lte(cutoffDate.getTime());
+            Query query = new Query(criteria);
+            query.with(Sort.by(Sort.Direction.DESC, "_id"));
+            query.limit(paramDTO.getBatchSize());
+
+            while (true) {
+                List<LogEntity> logsToDelete = mongoTemplate.find(query, LogEntity.class);
+                if (logsToDelete.isEmpty()) {
+                    break;
+                }
+
+                mongoTemplate.remove(query, LogEntity.class);
+                deletedCount += logsToDelete.size();
+                log.info("Cleaned up {} logs for scene: {} before timestamp: {}", logsToDelete.size(), paramDTO.getScene(), cutoffDate);
+
+                // 更新查询条件，使用最后一个日志的ID作为游标
+                LogEntity lastLog = logsToDelete.get(logsToDelete.size() - 1);
+                query.addCriteria(Criteria.where("_id").lt(lastLog.getId()));
+            }
+        } catch (Exception e) {
+            log.error("Error during log cleanup job", e);
+            return ReturnT.FAIL;
+        }
+
+        log.info("Ending log cleanup job, total cleaned up logs for scene {}: {}", paramDTO.getScene(), deletedCount);
+        return ReturnT.SUCCESS;
+    }
+
+    @Data
+    @Builder
+    @NoArgsConstructor
+    @AllArgsConstructor
+    private static class ParamDTO {
+
+        /**
+         * 每次删除的日志数量（页大小）
+         */
+        private int batchSize = 1000;
+
+        /**
+         * 保留天数
+         */
+        private int daysToKeep = 30;
+
+        /**
+         * 场景
+         */
+        private String scene = "networkLog";
+    }
+}