定时任务删除MongoDB历史数据

ops/2024/11/30 5:51:00/

前言

MongoDB数据过多，导致存储成本飙升，为了降低成本，需要将历史数据删除。

删除逻辑

添加配置文件控制删除逻辑

syncconfig:deleteMongoConfig:#同步状态，true同步，false不同步syncStatus: true#删除数据的时间，一个月前deleteDays: 31#分布式锁keylockRedisKey: order.sync.deleteMongo#每次删除数量limit: 2

配置类

@ConfigurationProperties(prefix = "syncconfig")
@Configuration
@RefreshScope
@Data
public class SyncConfig {private DeleteMongoDto deleteMongoConfig;@Data@NoArgsConstructor@AllArgsConstructorpublic static class DeleteMongoDto {/*** 同步状态，true同步，false不同步*/Boolean syncStatus = false;/*** 删除的天数（createTime）*/Integer deleteDays = 186;/*** locakRedisKey，用于做分布式锁，防止定时任务执行时间较长，两个任务重叠执行*/String lockRedisKey;/*** 每次数量*/Integer limit = 1000;}
}

核心代码

    @PostMapping("/deleteMongo")public BaseResponse<String> deleteMongo() {return orderService.deleteMongo();}

 @Overridepublic BaseResponse<String> deleteMongo() {//获取配置信息SyncConfig.DeleteMongoDto deleteMongoDto = syncConfig.getDeleteMongoConfig();//判断同步是否开启if (deleteMongoDto == null || !deleteMongoDto.getSyncStatus()) {log.info("deleteMongosyncStatus=false,not sync");return ResponseUtils.success("deleteMongosyncStatus=false,not sync");}Boolean lockStatus = false;try {lockStatus = redisLockService.acquireLock(deleteMongoDto.getLockRedisKey());if (!lockStatus) {log.info("deleteMongo get lock fail,not sync");return ResponseUtils.fail("deleteMongo get lock fail,not sync");}//根据时间获取orderIdLong orderId = getOrderId(deleteMongoDto.getDeleteDays());Query query = new Query(Criteria.where("orderId").lt(orderId));query.limit(deleteMongoDto.getLimit());// 使用投影来只选择orderId字段query.fields().include("orderId");// 这里不用分页是因为分页会查询total效率较低，数据量大的时候耗费资源/*Pageable pageable = PageRequest.of(0, deleteMongoDto.getLimit());Page<orderDao> page = orderDao.findByOrderIdLessThan(orderId, pageable);List<Order> list = page.getContent();*/List<Order> list=mongoTemplate.find(query,Order.class);for (Orderitem :list) {orderDao.deleteById(item.getOrderId());}int count = 0;if (CollectionUtils.isNotEmpty(list)) {count = list.size();}return ResponseUtils.success("deleteMongo success,count:" + count);} catch (Exception ex) {log.error("deleteMongo Exception：" + ex.getMessage());return ResponseUtils.fail(ex.getMessage());} finally {//释放分布式锁if (lockStatus) {redisLockService.releaseLock(deleteMongoDto.getLockRedisKey());}}}//生成半年前的orderIdprivate Long getOrderId(Integer deleteDays) {LocalDateTime deleteTime = LocalDateTime.now().minusDays(deleteDays);// 将LocalDateTime转换为毫秒值ZonedDateTime utcZonedDateTime = deleteTime.atZone(ZoneId.systemDefault());long milliseconds = utcZonedDateTime.toInstant().toEpochMilli();Long orderId = (milliseconds / 1000) << 32;return orderId;}