diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java index a43ce5a..d519999 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java @@ -27,8 +27,11 @@ public class CleanLaunchJob extends BaseCleanJob { private static final String IDS_PARAM = "ids"; private static final String PROJECT_ID_PARAM = "projectId"; private static final String START_TIME_PARAM = "startTime"; - private static final String SELECT_LAUNCH_ID_QUERY = "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= :startTime::TIMESTAMP;"; - private static final String DELETE_CLUSTER_QUERY = "DELETE FROM clusters WHERE clusters.launch_id IN (:ids);"; + private static final String SELECT_LAUNCH_ID_QUERY = + "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= " + + ":startTime::TIMESTAMP AND retention_policy = 'REGULAR'"; + private static final String DELETE_CLUSTER_QUERY = + "DELETE FROM clusters WHERE clusters.launch_id IN (:ids);"; private static final String DELETE_LAUNCH_QUERY = "DELETE FROM launch WHERE id IN (:ids);"; private final Integer batchSize; private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; @@ -39,9 +42,8 @@ public class CleanLaunchJob extends BaseCleanJob { public CleanLaunchJob( @Value("${rp.environment.variable.elements-counter.batch-size}") Integer batchSize, - JdbcTemplate jdbcTemplate, - NamedParameterJdbcTemplate namedParameterJdbcTemplate, CleanLogJob cleanLogJob, - IndexerServiceClient indexerServiceClient, + JdbcTemplate jdbcTemplate, NamedParameterJdbcTemplate namedParameterJdbcTemplate, + CleanLogJob cleanLogJob, IndexerServiceClient indexerServiceClient, ApplicationEventPublisher eventPublisher, SearchEngineClient searchEngineClient) { super(jdbcTemplate); this.batchSize = batchSize; @@ -53,23 +55,23 @@ public CleanLaunchJob( } @Override - @Scheduled(cron = "${rp.environment.variable.clean.launch.cron}") - @SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h") - public void execute() { - removeLaunches(); - cleanLogJob.removeLogs(); - } + @Scheduled(cron = "${rp.environment.variable.clean.launch.cron}") + @SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h") + public void execute() { + removeLaunches(); + cleanLogJob.removeLogs(); + } - private void removeLaunches() { - AtomicInteger counter = new AtomicInteger(0); - getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> { - final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); - final List launchIds = getLaunchIds(projectId, lessThanDate); - if (!launchIds.isEmpty()) { - deleteClusters(launchIds); -// final Long numberOfLaunchElements = countNumberOfLaunchElements(launchIds); - int deleted = namedParameterJdbcTemplate.update(DELETE_LAUNCH_QUERY, - Map.of(IDS_PARAM, launchIds)); + private void removeLaunches() { + AtomicInteger counter = new AtomicInteger(0); + getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> { + final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); + final List launchIds = getLaunchIds(projectId, lessThanDate); + if (!launchIds.isEmpty()) { + deleteClusters(launchIds); + // final Long numberOfLaunchElements = countNumberOfLaunchElements(launchIds); + int deleted = + namedParameterJdbcTemplate.update(DELETE_LAUNCH_QUERY, Map.of(IDS_PARAM, launchIds)); counter.addAndGet(deleted); LOGGER.info("Delete {} launches for project {}", deleted, projectId); // to avoid error message in analyzer log, doesn't find index @@ -79,12 +81,14 @@ private void removeLaunches() { deleteLogsFromSearchEngineByLaunchIdsAndProjectId(launchIds, projectId); -// eventPublisher.publishEvent(new ElementsDeletedEvent(launchIds, projectId, numberOfLaunchElements)); -// LOGGER.info("Send event with elements deleted number {} for project {}", deleted, projectId); - } - } - }); - } + // eventPublisher.publishEvent(new ElementsDeletedEvent(launchIds, + // projectId, numberOfLaunchElements)); + // LOGGER.info("Send event with elements deleted number {} for + // project {}", deleted, projectId); + } + } + }); + } private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List launchIds, Long projectId) { @@ -96,8 +100,7 @@ private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List launch private List getLaunchIds(Long projectId, LocalDateTime lessThanDate) { return namedParameterJdbcTemplate.queryForList(SELECT_LAUNCH_ID_QUERY, - Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), - Long.class + Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), Long.class ); } @@ -111,20 +114,16 @@ private Long countNumberOfLaunchElements(List launchIds) { "SELECT item_id FROM test_item WHERE launch_id IN (:ids) UNION " + "SELECT item_id FROM test_item WHERE retry_of IS NOT NULL AND retry_of IN " + "(SELECT item_id FROM test_item WHERE launch_id IN (:ids))", - Map.of(IDS_PARAM, launchIds), - Long.class + Map.of(IDS_PARAM, launchIds), Long.class ); resultedNumber.addAndGet(itemIds.size()); - Lists.partition(itemIds, batchSize) - .forEach(batch -> resultedNumber.addAndGet( - Optional.ofNullable(namedParameterJdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM log WHERE item_id IN (:ids);", - Map.of(IDS_PARAM, batch), - Long.class - )).orElse(0L))); + Lists.partition(itemIds, batchSize).forEach(batch -> resultedNumber.addAndGet( + Optional.ofNullable(namedParameterJdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM log WHERE item_id IN (:ids);", Map.of(IDS_PARAM, batch), + Long.class + )).orElse(0L))); resultedNumber.addAndGet(Optional.ofNullable(namedParameterJdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM log WHERE log.launch_id IN (:ids);", - Map.of(IDS_PARAM, launchIds), + "SELECT COUNT(*) FROM log WHERE log.launch_id IN (:ids);", Map.of(IDS_PARAM, launchIds), Long.class )).orElse(0L)); return resultedNumber.longValue(); diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java index 20c25d6..55fa6c1 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java @@ -23,8 +23,19 @@ public class CleanLogJob extends BaseCleanJob { private static final String PROJECT_ID_PARAM = "projectId"; private static final String START_TIME_PARAM = "startTime"; - private static final String DELETE_LOGS_QUERY = "DELETE FROM log WHERE project_id = ? AND log_time <= ?::TIMESTAMP;"; - private static final String SELECT_LAUNCH_ID_QUERY = "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= :startTime::TIMESTAMP;"; + private static final String DELETE_LOGS_QUERY = """ + DELETE FROM log + WHERE log.project_id = ? AND log.log_time <= ?::TIMESTAMP + AND COALESCE(log.launch_id, + (SELECT test_item.launch_id FROM test_item WHERE test_item.item_id = log.item_id), + (SELECT test_item.launch_id FROM test_item WHERE test_item.item_id = + (SELECT ti.retry_of FROM test_item ti WHERE ti.item_id = log.item_id) + ) + ) IN (SELECT launch.id FROM launch WHERE launch.retention_policy = 'REGULAR'); + """; + private static final String SELECT_LAUNCH_ID_QUERY = + "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= " + + ":startTime::TIMESTAMP;"; private final CleanAttachmentJob cleanAttachmentJob; private final IndexerServiceClient indexerServiceClient; @@ -45,34 +56,34 @@ public CleanLogJob(JdbcTemplate jdbcTemplate, CleanAttachmentJob cleanAttachment } @Override - @Scheduled(cron = "${rp.environment.variable.clean.log.cron}") - @SchedulerLock(name = "cleanLog", lockAtMostFor = "24h") - public void execute() { - removeLogs(); - cleanAttachmentJob.moveAttachments(); - } + @Scheduled(cron = "${rp.environment.variable.clean.log.cron}") + @SchedulerLock(name = "cleanLog", lockAtMostFor = "24h") + public void execute() { + removeLogs(); + cleanAttachmentJob.moveAttachments(); + } - void removeLogs() { - AtomicInteger counter = new AtomicInteger(0); - // TODO: Need to refactor Logs to keep real it's launchId and combine code with - // CleanLaunch to avoid duplication - getProjectsWithAttribute(KEEP_LOGS).forEach((projectId, duration) -> { - final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); - int deleted = jdbcTemplate.update(DELETE_LOGS_QUERY, projectId, lessThanDate); - counter.addAndGet(deleted); - LOGGER.info("Delete {} logs for project {}", deleted, projectId); - // to avoid error message in analyzer log, doesn't find index - if (deleted > 0) { - indexerServiceClient.removeFromIndexLessThanLogDate(projectId, lessThanDate); - LOGGER.info("Send message for deletion to analyzer for project {}", projectId); + void removeLogs() { + AtomicInteger counter = new AtomicInteger(0); + // TODO: Need to refactor Logs to keep real it's launchId and combine code with + // CleanLaunch to avoid duplication + getProjectsWithAttribute(KEEP_LOGS).forEach((projectId, duration) -> { + final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); + int deleted = jdbcTemplate.update(DELETE_LOGS_QUERY, projectId, lessThanDate); + counter.addAndGet(deleted); + LOGGER.info("Delete {} logs for project {}", deleted, projectId); + // to avoid error message in analyzer log, doesn't find index + if (deleted > 0) { + indexerServiceClient.removeFromIndexLessThanLogDate(projectId, lessThanDate); + LOGGER.info("Send message for deletion to analyzer for project {}", projectId); final List launchIds = getLaunchIds(projectId, lessThanDate); if (!launchIds.isEmpty()) { deleteLogsFromSearchEngineByLaunchIdsAndProjectId(launchIds, projectId); } - } - }); - } + } + }); + } private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List launchIds, Long projectId) { @@ -84,8 +95,7 @@ private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List launch private List getLaunchIds(Long projectId, LocalDateTime lessThanDate) { return namedParameterJdbcTemplate.queryForList(SELECT_LAUNCH_ID_QUERY, - Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), - Long.class + Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), Long.class ); } }