Skip to content

Commit

Permalink
EPMRPP-90918 || Add skip of important launches
Browse files Browse the repository at this point in the history
  • Loading branch information
@IvanKustau
IvanKustau committed May 23, 2024
1 parent 5ead2ef commit e52b2fa
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 66 deletions.
79 changes: 39 additions & 40 deletions src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@ public class CleanLaunchJob extends BaseCleanJob {
private static final String IDS_PARAM = "ids";
private static final String PROJECT_ID_PARAM = "projectId";
private static final String START_TIME_PARAM = "startTime";
private static final String SELECT_LAUNCH_ID_QUERY = "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= :startTime::TIMESTAMP;";
private static final String DELETE_CLUSTER_QUERY = "DELETE FROM clusters WHERE clusters.launch_id IN (:ids);";
private static final String SELECT_LAUNCH_ID_QUERY =
"SELECT id FROM launch WHERE project_id = :projectId AND start_time <= "
+ ":startTime::TIMESTAMP AND retention_policy = 'REGULAR'";
private static final String DELETE_CLUSTER_QUERY =
"DELETE FROM clusters WHERE clusters.launch_id IN (:ids);";
private static final String DELETE_LAUNCH_QUERY = "DELETE FROM launch WHERE id IN (:ids);";
private final Integer batchSize;
private final NamedParameterJdbcTemplate namedParameterJdbcTemplate;
Expand All @@ -39,9 +42,8 @@ public class CleanLaunchJob extends BaseCleanJob {

public CleanLaunchJob(
@Value("${rp.environment.variable.elements-counter.batch-size}") Integer batchSize,
JdbcTemplate jdbcTemplate,
NamedParameterJdbcTemplate namedParameterJdbcTemplate, CleanLogJob cleanLogJob,
IndexerServiceClient indexerServiceClient,
JdbcTemplate jdbcTemplate, NamedParameterJdbcTemplate namedParameterJdbcTemplate,
CleanLogJob cleanLogJob, IndexerServiceClient indexerServiceClient,
ApplicationEventPublisher eventPublisher, SearchEngineClient searchEngineClient) {
super(jdbcTemplate);
this.batchSize = batchSize;
Expand All @@ -53,23 +55,23 @@ public CleanLaunchJob(
}

@Override
@Scheduled(cron = "${rp.environment.variable.clean.launch.cron}")
@SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h")
public void execute() {
removeLaunches();
cleanLogJob.removeLogs();
}
@Scheduled(cron = "${rp.environment.variable.clean.launch.cron}")
@SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h")
public void execute() {
removeLaunches();
cleanLogJob.removeLogs();
}

private void removeLaunches() {
AtomicInteger counter = new AtomicInteger(0);
getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> {
final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
final List<Long> launchIds = getLaunchIds(projectId, lessThanDate);
if (!launchIds.isEmpty()) {
deleteClusters(launchIds);
// final Long numberOfLaunchElements = countNumberOfLaunchElements(launchIds);
int deleted = namedParameterJdbcTemplate.update(DELETE_LAUNCH_QUERY,
Map.of(IDS_PARAM, launchIds));
private void removeLaunches() {
AtomicInteger counter = new AtomicInteger(0);
getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> {
final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
final List<Long> launchIds = getLaunchIds(projectId, lessThanDate);
if (!launchIds.isEmpty()) {
deleteClusters(launchIds);
// final Long numberOfLaunchElements = countNumberOfLaunchElements(launchIds);
int deleted =
namedParameterJdbcTemplate.update(DELETE_LAUNCH_QUERY, Map.of(IDS_PARAM, launchIds));
counter.addAndGet(deleted);
LOGGER.info("Delete {} launches for project {}", deleted, projectId);
// to avoid error message in analyzer log, doesn't find index
Expand All @@ -79,12 +81,14 @@ private void removeLaunches() {

deleteLogsFromSearchEngineByLaunchIdsAndProjectId(launchIds, projectId);

// eventPublisher.publishEvent(new ElementsDeletedEvent(launchIds, projectId, numberOfLaunchElements));
// LOGGER.info("Send event with elements deleted number {} for project {}", deleted, projectId);
}
}
});
}
// eventPublisher.publishEvent(new ElementsDeletedEvent(launchIds,
// projectId, numberOfLaunchElements));
// LOGGER.info("Send event with elements deleted number {} for
// project {}", deleted, projectId);
}
}
});
}

private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List<Long> launchIds,
Long projectId) {
Expand All @@ -96,8 +100,7 @@ private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List<Long> launch

private List<Long> getLaunchIds(Long projectId, LocalDateTime lessThanDate) {
return namedParameterJdbcTemplate.queryForList(SELECT_LAUNCH_ID_QUERY,
Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate),
Long.class
Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), Long.class
);
}

Expand All @@ -111,20 +114,16 @@ private Long countNumberOfLaunchElements(List<Long> launchIds) {
"SELECT item_id FROM test_item WHERE launch_id IN (:ids) UNION "
+ "SELECT item_id FROM test_item WHERE retry_of IS NOT NULL AND retry_of IN "
+ "(SELECT item_id FROM test_item WHERE launch_id IN (:ids))",
Map.of(IDS_PARAM, launchIds),
Long.class
Map.of(IDS_PARAM, launchIds), Long.class
);
resultedNumber.addAndGet(itemIds.size());
Lists.partition(itemIds, batchSize)
.forEach(batch -> resultedNumber.addAndGet(
Optional.ofNullable(namedParameterJdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM log WHERE item_id IN (:ids);",
Map.of(IDS_PARAM, batch),
Long.class
)).orElse(0L)));
Lists.partition(itemIds, batchSize).forEach(batch -> resultedNumber.addAndGet(
Optional.ofNullable(namedParameterJdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM log WHERE item_id IN (:ids);", Map.of(IDS_PARAM, batch),
Long.class
)).orElse(0L)));
resultedNumber.addAndGet(Optional.ofNullable(namedParameterJdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM log WHERE log.launch_id IN (:ids);",
Map.of(IDS_PARAM, launchIds),
"SELECT COUNT(*) FROM log WHERE log.launch_id IN (:ids);", Map.of(IDS_PARAM, launchIds),
Long.class
)).orElse(0L));
return resultedNumber.longValue();
Expand Down
62 changes: 36 additions & 26 deletions src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,19 @@ public class CleanLogJob extends BaseCleanJob {
private static final String PROJECT_ID_PARAM = "projectId";
private static final String START_TIME_PARAM = "startTime";

private static final String DELETE_LOGS_QUERY = "DELETE FROM log WHERE project_id = ? AND log_time <= ?::TIMESTAMP;";
private static final String SELECT_LAUNCH_ID_QUERY = "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= :startTime::TIMESTAMP;";
private static final String DELETE_LOGS_QUERY = """
DELETE FROM log
WHERE log.project_id = ? AND log.log_time <= ?::TIMESTAMP
AND COALESCE(log.launch_id,
(SELECT test_item.launch_id FROM test_item WHERE test_item.item_id = log.item_id),
(SELECT test_item.launch_id FROM test_item WHERE test_item.item_id =
(SELECT ti.retry_of FROM test_item ti WHERE ti.item_id = log.item_id)
)
) IN (SELECT launch.id FROM launch WHERE launch.retention_policy = 'REGULAR');
""";
private static final String SELECT_LAUNCH_ID_QUERY =
"SELECT id FROM launch WHERE project_id = :projectId AND start_time <= "
+ ":startTime::TIMESTAMP;";

private final CleanAttachmentJob cleanAttachmentJob;
private final IndexerServiceClient indexerServiceClient;
Expand All @@ -45,34 +56,34 @@ public CleanLogJob(JdbcTemplate jdbcTemplate, CleanAttachmentJob cleanAttachment
}

@Override
@Scheduled(cron = "${rp.environment.variable.clean.log.cron}")
@SchedulerLock(name = "cleanLog", lockAtMostFor = "24h")
public void execute() {
removeLogs();
cleanAttachmentJob.moveAttachments();
}
@Scheduled(cron = "${rp.environment.variable.clean.log.cron}")
@SchedulerLock(name = "cleanLog", lockAtMostFor = "24h")
public void execute() {
removeLogs();
cleanAttachmentJob.moveAttachments();
}

void removeLogs() {
AtomicInteger counter = new AtomicInteger(0);
// TODO: Need to refactor Logs to keep real it's launchId and combine code with
// CleanLaunch to avoid duplication
getProjectsWithAttribute(KEEP_LOGS).forEach((projectId, duration) -> {
final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
int deleted = jdbcTemplate.update(DELETE_LOGS_QUERY, projectId, lessThanDate);
counter.addAndGet(deleted);
LOGGER.info("Delete {} logs for project {}", deleted, projectId);
// to avoid error message in analyzer log, doesn't find index
if (deleted > 0) {
indexerServiceClient.removeFromIndexLessThanLogDate(projectId, lessThanDate);
LOGGER.info("Send message for deletion to analyzer for project {}", projectId);
void removeLogs() {
AtomicInteger counter = new AtomicInteger(0);
// TODO: Need to refactor Logs to keep real it's launchId and combine code with
// CleanLaunch to avoid duplication
getProjectsWithAttribute(KEEP_LOGS).forEach((projectId, duration) -> {
final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
int deleted = jdbcTemplate.update(DELETE_LOGS_QUERY, projectId, lessThanDate);
counter.addAndGet(deleted);
LOGGER.info("Delete {} logs for project {}", deleted, projectId);
// to avoid error message in analyzer log, doesn't find index
if (deleted > 0) {
indexerServiceClient.removeFromIndexLessThanLogDate(projectId, lessThanDate);
LOGGER.info("Send message for deletion to analyzer for project {}", projectId);

final List<Long> launchIds = getLaunchIds(projectId, lessThanDate);
if (!launchIds.isEmpty()) {
deleteLogsFromSearchEngineByLaunchIdsAndProjectId(launchIds, projectId);
}
}
});
}
}
});
}

private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List<Long> launchIds,
Long projectId) {
Expand All @@ -84,8 +95,7 @@ private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List<Long> launch

private List<Long> getLaunchIds(Long projectId, LocalDateTime lessThanDate) {
return namedParameterJdbcTemplate.queryForList(SELECT_LAUNCH_ID_QUERY,
Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate),
Long.class
Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), Long.class
);
}
}

0 comments on commit e52b2fa

Please sign in to comment.