Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EPMRPP-90918 || Jobs should skip important #136

Merged
merged 2 commits into from
May 28, 2024
Merged

EPMRPP-90918 || Jobs should skip important #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e52b2fa
EPMRPP-90918 || Add skip of important launches
IvanKustau May 23, 2024
71fd27e
EPMRPP-90918 || Add skip attachments
IvanKustau May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 30 additions & 19 deletions src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,30 +17,41 @@
@Service
public class CleanAttachmentJob extends BaseCleanJob {

private static final String MOVING_QUERY =
"""
WITH moved_rows AS (DELETE FROM attachment WHERE project_id = ? AND creation_date <= ?::TIMESTAMP RETURNING *) \s
INSERT INTO attachment_deletion (id, file_id, thumbnail_id, creation_attachment_date, deletion_date)\s
SELECT id, file_id, thumbnail_id, creation_date, NOW() FROM moved_rows;""";
private static final String MOVING_QUERY = """
WITH moved_rows AS (
DELETE FROM attachment\s
WHERE project_id = ?\s
AND creation_date <= ?::TIMESTAMP\s
AND launch_id IN (
SELECT id FROM launch WHERE retention_policy='REGULAR'
)\s
RETURNING *
)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.whitespace.FileTabCharacterCheck> reported by reviewdog 🐶
Line contains a tab character.

INSERT INTO attachment_deletion (id, file_id, thumbnail_id, creation_attachment_date,
deletion_date)
SELECT id, file_id, thumbnail_id, creation_date, NOW() FROM moved_rows;""";

public CleanAttachmentJob(JdbcTemplate jdbcTemplate) {
super(jdbcTemplate);
}

@Override
@Scheduled(cron = "${rp.environment.variable.clean.attachment.cron}")
@SchedulerLock(name = "cleanAttachment", lockAtMostFor = "24h")
public void execute() {
moveAttachments();
}
@Scheduled(cron = "${rp.environment.variable.clean.attachment.cron}")
@SchedulerLock(name = "cleanAttachment", lockAtMostFor = "24h")
public void execute() {
moveAttachments();
}

void moveAttachments() {
AtomicInteger counter = new AtomicInteger(0);
getProjectsWithAttribute(KEEP_SCREENSHOTS).forEach((projectId, duration) -> {
LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
int movedCount = jdbcTemplate.update(MOVING_QUERY, projectId, lessThanDate);
counter.addAndGet(movedCount);
LOGGER.info("Moved {} attachments to the deletion table for project {}, lessThanDate {} ", movedCount, projectId, lessThanDate);
});
}
void moveAttachments() {
AtomicInteger counter = new AtomicInteger(0);
getProjectsWithAttribute(KEEP_SCREENSHOTS).forEach((projectId, duration) -> {
LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
int movedCount = jdbcTemplate.update(MOVING_QUERY, projectId, lessThanDate);
counter.addAndGet(movedCount);
LOGGER.info(
"Moved {} attachments to the deletion table for project {}, lessThanDate {} ", movedCount,
projectId, lessThanDate
);
});
}
}
79 changes: 39 additions & 40 deletions src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@ public class CleanLaunchJob extends BaseCleanJob {
private static final String IDS_PARAM = "ids";
private static final String PROJECT_ID_PARAM = "projectId";
private static final String START_TIME_PARAM = "startTime";
private static final String SELECT_LAUNCH_ID_QUERY = "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= :startTime::TIMESTAMP;";
private static final String DELETE_CLUSTER_QUERY = "DELETE FROM clusters WHERE clusters.launch_id IN (:ids);";
private static final String SELECT_LAUNCH_ID_QUERY =
"SELECT id FROM launch WHERE project_id = :projectId AND start_time <= "
+ ":startTime::TIMESTAMP AND retention_policy = 'REGULAR'";
private static final String DELETE_CLUSTER_QUERY =
"DELETE FROM clusters WHERE clusters.launch_id IN (:ids);";
private static final String DELETE_LAUNCH_QUERY = "DELETE FROM launch WHERE id IN (:ids);";
private final Integer batchSize;
private final NamedParameterJdbcTemplate namedParameterJdbcTemplate;
Expand All @@ -39,9 +42,8 @@ public class CleanLaunchJob extends BaseCleanJob {

public CleanLaunchJob(
@Value("${rp.environment.variable.elements-counter.batch-size}") Integer batchSize,
JdbcTemplate jdbcTemplate,
NamedParameterJdbcTemplate namedParameterJdbcTemplate, CleanLogJob cleanLogJob,
IndexerServiceClient indexerServiceClient,
JdbcTemplate jdbcTemplate, NamedParameterJdbcTemplate namedParameterJdbcTemplate,
CleanLogJob cleanLogJob, IndexerServiceClient indexerServiceClient,
ApplicationEventPublisher eventPublisher, SearchEngineClient searchEngineClient) {
super(jdbcTemplate);
this.batchSize = batchSize;
Expand All @@ -53,23 +55,23 @@ public CleanLaunchJob(
}

@Override
@Scheduled(cron = "${rp.environment.variable.clean.launch.cron}")
@SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h")
public void execute() {
removeLaunches();
cleanLogJob.removeLogs();
}
@Scheduled(cron = "${rp.environment.variable.clean.launch.cron}")
@SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h")
public void execute() {
removeLaunches();
cleanLogJob.removeLogs();
}

private void removeLaunches() {
AtomicInteger counter = new AtomicInteger(0);
getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> {
final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
final List<Long> launchIds = getLaunchIds(projectId, lessThanDate);
if (!launchIds.isEmpty()) {
deleteClusters(launchIds);
// final Long numberOfLaunchElements = countNumberOfLaunchElements(launchIds);
int deleted = namedParameterJdbcTemplate.update(DELETE_LAUNCH_QUERY,
Map.of(IDS_PARAM, launchIds));
private void removeLaunches() {
AtomicInteger counter = new AtomicInteger(0);
getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> {
final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
final List<Long> launchIds = getLaunchIds(projectId, lessThanDate);
if (!launchIds.isEmpty()) {
deleteClusters(launchIds);
// final Long numberOfLaunchElements = countNumberOfLaunchElements(launchIds);
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.sizes.LineLengthCheck> reported by reviewdog 🐶
Line is longer than 100 characters (found 115).

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.whitespace.FileTabCharacterCheck> reported by reviewdog 🐶
Line contains a tab character.

int deleted =
namedParameterJdbcTemplate.update(DELETE_LAUNCH_QUERY, Map.of(IDS_PARAM, launchIds));
counter.addAndGet(deleted);
LOGGER.info("Delete {} launches for project {}", deleted, projectId);
// to avoid error message in analyzer log, doesn't find index
Expand All @@ -79,12 +81,14 @@ private void removeLaunches() {

deleteLogsFromSearchEngineByLaunchIdsAndProjectId(launchIds, projectId);

// eventPublisher.publishEvent(new ElementsDeletedEvent(launchIds, projectId, numberOfLaunchElements));
// LOGGER.info("Send event with elements deleted number {} for project {}", deleted, projectId);
}
}
});
}
// eventPublisher.publishEvent(new ElementsDeletedEvent(launchIds,
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.sizes.LineLengthCheck> reported by reviewdog 🐶
Line is longer than 100 characters (found 111).

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.whitespace.FileTabCharacterCheck> reported by reviewdog 🐶
Line contains a tab character.

// projectId, numberOfLaunchElements));
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.whitespace.FileTabCharacterCheck> reported by reviewdog 🐶
Line contains a tab character.

// LOGGER.info("Send event with elements deleted number {} for
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.sizes.LineLengthCheck> reported by reviewdog 🐶
Line is longer than 100 characters (found 107).

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.whitespace.FileTabCharacterCheck> reported by reviewdog 🐶
Line contains a tab character.

// project {}", deleted, projectId);
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.whitespace.FileTabCharacterCheck> reported by reviewdog 🐶
Line contains a tab character.

}
}
});
}

private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List<Long> launchIds,
Long projectId) {
Expand All @@ -96,8 +100,7 @@ private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List<Long> launch

private List<Long> getLaunchIds(Long projectId, LocalDateTime lessThanDate) {
return namedParameterJdbcTemplate.queryForList(SELECT_LAUNCH_ID_QUERY,
Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate),
Long.class
Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), Long.class
);
}

Expand All @@ -111,20 +114,16 @@ private Long countNumberOfLaunchElements(List<Long> launchIds) {
"SELECT item_id FROM test_item WHERE launch_id IN (:ids) UNION "
+ "SELECT item_id FROM test_item WHERE retry_of IS NOT NULL AND retry_of IN "
+ "(SELECT item_id FROM test_item WHERE launch_id IN (:ids))",
Map.of(IDS_PARAM, launchIds),
Long.class
Map.of(IDS_PARAM, launchIds), Long.class
);
resultedNumber.addAndGet(itemIds.size());
Lists.partition(itemIds, batchSize)
.forEach(batch -> resultedNumber.addAndGet(
Optional.ofNullable(namedParameterJdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM log WHERE item_id IN (:ids);",
Map.of(IDS_PARAM, batch),
Long.class
)).orElse(0L)));
Lists.partition(itemIds, batchSize).forEach(batch -> resultedNumber.addAndGet(
Optional.ofNullable(namedParameterJdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM log WHERE item_id IN (:ids);", Map.of(IDS_PARAM, batch),
Long.class
)).orElse(0L)));
resultedNumber.addAndGet(Optional.ofNullable(namedParameterJdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM log WHERE log.launch_id IN (:ids);",
Map.of(IDS_PARAM, launchIds),
"SELECT COUNT(*) FROM log WHERE log.launch_id IN (:ids);", Map.of(IDS_PARAM, launchIds),
Long.class
)).orElse(0L));
return resultedNumber.longValue();
Expand Down
62 changes: 36 additions & 26 deletions src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,19 @@ public class CleanLogJob extends BaseCleanJob {
private static final String PROJECT_ID_PARAM = "projectId";
private static final String START_TIME_PARAM = "startTime";

private static final String DELETE_LOGS_QUERY = "DELETE FROM log WHERE project_id = ? AND log_time <= ?::TIMESTAMP;";
private static final String SELECT_LAUNCH_ID_QUERY = "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= :startTime::TIMESTAMP;";
private static final String DELETE_LOGS_QUERY = """
DELETE FROM log
WHERE log.project_id = ? AND log.log_time <= ?::TIMESTAMP
AND COALESCE(log.launch_id,
(SELECT test_item.launch_id FROM test_item WHERE test_item.item_id = log.item_id),
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.sizes.LineLengthCheck> reported by reviewdog 🐶
Line is longer than 100 characters (found 103).

(SELECT test_item.launch_id FROM test_item WHERE test_item.item_id =
(SELECT ti.retry_of FROM test_item ti WHERE ti.item_id = log.item_id)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ [reviewdog] <com.puppycrawl.tools.checkstyle.checks.sizes.LineLengthCheck> reported by reviewdog 🐶
Line is longer than 100 characters (found 139).

)
) IN (SELECT launch.id FROM launch WHERE launch.retention_policy = 'REGULAR');
""";
private static final String SELECT_LAUNCH_ID_QUERY =
"SELECT id FROM launch WHERE project_id = :projectId AND start_time <= "
+ ":startTime::TIMESTAMP;";

private final CleanAttachmentJob cleanAttachmentJob;
private final IndexerServiceClient indexerServiceClient;
Expand All @@ -45,34 +56,34 @@ public CleanLogJob(JdbcTemplate jdbcTemplate, CleanAttachmentJob cleanAttachment
}

@Override
@Scheduled(cron = "${rp.environment.variable.clean.log.cron}")
@SchedulerLock(name = "cleanLog", lockAtMostFor = "24h")
public void execute() {
removeLogs();
cleanAttachmentJob.moveAttachments();
}
@Scheduled(cron = "${rp.environment.variable.clean.log.cron}")
@SchedulerLock(name = "cleanLog", lockAtMostFor = "24h")
public void execute() {
removeLogs();
cleanAttachmentJob.moveAttachments();
}

void removeLogs() {
AtomicInteger counter = new AtomicInteger(0);
// TODO: Need to refactor Logs to keep real it's launchId and combine code with
// CleanLaunch to avoid duplication
getProjectsWithAttribute(KEEP_LOGS).forEach((projectId, duration) -> {
final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
int deleted = jdbcTemplate.update(DELETE_LOGS_QUERY, projectId, lessThanDate);
counter.addAndGet(deleted);
LOGGER.info("Delete {} logs for project {}", deleted, projectId);
// to avoid error message in analyzer log, doesn't find index
if (deleted > 0) {
indexerServiceClient.removeFromIndexLessThanLogDate(projectId, lessThanDate);
LOGGER.info("Send message for deletion to analyzer for project {}", projectId);
void removeLogs() {
AtomicInteger counter = new AtomicInteger(0);
// TODO: Need to refactor Logs to keep real it's launchId and combine code with
// CleanLaunch to avoid duplication
getProjectsWithAttribute(KEEP_LOGS).forEach((projectId, duration) -> {
final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration);
int deleted = jdbcTemplate.update(DELETE_LOGS_QUERY, projectId, lessThanDate);
counter.addAndGet(deleted);
LOGGER.info("Delete {} logs for project {}", deleted, projectId);
// to avoid error message in analyzer log, doesn't find index
if (deleted > 0) {
indexerServiceClient.removeFromIndexLessThanLogDate(projectId, lessThanDate);
LOGGER.info("Send message for deletion to analyzer for project {}", projectId);

final List<Long> launchIds = getLaunchIds(projectId, lessThanDate);
if (!launchIds.isEmpty()) {
deleteLogsFromSearchEngineByLaunchIdsAndProjectId(launchIds, projectId);
}
}
});
}
}
});
}

private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List<Long> launchIds,
Long projectId) {
Expand All @@ -84,8 +95,7 @@ private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List<Long> launch

private List<Long> getLaunchIds(Long projectId, LocalDateTime lessThanDate) {
return namedParameterJdbcTemplate.queryForList(SELECT_LAUNCH_ID_QUERY,
Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate),
Long.class
Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), Long.class
);
}
}
Loading