From 9a8815ee3becf07ae508e2f06e41374c68ead9b4 Mon Sep 17 00:00:00 2001 From: siarhei_hrabko Date: Thu, 27 Jun 2024 11:13:07 +0300 Subject: [PATCH 1/4] EPMRPP-88736 implement send analytics job --- .../config/JacksonConfiguration.java | 47 +++++ .../statistics/ManualLaunchStatisticsJob.java | 194 ++++++++++++++++++ src/main/resources/application.yml | 4 + 3 files changed, 245 insertions(+) create mode 100644 src/main/java/com/epam/reportportal/config/JacksonConfiguration.java create mode 100644 src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java diff --git a/src/main/java/com/epam/reportportal/config/JacksonConfiguration.java b/src/main/java/com/epam/reportportal/config/JacksonConfiguration.java new file mode 100644 index 0000000..8cd7255 --- /dev/null +++ b/src/main/java/com/epam/reportportal/config/JacksonConfiguration.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.config; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Siarhei Hrabko + */ +@Configuration +public class JacksonConfiguration { + + /** + * @return Configured object mapper + */ + @Bean(name = "objectMapper") + public ObjectMapper objectMapper() { + ObjectMapper om = new ObjectMapper(); + om.setAnnotationIntrospector(new JacksonAnnotationIntrospector()); + om.configure(MapperFeature.DEFAULT_VIEW_INCLUSION, true); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + om.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + om.registerModule(new JavaTimeModule()); + return om; + } +} diff --git a/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java b/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java new file mode 100644 index 0000000..6fdc3a3 --- /dev/null +++ b/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java @@ -0,0 +1,194 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.jobs.statistics; + +import static org.springframework.http.HttpMethod.POST; + +import com.epam.reportportal.jobs.BaseJob; +import java.security.SecureRandom; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.Set; +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.client.RestTemplate; + +/** + * Sends statistics about amounts of manual analyzed items to the GA4 service. + * + * @author Maksim Antonov + */ +@Service +public class ManualLaunchStatisticsJob extends BaseJob { + + private static final String GA_URL = "https://www.google-analytics.com/mp/collect?measurement_id=%s&api_secret=%s"; + private static final String DATE_BEFORE = "date_before"; + + private static final String SELECT_INSTANCE_ID_QUERY = "SELECT value FROM server_settings WHERE key = 'server.details.instance';"; + private static final String SELECT_ANALYZER_MANUAL_START_QUERY = "SELECT * FROM analytics_data WHERE type = 'ANALYZER_MANUAL_START' AND created_at >= :date_before::TIMESTAMP;"; + private static final String DELETE_ANALYZER_MANUAL_START_QUERY = "DELETE FROM analytics_data WHERE type = 'ANALYZER_MANUAL_START';"; + + private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + private final RestTemplate restTemplate; + + private final String measurementId; + private final String apiSecret; + + + /** + * Initializes {@link ManualLaunchStatisticsJob}. + * + * @param jdbcTemplate {@link JdbcTemplate} + */ + @Autowired + public ManualLaunchStatisticsJob(JdbcTemplate jdbcTemplate, + @Value("${rp.environment.variable.ga.measurementId}") String measurementId, + @Value("${rp.environment.variable.ga.apiSecret}") String apiSecret, + NamedParameterJdbcTemplate namedParameterJdbcTemplate) { + super(jdbcTemplate); + this.measurementId = measurementId; + this.apiSecret = apiSecret; + this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; + this.restTemplate = new RestTemplate(); + } + + + /** + * Sends manual analyzed items statistics. + */ + @Override + @Scheduled(cron = "${rp.environment.variable.ga.cron}") + @SchedulerLock(name = "manualAnalyzedStatisticsJob", lockAtMostFor = "24h") + @Transactional + public void execute() { + LOGGER.info("Start sending analyzer manual start item statistics"); + if (StringUtils.isEmpty(measurementId) || StringUtils.isEmpty(apiSecret)) { + LOGGER.info( + "Both 'measurementId' and 'apiSecret' environment variables should be provided in order to run the job 'manualAnalyzedStatisticsJob'"); + return; + } + + var now = Instant.now(); + var dateBefore = now.minus(1, ChronoUnit.DAYS) + .atOffset(ZoneOffset.UTC) + .toLocalDateTime(); + MapSqlParameterSource queryParams = new MapSqlParameterSource(); + queryParams.addValue(DATE_BEFORE, dateBefore); + + namedParameterJdbcTemplate.query(SELECT_ANALYZER_MANUAL_START_QUERY, queryParams, rs -> { + int autoAnalyzed = 0; + int userAnalyzed = 0; + String version = null; + boolean analyzerEnabled; + Set status = new HashSet<>(); + Set autoAnalysisState = new HashSet<>(); + + do { + var metadata = new JSONObject(rs.getString("metadata")).getJSONObject("metadata"); + + userAnalyzed += metadata.optInt("manuallyAnalyzed"); + autoAnalyzed += metadata.optInt("autoAnalyzed"); + + if (metadata.optInt("manuallyAnalyzed") > 0) { + status.add("manually"); + } else if (metadata.optInt("autoAnalyzed") > 0) { + status.add("automatically"); + } + analyzerEnabled = metadata.getBoolean("analyzerEnabled"); + if (analyzerEnabled) { + autoAnalysisState.add(metadata.getBoolean("auto_analysis") ? "on" : "off"); + } + + if (version == null) { + version = metadata.getString("version"); + } + + } while (rs.next()); + + var instanceId = jdbcTemplate.queryForObject(SELECT_INSTANCE_ID_QUERY, String.class); + var params = new JSONObject(); + params.put("category", "analyzer"); + params.put("instanceID", instanceId); + params.put("timestamp", now.toEpochMilli()); + params.put("version", version); // get from table + params.put("type", analyzerEnabled ? "is_analyzer" : "not_analyzer"); + if (analyzerEnabled) { + params.put("number", autoAnalyzed + "#" + userAnalyzed); + params.put("auto_analysis", String.join("#", autoAnalysisState)); + params.put("status", String.join("#", status)); + } + + var event = new JSONObject(); + event.put("name", "analyze_analyzer"); + event.put("params", params); + + JSONArray events = new JSONArray(); + events.put(event); + + JSONObject requestBody = new JSONObject(); + requestBody.put("client_id", + now.toEpochMilli() + "." + new SecureRandom().nextInt(100_000, 999_999)); + requestBody.put("events", events); + + sendRequest(requestBody); + + }); + + LOGGER.info("Completed analyzer manual start item statistics job"); + + } + + private void sendRequest(JSONObject requestBody) { + try { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Sending statistics data: {}", requestBody); + } + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity request = new HttpEntity<>(requestBody.toString(), headers); + + String url = String.format(GA_URL, measurementId, apiSecret); + + var response = restTemplate.exchange(url, POST, request, String.class); + if (response.getStatusCodeValue() != 204) { + LOGGER.error("Failed to send statistics: {}", response); + } + } catch (Exception e) { + LOGGER.error("Failed to send statistics", e); + } finally { + jdbcTemplate.execute(DELETE_ANALYZER_MANUAL_START_QUERY); + } + } + +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 588149e..26b2ddd 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -39,6 +39,10 @@ rp: project: ## 1 minute cron: '0 */1 * * * *' + ga: + apiSecret: + measurementId: + cron: '0 0 */24 * * *' executor: pool: storage: From 8dd65ab6c3a47d69ef64aacf708ab3836d75ca5e Mon Sep 17 00:00:00 2001 From: siarhei_hrabko Date: Fri, 28 Jun 2024 21:34:14 +0300 Subject: [PATCH 2/4] EPMRPP-88736 code refactor --- .../statistics/ManualLaunchStatisticsJob.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java b/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java index 6fdc3a3..89da10f 100644 --- a/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java +++ b/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java @@ -108,30 +108,31 @@ public void execute() { namedParameterJdbcTemplate.query(SELECT_ANALYZER_MANUAL_START_QUERY, queryParams, rs -> { int autoAnalyzed = 0; int userAnalyzed = 0; + int sentToAnalyze = 0; String version = null; boolean analyzerEnabled; Set status = new HashSet<>(); Set autoAnalysisState = new HashSet<>(); do { - var metadata = new JSONObject(rs.getString("metadata")).getJSONObject("metadata"); + var metadata = new JSONObject(rs.getString("metadata")) + .getJSONObject("metadata"); - userAnalyzed += metadata.optInt("manuallyAnalyzed"); - autoAnalyzed += metadata.optInt("autoAnalyzed"); + analyzerEnabled = metadata.getBoolean("analyzerEnabled"); + if (analyzerEnabled) { + autoAnalysisState.add(metadata.getBoolean("autoAnalysisOn") ? "on" : "off"); + } - if (metadata.optInt("manuallyAnalyzed") > 0) { + if (metadata.optInt("userAnalyzed") > 0) { status.add("manually"); } else if (metadata.optInt("autoAnalyzed") > 0) { status.add("automatically"); } - analyzerEnabled = metadata.getBoolean("analyzerEnabled"); - if (analyzerEnabled) { - autoAnalysisState.add(metadata.getBoolean("auto_analysis") ? "on" : "off"); - } - if (version == null) { - version = metadata.getString("version"); - } + userAnalyzed += metadata.optInt("userAnalyzed"); + autoAnalyzed += metadata.optInt("autoAnalyzed"); + sentToAnalyze += metadata.optInt("userAnalyzed") + metadata.optInt("sentToAnalyze"); + version = metadata.getString("version"); } while (rs.next()); @@ -140,10 +141,10 @@ public void execute() { params.put("category", "analyzer"); params.put("instanceID", instanceId); params.put("timestamp", now.toEpochMilli()); - params.put("version", version); // get from table + params.put("version", version); params.put("type", analyzerEnabled ? "is_analyzer" : "not_analyzer"); if (analyzerEnabled) { - params.put("number", autoAnalyzed + "#" + userAnalyzed); + params.put("number", autoAnalyzed + "#" + userAnalyzed + "#" + sentToAnalyze); params.put("auto_analysis", String.join("#", autoAnalysisState)); params.put("status", String.join("#", status)); } From ba6129fc09638ac593012571ce04cbdf1cd940e4 Mon Sep 17 00:00:00 2001 From: siarhei_hrabko Date: Mon, 1 Jul 2024 12:41:30 +0300 Subject: [PATCH 3/4] EPMRPP-88736 code refactor --- .../reportportal/jobs/statistics/ManualLaunchStatisticsJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java b/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java index 89da10f..0152573 100644 --- a/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java +++ b/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java @@ -125,7 +125,7 @@ public void execute() { if (metadata.optInt("userAnalyzed") > 0) { status.add("manually"); - } else if (metadata.optInt("autoAnalyzed") > 0) { + } else { status.add("automatically"); } From 8843299c2e7f6abdd2328d29f5d83f5894ffb685 Mon Sep 17 00:00:00 2001 From: siarhei_hrabko Date: Mon, 1 Jul 2024 14:52:30 +0300 Subject: [PATCH 4/4] EPMRPP-88736 code refactor --- ...ob.java => DefectUpdateStatisticsJob.java} | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) rename src/main/java/com/epam/reportportal/jobs/statistics/{ManualLaunchStatisticsJob.java => DefectUpdateStatisticsJob.java} (84%) diff --git a/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java b/src/main/java/com/epam/reportportal/jobs/statistics/DefectUpdateStatisticsJob.java similarity index 84% rename from src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java rename to src/main/java/com/epam/reportportal/jobs/statistics/DefectUpdateStatisticsJob.java index 0152573..a193ed8 100644 --- a/src/main/java/com/epam/reportportal/jobs/statistics/ManualLaunchStatisticsJob.java +++ b/src/main/java/com/epam/reportportal/jobs/statistics/DefectUpdateStatisticsJob.java @@ -48,14 +48,14 @@ * @author Maksim Antonov */ @Service -public class ManualLaunchStatisticsJob extends BaseJob { +public class DefectUpdateStatisticsJob extends BaseJob { private static final String GA_URL = "https://www.google-analytics.com/mp/collect?measurement_id=%s&api_secret=%s"; private static final String DATE_BEFORE = "date_before"; private static final String SELECT_INSTANCE_ID_QUERY = "SELECT value FROM server_settings WHERE key = 'server.details.instance';"; - private static final String SELECT_ANALYZER_MANUAL_START_QUERY = "SELECT * FROM analytics_data WHERE type = 'ANALYZER_MANUAL_START' AND created_at >= :date_before::TIMESTAMP;"; - private static final String DELETE_ANALYZER_MANUAL_START_QUERY = "DELETE FROM analytics_data WHERE type = 'ANALYZER_MANUAL_START';"; + private static final String SELECT_STATISTICS_QUERY = "SELECT * FROM analytics_data WHERE type = 'DEFECT_UPDATE_STATISTICS' AND created_at >= :date_before::TIMESTAMP;"; + private static final String DELETE_STATISTICS_QUERY = "DELETE FROM analytics_data WHERE type = 'DEFECT_UPDATE_STATISTICS';"; private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; @@ -66,12 +66,12 @@ public class ManualLaunchStatisticsJob extends BaseJob { /** - * Initializes {@link ManualLaunchStatisticsJob}. + * Initializes {@link DefectUpdateStatisticsJob}. * * @param jdbcTemplate {@link JdbcTemplate} */ @Autowired - public ManualLaunchStatisticsJob(JdbcTemplate jdbcTemplate, + public DefectUpdateStatisticsJob(JdbcTemplate jdbcTemplate, @Value("${rp.environment.variable.ga.measurementId}") String measurementId, @Value("${rp.environment.variable.ga.apiSecret}") String apiSecret, NamedParameterJdbcTemplate namedParameterJdbcTemplate) { @@ -84,17 +84,17 @@ public ManualLaunchStatisticsJob(JdbcTemplate jdbcTemplate, /** - * Sends manual analyzed items statistics. + * Sends analyzed items statistics. */ @Override @Scheduled(cron = "${rp.environment.variable.ga.cron}") - @SchedulerLock(name = "manualAnalyzedStatisticsJob", lockAtMostFor = "24h") + @SchedulerLock(name = "defectUpdateStatisticsJob", lockAtMostFor = "24h") @Transactional public void execute() { - LOGGER.info("Start sending analyzer manual start item statistics"); + LOGGER.info("Start sending items defect update statistics"); if (StringUtils.isEmpty(measurementId) || StringUtils.isEmpty(apiSecret)) { LOGGER.info( - "Both 'measurementId' and 'apiSecret' environment variables should be provided in order to run the job 'manualAnalyzedStatisticsJob'"); + "Both 'measurementId' and 'apiSecret' environment variables should be provided in order to run the job 'defectUpdateStatisticsJob'"); return; } @@ -105,11 +105,11 @@ public void execute() { MapSqlParameterSource queryParams = new MapSqlParameterSource(); queryParams.addValue(DATE_BEFORE, dateBefore); - namedParameterJdbcTemplate.query(SELECT_ANALYZER_MANUAL_START_QUERY, queryParams, rs -> { + namedParameterJdbcTemplate.query(SELECT_STATISTICS_QUERY, queryParams, rs -> { int autoAnalyzed = 0; int userAnalyzed = 0; int sentToAnalyze = 0; - String version = null; + String version; boolean analyzerEnabled; Set status = new HashSet<>(); Set autoAnalysisState = new HashSet<>(); @@ -118,7 +118,7 @@ public void execute() { var metadata = new JSONObject(rs.getString("metadata")) .getJSONObject("metadata"); - analyzerEnabled = metadata.getBoolean("analyzerEnabled"); + analyzerEnabled = metadata.optBoolean("analyzerEnabled"); if (analyzerEnabled) { autoAnalysisState.add(metadata.getBoolean("autoAnalysisOn") ? "on" : "off"); } @@ -130,7 +130,7 @@ public void execute() { } userAnalyzed += metadata.optInt("userAnalyzed"); - autoAnalyzed += metadata.optInt("autoAnalyzed"); + autoAnalyzed += metadata.optInt("analyzed"); sentToAnalyze += metadata.optInt("userAnalyzed") + metadata.optInt("sentToAnalyze"); version = metadata.getString("version"); @@ -165,7 +165,7 @@ public void execute() { }); - LOGGER.info("Completed analyzer manual start item statistics job"); + LOGGER.info("Completed items defect update statistics job"); } @@ -188,7 +188,7 @@ private void sendRequest(JSONObject requestBody) { } catch (Exception e) { LOGGER.error("Failed to send statistics", e); } finally { - jdbcTemplate.execute(DELETE_ANALYZER_MANUAL_START_QUERY); + jdbcTemplate.execute(DELETE_STATISTICS_QUERY); } }