From 9cc9ec92b4833a7e4ab9d9cc9c4ad068386d53f1 Mon Sep 17 00:00:00 2001 From: mzumi Date: Fri, 10 Nov 2023 08:13:06 +0000 Subject: [PATCH 1/6] Supoort resource type of change_event --- .../embulk/input/google_ads/GoogleAdsReporter.java | 11 ++++++++++- .../java/org/embulk/input/google_ads/PluginTask.java | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index a03ac70..5a8dff3 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -219,6 +219,11 @@ public String buildQuery(PluginTask task) sb.append(String.join(" AND ", whereClause)); } + if (task.getLimit().isPresent()) { + sb.append(" LIMIT "); + sb.append(task.getLimit().get()); + } + return sb.toString(); } @@ -231,7 +236,11 @@ public List buildWhereClauseConditions(PluginTask task) if (task.getDateRange().isPresent()) { StringBuilder dateSb = new StringBuilder(); - dateSb.append("segments.date BETWEEN '"); + if (task.getResourceType().equals("change_event")) { + dateSb.append("change_event.change_date_time BETWEEN '"); + } else { + dateSb.append("segments.date BETWEEN '"); + } dateSb.append(task.getDateRange().get().getStartDate()); dateSb.append("' AND '"); dateSb.append(task.getDateRange().get().getEndDate()); diff --git a/src/main/java/org/embulk/input/google_ads/PluginTask.java b/src/main/java/org/embulk/input/google_ads/PluginTask.java index f5b1b28..ced63a8 100644 --- a/src/main/java/org/embulk/input/google_ads/PluginTask.java +++ b/src/main/java/org/embulk/input/google_ads/PluginTask.java @@ -43,6 +43,10 @@ public interface PluginTask extends Task @ConfigDefault("null") Optional getDateRange(); + @Config("limit") + @ConfigDefault("null") + Optional getLimit(); + @Config("_use_micro") @ConfigDefault("true") boolean getUseMicro(); From 87f09745928b05dbaf0e89d8ea9f54e6fe639bd7 Mon Sep 17 00:00:00 2001 From: mzumi Date: Thu, 7 Dec 2023 23:48:07 +0000 Subject: [PATCH 2/6] Change query to get over limited rows --- .../input/google_ads/GoogleAdsReporter.java | 72 +++++++++++++++---- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index 5a8dff3..8eb0a4b 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.ads.googleads.lib.GoogleAdsClient; +import com.google.ads.googleads.v13.services.GoogleAdsRow; import com.google.ads.googleads.v13.services.GoogleAdsServiceClient; import com.google.ads.googleads.v13.services.SearchGoogleAdsRequest; import com.google.auth.oauth2.UserCredentials; @@ -27,6 +28,7 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; public class GoogleAdsReporter { @@ -54,12 +56,33 @@ private UserCredentials buildCredential(PluginTask task) public Iterable getReportPage() { - String query = buildQuery(task); - logger.info(query); - SearchGoogleAdsRequest request = buildRequest(task, query); - GoogleAdsServiceClient googleAdsService = client.getVersion13().createGoogleAdsServiceClient(); - GoogleAdsServiceClient.SearchPagedResponse response = googleAdsService.search(request); - return response.iteratePages(); + List pages = new ArrayList(); + + List splitedPages = new ArrayList(); + String startDateTime = null; + do { + splitedPages.clear(); + + String query = buildQuery(task, startDateTime); + logger.info(query); + SearchGoogleAdsRequest request = buildRequest(task, query); + GoogleAdsServiceClient googleAdsService = client.getVersion13().createGoogleAdsServiceClient(); + GoogleAdsServiceClient.SearchPagedResponse response = googleAdsService.search(request); + response.iteratePages().iterator().forEachRemaining(splitedPages::add); + + if (task.getResourceType().equals("change_event")) { + GoogleAdsServiceClient.SearchPage lastPage = splitedPages.get(splitedPages.size() - 1); + GoogleAdsRow lastRow = StreamSupport.stream(lastPage.getValues().spliterator(), false).reduce((first, second) -> second).orElse(null); + if (lastRow == null) { + break; + } else { + startDateTime = lastRow.getChangeEvent().getChangeDateTime(); + } + } + splitedPages.iterator().forEachRemaining(pages::add); + } while (startDateTime != null); + + return pages; } public void flattenResource(String resourceName, Map fields, Map result) @@ -203,7 +226,7 @@ public SearchGoogleAdsRequest buildRequest(PluginTask task, String query) .build(); } - public String buildQuery(PluginTask task) + public String buildQuery(PluginTask task, String startDateTime) { StringBuilder sb = new StringBuilder(); @@ -213,7 +236,7 @@ public String buildQuery(PluginTask task) sb.append(" FROM "); sb.append(task.getResourceType()); - List whereClause = buildWhereClauseConditions(task); + List whereClause = buildWhereClauseConditions(task, startDateTime); if (!whereClause.isEmpty()) { sb.append(" WHERE "); sb.append(String.join(" AND ", whereClause)); @@ -228,7 +251,7 @@ public String buildQuery(PluginTask task) } @VisibleForTesting - public List buildWhereClauseConditions(PluginTask task) + public List buildWhereClauseConditions(PluginTask task, String startDateTime) { List whereConditions = new ArrayList() { @@ -237,14 +260,14 @@ public List buildWhereClauseConditions(PluginTask task) if (task.getDateRange().isPresent()) { StringBuilder dateSb = new StringBuilder(); if (task.getResourceType().equals("change_event")) { - dateSb.append("change_event.change_date_time BETWEEN '"); + dateSb.append(buildWhereClauseConditionsForChangeEvent(startDateTime)); } else { dateSb.append("segments.date BETWEEN '"); + dateSb.append(task.getDateRange().get().getStartDate()); + dateSb.append("' AND '"); + dateSb.append(task.getDateRange().get().getEndDate()); + dateSb.append("'"); } - dateSb.append(task.getDateRange().get().getStartDate()); - dateSb.append("' AND '"); - dateSb.append(task.getDateRange().get().getEndDate()); - dateSb.append("'"); whereConditions.add(dateSb.toString()); } @@ -266,4 +289,25 @@ public void connect() } this.client = builder.build(); } + + private String buildWhereClauseConditionsForChangeEvent(String startDateTime) + { + StringBuilder dateSb = new StringBuilder(); + dateSb.append("change_event.change_date_time "); + if (startDateTime == null) { + dateSb.append(" >= '"); + dateSb.append(task.getDateRange().get().getStartDate()); + } else { + dateSb.append(" > '"); + dateSb.append(startDateTime); + } + dateSb.append("' AND "); + dateSb.append("change_event.change_date_time "); + dateSb.append(" <= '"); + dateSb.append(task.getDateRange().get().getEndDate()); + dateSb.append("'"); + dateSb.append(" ORDER BY change_event.change_date_time ASC"); + + return dateSb.toString(); + } } From 3ab89e076f16bffb1cbc8b8cfbe8b12c4732caa2 Mon Sep 17 00:00:00 2001 From: mzumi Date: Mon, 11 Dec 2023 08:11:49 +0000 Subject: [PATCH 3/6] Change logic about fetching last row --- .../org/embulk/input/google_ads/GoogleAdsReporter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index 8eb0a4b..eead9e3 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.stream.StreamSupport; public class GoogleAdsReporter { @@ -72,7 +71,11 @@ public Iterable getReportPage() if (task.getResourceType().equals("change_event")) { GoogleAdsServiceClient.SearchPage lastPage = splitedPages.get(splitedPages.size() - 1); - GoogleAdsRow lastRow = StreamSupport.stream(lastPage.getValues().spliterator(), false).reduce((first, second) -> second).orElse(null); + GoogleAdsRow lastRow = null; + for(GoogleAdsRow row : lastPage.getValues()) { + lastRow = row; + } + if (lastRow == null) { break; } else { From 46a0cf43b175c4e538ca20c81c166ee00364ab80 Mon Sep 17 00:00:00 2001 From: mzumi Date: Mon, 11 Dec 2023 08:45:40 +0000 Subject: [PATCH 4/6] Remove unnecessary temporary variable --- .../org/embulk/input/google_ads/GoogleAdsReporter.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index eead9e3..b00c63a 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -57,20 +57,17 @@ public Iterable getReportPage() { List pages = new ArrayList(); - List splitedPages = new ArrayList(); String startDateTime = null; do { - splitedPages.clear(); - String query = buildQuery(task, startDateTime); logger.info(query); SearchGoogleAdsRequest request = buildRequest(task, query); GoogleAdsServiceClient googleAdsService = client.getVersion13().createGoogleAdsServiceClient(); GoogleAdsServiceClient.SearchPagedResponse response = googleAdsService.search(request); - response.iteratePages().iterator().forEachRemaining(splitedPages::add); + response.iteratePages().iterator().forEachRemaining(pages::add); if (task.getResourceType().equals("change_event")) { - GoogleAdsServiceClient.SearchPage lastPage = splitedPages.get(splitedPages.size() - 1); + GoogleAdsServiceClient.SearchPage lastPage = pages.get(pages.size() - 1); GoogleAdsRow lastRow = null; for(GoogleAdsRow row : lastPage.getValues()) { lastRow = row; @@ -82,7 +79,6 @@ public Iterable getReportPage() startDateTime = lastRow.getChangeEvent().getChangeDateTime(); } } - splitedPages.iterator().forEachRemaining(pages::add); } while (startDateTime != null); return pages; From e207c4ad08ff54ad10f2dada55353e36bba78b36 Mon Sep 17 00:00:00 2001 From: mzumi Date: Mon, 11 Dec 2023 09:29:42 +0000 Subject: [PATCH 5/6] Add condition when change_event.change_date_time is not designated --- .../java/org/embulk/input/google_ads/GoogleAdsReporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index b00c63a..615e0c8 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -79,7 +79,7 @@ public Iterable getReportPage() startDateTime = lastRow.getChangeEvent().getChangeDateTime(); } } - } while (startDateTime != null); + } while (startDateTime != null && !startDateTime.isEmpty()); return pages; } From ababe299113256067071935af478cefe8d1147e2 Mon Sep 17 00:00:00 2001 From: mzumi Date: Tue, 12 Dec 2023 08:32:15 +0000 Subject: [PATCH 6/6] Add condition to stop loop --- .../java/org/embulk/input/google_ads/GoogleAdsReporter.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index 615e0c8..bbc0e25 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -64,6 +64,11 @@ public Iterable getReportPage() SearchGoogleAdsRequest request = buildRequest(task, query); GoogleAdsServiceClient googleAdsService = client.getVersion13().createGoogleAdsServiceClient(); GoogleAdsServiceClient.SearchPagedResponse response = googleAdsService.search(request); + + if (response.getPage().getResponse().getResultsCount() == 0) { + return pages; + } + response.iteratePages().iterator().forEachRemaining(pages::add); if (task.getResourceType().equals("change_event")) {