From 362c78224ea3c4288f046ce0efd40d2711091005 Mon Sep 17 00:00:00 2001 From: Tetsuro Sano Date: Mon, 10 Jun 2024 18:59:06 +0900 Subject: [PATCH 1/5] Get login customer id with customer id --- .../input/google_ads/GoogleAdsReporter.java | 69 +++++++++++++++++-- .../embulk/input/google_ads/PluginTask.java | 2 +- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index 93dc899..ea4256e 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -7,9 +7,14 @@ import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.ads.googleads.lib.GoogleAdsClient; +import com.google.ads.googleads.v16.resources.CustomerName; +import com.google.ads.googleads.v16.services.CustomerServiceClient; import com.google.ads.googleads.v16.services.GoogleAdsRow; import com.google.ads.googleads.v16.services.GoogleAdsServiceClient; +import com.google.ads.googleads.v16.services.ListAccessibleCustomersRequest; import com.google.ads.googleads.v16.services.SearchGoogleAdsRequest; +import com.google.ads.googleads.v16.services.SearchGoogleAdsStreamRequest; +import com.google.ads.googleads.v16.services.SearchGoogleAdsStreamResponse; import com.google.auth.oauth2.UserCredentials; import com.google.common.base.CaseFormat; import com.google.protobuf.Descriptors; @@ -23,6 +28,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -37,10 +43,8 @@ public class GoogleAdsReporter private final Logger logger = LoggerFactory.getLogger(GoogleAdsReporter.class); private final PluginTask task; private final UserCredentials credentials; + private final ObjectMapper mapper = new ObjectMapper(); private GoogleAdsClient client; - private ObjectMapper mapper = new ObjectMapper(); - - private Iterable searchResult; public GoogleAdsReporter(PluginTask task) { @@ -61,7 +65,7 @@ private Iterable search(Map p String query = buildQuery(task, params); logger.info(query); SearchGoogleAdsRequest request = buildRequest(task, query); - GoogleAdsServiceClient googleAdsService = client.getVersion16().createGoogleAdsServiceClient(); + GoogleAdsServiceClient googleAdsService = client.getLatestVersion().createGoogleAdsServiceClient(); GoogleAdsServiceClient.SearchPagedResponse response = googleAdsService.search(request); return response.iteratePages(); } @@ -282,14 +286,65 @@ public List buildWhereClauseConditions(PluginTask task, Map getLoginCustomerId(task.getCustomerId()))); + } + + private Long getLoginCustomerId(String customerId) + { + try (CustomerServiceClient client = buildClient(null).getLatestVersion().createCustomerServiceClient()) { + return client.listAccessibleCustomers(ListAccessibleCustomersRequest.newBuilder().build()) + .getResourceNamesList() + .stream() + .map(CustomerName::parse) + .map(CustomerName::getCustomerId) + .map(this::getLoginCustomerClients) + .flatMap(Collection::stream) + .filter(loginCustomerClient -> customerId.equals(loginCustomerClient.customerClientId)) + .findFirst() + .orElseThrow(() -> new RuntimeException("login customer not found [customer id: " + customerId + "]")) + .loginCustomerId; + } + } + + private List getLoginCustomerClients(String customerId) + { + try (GoogleAdsServiceClient client = buildClient(null).getLatestVersion().createGoogleAdsServiceClient()) { + return client.searchStreamCallable().call(SearchGoogleAdsStreamRequest.newBuilder() + .setCustomerId(customerId) + .setQuery("SELECT customer_client.id, customer_client.manager FROM customer_client") + .build()) + .stream() + .map(SearchGoogleAdsStreamResponse::getResultsList) + .flatMap(Collection::stream) + .map(GoogleAdsRow::getCustomerClient) + .filter(customerClient -> !customerClient.getManager()) + .map(customerClient -> new LoginCustomerClient(customerId, customerClient.getId())) + .collect(Collectors.toList()); + } + } + + private static class LoginCustomerClient + { + public LoginCustomerClient(String loginCustomerId, Long customerClientId) + { + this.loginCustomerId = Long.valueOf(loginCustomerId); + this.customerClientId = String.valueOf(customerClientId); + } + + final Long loginCustomerId; + final String customerClientId; + } + + private GoogleAdsClient buildClient(Long loginCustomerId) { GoogleAdsClient.Builder builder = GoogleAdsClient.newBuilder() .setDeveloperToken(task.getDeveloperToken()) .setCredentials(credentials); - if (task.getLoginCustomerId().isPresent()) { - builder.setLoginCustomerId(Long.parseLong(task.getLoginCustomerId().get())); + if (loginCustomerId != null) { + builder.setLoginCustomerId(loginCustomerId); } - this.client = builder.build(); + return builder.build(); } private String buildWhereClauseConditionsForChangeEvent(String startDateTime) diff --git a/src/main/java/org/embulk/input/google_ads/PluginTask.java b/src/main/java/org/embulk/input/google_ads/PluginTask.java index ced63a8..bad94d8 100644 --- a/src/main/java/org/embulk/input/google_ads/PluginTask.java +++ b/src/main/java/org/embulk/input/google_ads/PluginTask.java @@ -15,7 +15,7 @@ public interface PluginTask extends Task @Config("login_customer_id") @ConfigDefault("null") - Optional getLoginCustomerId(); + Optional getLoginCustomerId(); @Config("client_id") String getClientId(); From 00f10c566441ad61732ea50a931ce8547ead37b9 Mon Sep 17 00:00:00 2001 From: Tetsuro Sano Date: Wed, 12 Jun 2024 15:48:25 +0900 Subject: [PATCH 2/5] Set login customer id when searching for customer clients --- .../java/org/embulk/input/google_ads/GoogleAdsReporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index ea4256e..8dce8e3 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -309,7 +309,7 @@ private Long getLoginCustomerId(String customerId) private List getLoginCustomerClients(String customerId) { - try (GoogleAdsServiceClient client = buildClient(null).getLatestVersion().createGoogleAdsServiceClient()) { + try (GoogleAdsServiceClient client = buildClient(Long.valueOf(customerId)).getLatestVersion().createGoogleAdsServiceClient()) { return client.searchStreamCallable().call(SearchGoogleAdsStreamRequest.newBuilder() .setCustomerId(customerId) .setQuery("SELECT customer_client.id, customer_client.manager FROM customer_client") From 71e9e1e57f51b13a3c7f03fd43d171b6179bbd36 Mon Sep 17 00:00:00 2001 From: Tetsuro Sano Date: Wed, 12 Jun 2024 21:04:09 +0900 Subject: [PATCH 3/5] Find login customer as close to the top-level manager account as possible --- .../input/google_ads/GoogleAdsReporter.java | 43 ++++++++++++++++--- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index 8dce8e3..df5f89a 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -29,10 +29,12 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.HashMap; +import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -300,10 +302,15 @@ private Long getLoginCustomerId(String customerId) .map(CustomerName::getCustomerId) .map(this::getLoginCustomerClients) .flatMap(Collection::stream) - .filter(loginCustomerClient -> customerId.equals(loginCustomerClient.customerClientId)) + .collect(Collectors.groupingBy(LoginCustomerClient::getCustomerClientId, Collectors.maxBy(Comparator.comparing(LoginCustomerClient::getLevel)))) + .values() + .stream() + .filter(Optional::isPresent) + .map(Optional::get) + .filter(loginCustomerClient -> loginCustomerClient.isMe(customerId)) .findFirst() .orElseThrow(() -> new RuntimeException("login customer not found [customer id: " + customerId + "]")) - .loginCustomerId; + .getLoginCustomerId(); } } @@ -312,28 +319,50 @@ private List getLoginCustomerClients(String customerId) try (GoogleAdsServiceClient client = buildClient(Long.valueOf(customerId)).getLatestVersion().createGoogleAdsServiceClient()) { return client.searchStreamCallable().call(SearchGoogleAdsStreamRequest.newBuilder() .setCustomerId(customerId) - .setQuery("SELECT customer_client.id, customer_client.manager FROM customer_client") + .setQuery("SELECT customer_client.id, customer_client.manager, customer_client.level FROM customer_client") .build()) .stream() .map(SearchGoogleAdsStreamResponse::getResultsList) .flatMap(Collection::stream) .map(GoogleAdsRow::getCustomerClient) .filter(customerClient -> !customerClient.getManager()) - .map(customerClient -> new LoginCustomerClient(customerId, customerClient.getId())) + .map(customerClient -> new LoginCustomerClient(customerId, customerClient.getId(), customerClient.getLevel())) .collect(Collectors.toList()); } } private static class LoginCustomerClient { - public LoginCustomerClient(String loginCustomerId, Long customerClientId) + LoginCustomerClient(String loginCustomerId, Long customerClientId, long level) { - this.loginCustomerId = Long.valueOf(loginCustomerId); + this.loginCustomerId = loginCustomerId; this.customerClientId = String.valueOf(customerClientId); + this.level = level; } - final Long loginCustomerId; + final String loginCustomerId; final String customerClientId; + final long level; + + Long getLoginCustomerId() + { + return Long.valueOf(loginCustomerId); + } + + String getCustomerClientId() + { + return customerClientId; + } + + long getLevel() + { + return level; + } + + boolean isMe(String customerId) + { + return customerId.equals(customerClientId) || customerId.equals(loginCustomerId); + } } private GoogleAdsClient buildClient(Long loginCustomerId) From ae5a3c75eb1193790876ac00fe97ddd6e5f0e46e Mon Sep 17 00:00:00 2001 From: Tetsuro Sano Date: Wed, 10 Jul 2024 11:04:57 +0900 Subject: [PATCH 4/5] Log a warning when ambiguous login customers are found --- .../input/google_ads/GoogleAdsReporter.java | 55 ++++++------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index df5f89a..217c6f2 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -29,12 +29,10 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.HashMap; -import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -293,6 +291,18 @@ public void connect() } private Long getLoginCustomerId(String customerId) + { + List loginCustomerIds = getLoginCustomerIds(customerId); + if (loginCustomerIds.isEmpty()) { + throw new RuntimeException("login customer not found [customer id: " + customerId + "]"); + } + if (loginCustomerIds.size() > 1) { + logger.warn("ambiguous login customers found [customer id: {}, login customer ids: {}]", customerId, loginCustomerIds.stream().map(Object::toString).collect(Collectors.joining(", "))); + } + return loginCustomerIds.get(0); + } + + private List getLoginCustomerIds(String customerId) { try (CustomerServiceClient client = buildClient(null).getLatestVersion().createCustomerServiceClient()) { return client.listAccessibleCustomers(ListAccessibleCustomersRequest.newBuilder().build()) @@ -302,15 +312,9 @@ private Long getLoginCustomerId(String customerId) .map(CustomerName::getCustomerId) .map(this::getLoginCustomerClients) .flatMap(Collection::stream) - .collect(Collectors.groupingBy(LoginCustomerClient::getCustomerClientId, Collectors.maxBy(Comparator.comparing(LoginCustomerClient::getLevel)))) - .values() - .stream() - .filter(Optional::isPresent) - .map(Optional::get) - .filter(loginCustomerClient -> loginCustomerClient.isMe(customerId)) - .findFirst() - .orElseThrow(() -> new RuntimeException("login customer not found [customer id: " + customerId + "]")) - .getLoginCustomerId(); + .filter(loginCustomerClient -> loginCustomerClient.customerClientId.equals(customerId)) + .map(loginCustomerClient -> Long.valueOf(loginCustomerClient.loginCustomerId)) + .collect(Collectors.toList()); } } @@ -319,50 +323,27 @@ private List getLoginCustomerClients(String customerId) try (GoogleAdsServiceClient client = buildClient(Long.valueOf(customerId)).getLatestVersion().createGoogleAdsServiceClient()) { return client.searchStreamCallable().call(SearchGoogleAdsStreamRequest.newBuilder() .setCustomerId(customerId) - .setQuery("SELECT customer_client.id, customer_client.manager, customer_client.level FROM customer_client") + .setQuery("SELECT customer_client.id FROM customer_client") .build()) .stream() .map(SearchGoogleAdsStreamResponse::getResultsList) .flatMap(Collection::stream) .map(GoogleAdsRow::getCustomerClient) - .filter(customerClient -> !customerClient.getManager()) - .map(customerClient -> new LoginCustomerClient(customerId, customerClient.getId(), customerClient.getLevel())) + .map(customerClient -> new LoginCustomerClient(customerId, customerClient.getId())) .collect(Collectors.toList()); } } private static class LoginCustomerClient { - LoginCustomerClient(String loginCustomerId, Long customerClientId, long level) + LoginCustomerClient(String loginCustomerId, Long customerClientId) { this.loginCustomerId = loginCustomerId; this.customerClientId = String.valueOf(customerClientId); - this.level = level; } final String loginCustomerId; final String customerClientId; - final long level; - - Long getLoginCustomerId() - { - return Long.valueOf(loginCustomerId); - } - - String getCustomerClientId() - { - return customerClientId; - } - - long getLevel() - { - return level; - } - - boolean isMe(String customerId) - { - return customerId.equals(customerClientId) || customerId.equals(loginCustomerId); - } } private GoogleAdsClient buildClient(Long loginCustomerId) From f876e473cab0db4f8ea7c8e1cd966c16aa56a85f Mon Sep 17 00:00:00 2001 From: Tetsuro Sano Date: Fri, 9 Aug 2024 21:54:38 +0900 Subject: [PATCH 5/5] Change the log message --- .../java/org/embulk/input/google_ads/GoogleAdsReporter.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index 217c6f2..9534c32 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -297,9 +297,11 @@ private Long getLoginCustomerId(String customerId) throw new RuntimeException("login customer not found [customer id: " + customerId + "]"); } if (loginCustomerIds.size() > 1) { - logger.warn("ambiguous login customers found [customer id: {}, login customer ids: {}]", customerId, loginCustomerIds.stream().map(Object::toString).collect(Collectors.joining(", "))); + logger.info("multiple login customers found [login customer ids: {}]", loginCustomerIds.stream().map(Object::toString).collect(Collectors.joining(", "))); } - return loginCustomerIds.get(0); + Long loginCustomerId = loginCustomerIds.get(0); + logger.info("use this customer [customer id: {}, login customer id: {}] to login", customerId, loginCustomerId); + return loginCustomerId; } private List getLoginCustomerIds(String customerId)