Skip to content

Commit

Permalink
EVDOC01-155: Millores de rendiment a la sincronització i clarificació…
Browse files Browse the repository at this point in the history 
… als logs
  • Loading branch information
@mollerentornos
mollerentornos committed Oct 15, 2024
1 parent efa734a commit 72039ee
Show file tree
Hide file tree
Showing 9 changed files with 326 additions and 65 deletions.
6 changes: 6 additions & 0 deletions .../admin-tool/src/main/java/org/sakaiproject/microsoft/controller/AutoConfigController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.sakaiproject.microsoft.api.data.CreationStatus;
import org.sakaiproject.microsoft.api.data.MicrosoftChannel;
import org.sakaiproject.microsoft.api.data.MicrosoftCredentials;
import org.sakaiproject.microsoft.api.data.MicrosoftLogInvokers;
import org.sakaiproject.microsoft.api.data.MicrosoftTeam;
import org.sakaiproject.microsoft.api.data.SynchronizationStatus;
import org.sakaiproject.microsoft.api.model.GroupSynchronization;
Expand Down Expand Up @@ -300,6 +301,7 @@ public void handleNewTeamCreation(AutoConfigSessionBean autoConfigSessionBean, S
microsoftLoggingService.saveLog(MicrosoftLog.builder()
.event(MicrosoftLog.ERROR_TEAM_ID_NULL)
.status(MicrosoftLog.Status.KO)
.addData("origin", MicrosoftLogInvokers.MANUAL.getCode())
.addData("teamId", teamId)
.addData("siteId", site.getId())
.addData("siteTitle", site.getTitle())
Expand All @@ -311,6 +313,7 @@ public void handleNewTeamCreation(AutoConfigSessionBean autoConfigSessionBean, S
microsoftLoggingService.saveLog(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_CREATE_TEAM_FROM_SITE)
.status(MicrosoftLog.Status.OK)
.addData("origin", MicrosoftLogInvokers.MANUAL.getCode())
.addData("siteId", site.getId())
.addData("siteTitle", site.getTitle())
.addData("teamId", teamId)
Expand Down Expand Up @@ -360,6 +363,7 @@ public void handleNewTeamCreation(AutoConfigSessionBean autoConfigSessionBean, S
microsoftLoggingService.saveLog(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_CHANNEL_PRESENT_ON_GROUP)
.status(MicrosoftLog.Status.OK)
.addData("origin", MicrosoftLogInvokers.MANUAL.getCode())
.addData("siteId", site.getId())
.addData("siteTitle", site.getTitle())
.addData("processGroupsIds", groupsIds)
Expand All @@ -380,6 +384,7 @@ public void handleExistingTeamBinding(AutoConfigSessionBean autoConfigSessionBea
microsoftLoggingService.saveLog(MicrosoftLog.builder()
.event(MicrosoftLog.BINDING_TEAM_FROM_SITE)
.status(!site.getId().isBlank() ? MicrosoftLog.Status.OK : MicrosoftLog.Status.KO)
.addData("origin", MicrosoftLogInvokers.MANUAL.getCode())
.addData("siteId", site.getId())
.addData("siteTitle", site.getTitle())
.addData("teamId", ss.getTeamId())
Expand Down Expand Up @@ -447,6 +452,7 @@ public void handleExistingTeamBinding(AutoConfigSessionBean autoConfigSessionBea
microsoftLoggingService.saveLog(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_CHANNEL_PRESENT_ON_GROUP)
.status(MicrosoftLog.Status.OK)
.addData("origin", MicrosoftLogInvokers.MANUAL.getCode())
.addData("siteId", site.getId())
.addData("siteTitle", site.getTitle())
.addData("processGroupsIds", groupsIds)
Expand Down
4 changes: 4 additions & 0 deletions ...ration/admin-tool/src/main/java/org/sakaiproject/microsoft/controller/MainController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.sakaiproject.microsoft.api.MicrosoftConfigurationService;
import org.sakaiproject.microsoft.api.MicrosoftSynchronizationService;
import org.sakaiproject.microsoft.api.SakaiProxy;
import org.sakaiproject.microsoft.api.data.MicrosoftLogInvokers;
import org.sakaiproject.microsoft.api.data.MicrosoftTeam;
import org.sakaiproject.microsoft.api.data.SakaiSiteFilter;
import org.sakaiproject.microsoft.api.data.SynchronizationStatus;
Expand All @@ -31,6 +32,7 @@
import org.sakaiproject.microsoft.controller.auxiliar.FilterRequest;
import org.sakaiproject.microsoft.controller.auxiliar.MainSessionBean;
import org.sakaiproject.site.api.Group;
import org.sakaiproject.tool.api.Session;
import org.sakaiproject.util.ResourceLoader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.format.annotation.DateTimeFormat;
Expand Down Expand Up @@ -91,6 +93,8 @@ public class MainController {

@GetMapping(value = {"/", "/index"})
public String index(Model model) {
Session session = sakaiProxy.getCurrentSession();
session.setAttribute("origin", MicrosoftLogInvokers.MANUAL.getCode());

return INDEX_TEMPLATE;
}
Expand Down
7 changes: 7 additions & 0 deletions ...osoft-integration/api/src/java/org/sakaiproject/microsoft/api/MicrosoftCommonService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ public static enum PermissionRoles { READ, WRITE }

// ---------------------------------------- USERS ------------------------------------------------
List<MicrosoftUser> getUsers() throws MicrosoftCredentialsException;
List<MicrosoftUser> getUsers(Set<String> strings, MicrosoftUserIdentifier mappedMicrosoftUserId) throws MicrosoftCredentialsException;

Map<String, Set<User>> getErrorUsers();
void addErrorUsers(String id, User user);
Map<String, Set<User>> getErrorGroupsUsers();
Expand All @@ -77,6 +79,11 @@ public static enum PermissionRoles { READ, WRITE }
MicrosoftUser getUserById(String id) throws MicrosoftCredentialsException;
MicrosoftUser getUserByEmail(String email) throws MicrosoftCredentialsException;

List<MicrosoftUser> getUsersById(Set<String> userIds) throws MicrosoftCredentialsException;

List<MicrosoftUser> getUsersByEmail(Set<String> userEmails) throws MicrosoftCredentialsException;


void clearErrorGroupsUsers(String id);

boolean checkUser(String identifier, MicrosoftUserIdentifier key) throws MicrosoftCredentialsException;
Expand Down
2 changes: 2 additions & 0 deletions microsoft-integration/api/src/java/org/sakaiproject/microsoft/api/SakaiProxy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ public interface SakaiProxy {
// --------------------------------------------- SESSION -----------------------------------------------------
Session getCurrentSession();

String getActionOrigin();

// ------------------------------------------ USERS ----------------------------------------------------
String getCurrentUserId();
User getCurrentUser();
Expand Down
19 changes: 19 additions & 0 deletions ...ft-integration/api/src/java/org/sakaiproject/microsoft/api/data/MicrosoftLogInvokers.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package org.sakaiproject.microsoft.api.data;

import lombok.Getter;

@Getter
public enum MicrosoftLogInvokers {

UNKNOWN("unknown"),
HOOK("hook"),
JOB("job"),
MANUAL("manual");

private String code;

private MicrosoftLogInvokers(String code) {
this.code = code;
}

}
190 changes: 189 additions & 1 deletion ...ration/impl/src/main/java/org/sakaiproject/microsoft/impl/MicrosoftCommonServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
Expand All @@ -43,11 +42,13 @@

import com.microsoft.graph.content.BatchRequestContent;
import com.microsoft.graph.content.BatchResponseContent;
import com.microsoft.graph.content.BatchResponseStep;
import com.microsoft.graph.http.GraphServiceException;
import com.microsoft.graph.http.HttpMethod;
import com.microsoft.graph.options.QueryOption;
import com.microsoft.graph.requests.ConversationMemberCollectionRequest;
import com.microsoft.graph.requests.GroupRequest;
import com.microsoft.graph.requests.UserCollectionRequest;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.LoggerContext;
Expand Down Expand Up @@ -401,6 +402,126 @@ public MicrosoftUser getUserByEmail(String email) throws MicrosoftCredentialsExc
return null;
}

@Override
public List<MicrosoftUser> getUsersById(Set<String> userIds) throws MicrosoftCredentialsException {
List<MicrosoftUser> users = new ArrayList<>();

//get from cache
Map<String, MicrosoftUser> usersMap = new HashMap<>();
Cache.ValueWrapper cachedValue = getCache().get(CACHE_USERS);
if (cachedValue != null) {
usersMap = (Map<String, MicrosoftUser>) cachedValue.get();
for (MicrosoftUser user : usersMap.values()) {
if (userIds.contains(user.getId().toLowerCase())) {
users.add(user);
}
}
}

final int MAX_LENGTH = 20;
int pointer = 0;
LinkedList<Option> requestOptions = new LinkedList<Option>();
requestOptions.add(new HeaderOption("ConsistencyLevel", "eventual"));
requestOptions.add(new QueryOption("$select", "id,displayName,mail,userType"));

GraphServiceClient graph = getGraphClient();
Set<String> pendingUsers = userIds.stream().filter(id -> !id.startsWith("EMPTY_") && users.stream().noneMatch(u -> u.getId().equalsIgnoreCase(id))).collect(Collectors.toSet());
List<String> usersToProcess;
//sometimes microsoft fails creating -> loop for retry failed ones
while (!pendingUsers.isEmpty()) {
usersToProcess = pendingUsers.stream().skip(pointer).limit(MAX_LENGTH).collect(Collectors.toList());

BatchRequestContent batchRequestContent = new BatchRequestContent();

usersToProcess.forEach(id -> {
UserCollectionRequest getUsers = graph.users()
.buildRequest(requestOptions)
.filter("userId eq '" + id + "'");

batchRequestContent.addBatchRequestStep(getUsers, HttpMethod.GET);
});

BatchResponseContent responseContent = getGraphClient().batch().buildRequest().post(batchRequestContent);

HashMap<String, ?> teamsResponse = parseBatchResponse(responseContent, usersToProcess);

users.addAll((List<MicrosoftUser>) teamsResponse.get("success"));
pendingUsers.removeAll(usersToProcess);
}

//store in cache
usersMap.putAll(users.stream().collect(Collectors.toMap(MicrosoftUser::getId, u -> u)));
getCache().put(CACHE_USERS, usersMap);

return users;
}

@Override
public List<MicrosoftUser> getUsers(Set<String> ids, MicrosoftUserIdentifier mappedMicrosoftUserId) throws MicrosoftCredentialsException {
switch (mappedMicrosoftUserId) {
case USER_ID:
return getUsersById(ids);
case EMAIL:
return getUsersByEmail(ids);
default:
return null;
}
}

@Override
public List<MicrosoftUser> getUsersByEmail(Set<String> userEmails) throws MicrosoftCredentialsException {
List<MicrosoftUser> users = new ArrayList<>();

//get from cache
Map<String, MicrosoftUser> usersMap = new HashMap<>();
Cache.ValueWrapper cachedValue = getCache().get(CACHE_USERS);
if (cachedValue != null) {
usersMap = (Map<String, MicrosoftUser>) cachedValue.get();
for (MicrosoftUser user : usersMap.values()) {
if (userEmails.contains(user.getEmail().toLowerCase())) {
users.add(user);
}
}
}

final int MAX_LENGTH = 20;
int pointer = 0;
LinkedList<Option> requestOptions = new LinkedList<Option>();
requestOptions.add(new HeaderOption("ConsistencyLevel", "eventual"));
requestOptions.add(new QueryOption("$select", "id,displayName,mail,userType"));

GraphServiceClient graph = getGraphClient();
Set<String> pendingUsers = userEmails.stream().filter(email -> !email.startsWith("EMPTY_") && users.stream().noneMatch(u -> u.getEmail().equalsIgnoreCase(email))).collect(Collectors.toSet());
List<String> usersToProcess;
//sometimes microsoft fails creating -> loop for retry failed ones
while (!pendingUsers.isEmpty()) {
usersToProcess = pendingUsers.stream().skip(pointer).limit(MAX_LENGTH).collect(Collectors.toList());

BatchRequestContent batchRequestContent = new BatchRequestContent();

usersToProcess.forEach(email -> {
UserCollectionRequest getUsers = graph.users()
.buildRequest(requestOptions)
.filter("mail eq '" + email + "'");

batchRequestContent.addBatchRequestStep(getUsers, HttpMethod.GET);
});

BatchResponseContent responseContent = getGraphClient().batch().buildRequest().post(batchRequestContent);

HashMap<String, ?> teamsResponse = parseBatchResponse(responseContent, usersToProcess);

users.addAll((List<MicrosoftUser>) teamsResponse.get("success"));
pendingUsers.removeAll(usersToProcess);
}

//store in cache
usersMap.putAll(users.stream().collect(Collectors.toMap(MicrosoftUser::getId, u -> u)));
getCache().put(CACHE_USERS, usersMap);

return users;
}

@Override
public MicrosoftUser getUserById(String id) throws MicrosoftCredentialsException {
try {
Expand Down Expand Up @@ -1130,6 +1251,7 @@ public SynchronizationStatus addUsersToTeamOrGroup(String teamId, List<Microsoft
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_ADD_MEMBER)
.status((pendingMember != null && pendingMember.isGuest()) ? MicrosoftLog.Status.OK : MicrosoftLog.Status.KO)
.addData("origin", sakaiProxy.getActionOrigin())
.addData("teamId", teamId)
.addData(dataKey, pendingMember != null ? pendingMember.getId() : "null")
.build());
Expand All @@ -1141,6 +1263,7 @@ public SynchronizationStatus addUsersToTeamOrGroup(String teamId, List<Microsoft
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_ADD_MEMBER)
.status(MicrosoftLog.Status.OK)
.addData("origin", sakaiProxy.getActionOrigin())
.addData("teamId", teamId)
.addData(dataKey, member.getId())
.build());
Expand Down Expand Up @@ -1472,6 +1595,9 @@ public List<MicrosoftChannel> createChannels(List<org.sakaiproject.site.api.Grou
case "SiteSynchronization":
resultMap = parseBatchResponseToMicrosoftTeam(responseContent, listToProcess);
break;
case "String":
resultMap = parseBatchResponseToMicrosoftUserFromStringList(responseContent,(List<String>) listToProcess);
break;
}

return resultMap;
Expand Down Expand Up @@ -1524,6 +1650,64 @@ private HashMap<String,?> parseBatchResponseToMicrosoftUser(BatchResponseContent
return responseMap;
}


private HashMap<String,?> parseBatchResponseToMicrosoftUserFromStringList(BatchResponseContent responseContent, List<String> listToProcess) {
HashMap<String, Object> responseMap = new HashMap<>();

List<BatchResponseStep<JsonElement>> nonEmptyResponses = responseContent.responses.stream()
.filter(r -> r.status <= 299)
.collect(Collectors.toList())
.stream().filter(r -> !r.body.getAsJsonObject().get("value").getAsJsonArray().isEmpty())
.collect(Collectors.toList());

Map<String, MicrosoftUser> successRequests =
nonEmptyResponses
.stream().map(r -> {
Map.Entry<String, MicrosoftUser> entry = new AbstractMap.SimpleEntry<>(
r.body.getAsJsonObject().get("value").getAsJsonArray().get(0).getAsJsonObject().get("id").getAsString(),
MicrosoftUser.builder()
.id(r.body.getAsJsonObject().get("value").getAsJsonArray().get(0).getAsJsonObject().get("id").getAsString())
.email(r.body.getAsJsonObject().get("value").getAsJsonArray().get(0).getAsJsonObject().get("mail").getAsString())
.build()
);
return entry;
}).collect(Collectors.toMap(
Map.Entry::getKey,
Map.Entry::getValue
));

List<String> notFoundUsers =
listToProcess.stream()
.filter(email -> successRequests.entrySet().stream().noneMatch(r -> r.getValue().getEmail().equalsIgnoreCase(email)))
.collect(Collectors.toList());

List<Map<String, ?>> errors = responseContent.responses.stream()
.filter(r -> r.status > 299)
.map(r -> {
String code, innerError;
try {
code = r.body.getAsJsonObject().get("error").getAsJsonObject().get("code").getAsString();
innerError = r.body.getAsJsonObject().get("error").getAsJsonObject().get("innerError").getAsJsonObject().get("code").getAsString();
} catch (Exception e) {
code = "Failure";
innerError = "Failure";
}
return Map.of(
"status", r.status,
"retryAfter", r.headers.containsKey("Retry-After") ? r.headers.get("Retry-After") : 5,
"code", code,
"innerError", innerError);
})
.collect(Collectors.toList());


responseMap.put("success", new ArrayList<>(successRequests.values()));
responseMap.put("failed", notFoundUsers);
responseMap.put("errors", errors);

return responseMap;
}

private HashMap<String, ?> parseBatchResponseToMicrosoftTeam(BatchResponseContent responseContent, List<?> listToProcess) {
HashMap<String, Object> responseMap = new HashMap<>();

Expand Down Expand Up @@ -1843,6 +2027,7 @@ public SynchronizationStatus addUsersToChannel(SiteSynchronization ss, GroupSync
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_USER_ADDED_TO_CHANNEL)
.status(MicrosoftLog.Status.KO)
.addData("origin", sakaiProxy.getActionOrigin())
.addData("email", pendingMember.getEmail())
.addData("microsoftUserId", pendingMember.getId())
.addData("siteId", ss.getSiteId())
Expand All @@ -1860,6 +2045,7 @@ public SynchronizationStatus addUsersToChannel(SiteSynchronization ss, GroupSync
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_USER_ADDED_TO_CHANNEL)
.status(MicrosoftLog.Status.OK)
.addData("origin", sakaiProxy.getActionOrigin())
.addData("email", member.getEmail())
.addData("microsoftUserId", member.getId())
.addData("siteId", ss.getSiteId())
Expand All @@ -1882,6 +2068,7 @@ private void handleMicrosoftExceptions(List<Map<String,?>> errors) {
Map<String, ?> error = errors.stream().filter(e -> e.containsValue(429)).findFirst().get();
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_TOO_MANY_REQUESTS)
.addData("origin", sakaiProxy.getActionOrigin())
.addData("Status", error.get("status").toString())
.addData("Code", error.get("code").toString())
.addData("RetryAfter", error.get("retryAfter").toString())
Expand All @@ -1897,6 +2084,7 @@ private void handleMicrosoftExceptions(List<Map<String,?>> errors) {
Map<String, ?> error = errors.stream().filter(e -> e.containsValue(404)).findFirst().get();
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_USER_NOT_FOUND_ON_TEAM)
.addData("origin", sakaiProxy.getActionOrigin())
.addData("Status", error.get("status").toString())
.addData("Code", error.get("code").toString())
.addData("RetryAfter", error.get("retryAfter").toString())
Expand Down
Loading

0 comments on commit 72039ee

Please sign in to comment.