Skip to content

Commit

Permalink
Microsoft Integration: Correccions i millores (asicudl#30)
Browse files Browse the repository at this point in the history 
* EVDOC01-155: Correcció en estat "error de convidats" quan la sincronització

* EVDOC01-155: Creació de users a microsoft per batch i control de excepcions de Microsoft
  • Loading branch information
@mollerentornos @mateullas
mollerentornos authored and mateullas committed Oct 3, 2024
1 parent 0cf0808 commit 8af4765
Show file tree
Hide file tree
Showing 4 changed files with 306 additions and 48 deletions.
10 changes: 8 additions & 2 deletions ...osoft-integration/api/src/java/org/sakaiproject/microsoft/api/MicrosoftCommonService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@
import org.sakaiproject.microsoft.api.data.MicrosoftTeam;
import org.sakaiproject.microsoft.api.data.MicrosoftUser;
import org.sakaiproject.microsoft.api.data.MicrosoftUserIdentifier;
import org.sakaiproject.microsoft.api.data.SynchronizationStatus;
import org.sakaiproject.microsoft.api.data.TeamsMeetingData;
import org.sakaiproject.microsoft.api.exceptions.MicrosoftCredentialsException;
import org.sakaiproject.microsoft.api.exceptions.MicrosoftGenericException;
import org.sakaiproject.microsoft.api.model.GroupSynchronization;
import org.sakaiproject.microsoft.api.model.SiteSynchronization;
import org.sakaiproject.site.api.Group;
import org.sakaiproject.user.api.User;
Expand All @@ -35,6 +37,7 @@
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -110,7 +113,9 @@ public static enum PermissionRoles { READ, WRITE }
boolean addOwnerToGroup(String userId, String groupId) throws MicrosoftCredentialsException;
boolean addMemberToTeam(String userId, String teamId) throws MicrosoftCredentialsException;
boolean addOwnerToTeam(String userId, String teamId) throws MicrosoftCredentialsException;


SynchronizationStatus addUsersToTeamOrGroup(String teamId, List<MicrosoftUser> members, SynchronizationStatus status, LinkedList<String> roles) throws MicrosoftCredentialsException;

boolean removeUserFromGroup(String userId, String groupId) throws MicrosoftCredentialsException;
boolean removeMemberFromTeam(String memberId, String teamId) throws MicrosoftCredentialsException;
boolean removeAllMembersFromTeam(String teamId) throws MicrosoftCredentialsException;
Expand All @@ -135,7 +140,8 @@ public static enum PermissionRoles { READ, WRITE }

boolean addMemberToChannel(String userId, String teamId, String channelId) throws MicrosoftCredentialsException;
boolean addOwnerToChannel(String userId, String teamId, String channelId) throws MicrosoftCredentialsException;

SynchronizationStatus addUsersToChannel(SiteSynchronization ss, GroupSynchronization gs, List<MicrosoftUser> members, SynchronizationStatus status, LinkedList<String> roles) throws MicrosoftCredentialsException;

boolean removeMemberFromChannel(String memberId, String teamId, String channelId) throws MicrosoftCredentialsException;

// ---------------------------------------- ONLINE MEETINGS --------------------------------------------------
Expand Down
3 changes: 3 additions & 0 deletions microsoft-integration/api/src/java/org/sakaiproject/microsoft/api/model/MicrosoftLog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ public class MicrosoftLog {

public static final String EVENT_AUTOCONFIG = "event.autoconfig";

public static final String EVENT_TOO_MANY_REQUESTS = "event.too_many_requests";
public static final String EVENT_USER_NOT_FOUND_ON_TEAM = "event.user_not_found_on_team";



public enum Status {
Expand Down
259 changes: 250 additions & 9 deletions ...ration/impl/src/main/java/org/sakaiproject/microsoft/impl/MicrosoftCommonServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
Expand All @@ -45,6 +46,7 @@
import com.microsoft.graph.http.GraphServiceException;
import com.microsoft.graph.http.HttpMethod;
import com.microsoft.graph.options.QueryOption;
import com.microsoft.graph.requests.ConversationMemberCollectionRequest;
import com.microsoft.graph.requests.GroupRequest;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.logging.log4j.LogManager;
Expand All @@ -66,12 +68,15 @@
import org.sakaiproject.microsoft.api.data.MicrosoftTeam;
import org.sakaiproject.microsoft.api.data.MicrosoftUser;
import org.sakaiproject.microsoft.api.data.MicrosoftUserIdentifier;
import org.sakaiproject.microsoft.api.data.SynchronizationStatus;
import org.sakaiproject.microsoft.api.data.TeamsMeetingData;
import org.sakaiproject.microsoft.api.exceptions.MicrosoftCredentialsException;
import org.sakaiproject.microsoft.api.exceptions.MicrosoftGenericException;
import org.sakaiproject.microsoft.api.exceptions.MicrosoftInvalidCredentialsException;
import org.sakaiproject.microsoft.api.exceptions.MicrosoftInvalidInvitationException;
import org.sakaiproject.microsoft.api.exceptions.MicrosoftNoCredentialsException;
import org.sakaiproject.microsoft.api.model.GroupSynchronization;
import org.sakaiproject.microsoft.api.model.MicrosoftLog;
import org.sakaiproject.microsoft.api.model.SiteSynchronization;
import org.sakaiproject.microsoft.api.persistence.MicrosoftConfigRepository;
import org.sakaiproject.microsoft.api.persistence.MicrosoftLoggingRepository;
Expand Down Expand Up @@ -287,11 +292,15 @@ public List<MicrosoftUser> getUsers() throws MicrosoftCredentialsException {
.email(u.mail)
.guest(MicrosoftUser.GUEST.equalsIgnoreCase(u.userType))
.build()).collect(Collectors.toList()));
userList.forEach(u -> log.debug(u.toString()));
UserCollectionRequestBuilder builder = page.getNextPage();
if (builder == null) break;
page = builder.buildRequest().get();
}

HashMap<String, MicrosoftUser> userMap = (HashMap<String, MicrosoftUser>) userList.stream().collect(Collectors.toMap(MicrosoftUser::getId, u -> u));
getCache().put(CACHE_USERS, userMap);

return userList;
}

Expand Down Expand Up @@ -878,12 +887,6 @@ public boolean deleteTeam(String teamId) throws MicrosoftCredentialsException {

@Override
public MicrosoftMembersCollection getTeamMembers(String id, MicrosoftUserIdentifier key) throws MicrosoftCredentialsException {
Cache.ValueWrapper cachedValue = getCache().get(CACHE_MEMBERS);
if (cachedValue != null) {
MicrosoftMembersCollection membersMap = (MicrosoftMembersCollection) cachedValue.get();
return membersMap;
}

MicrosoftMembersCollection ret = new MicrosoftMembersCollection();
try {
MicrosoftCredentials credentials = microsoftConfigRepository.getCredentials();
Expand All @@ -893,7 +896,6 @@ public MicrosoftMembersCollection getTeamMembers(String id, MicrosoftUserIdentif
.members()
.buildRequest()
.get();

while (page != null) {
for (ConversationMember m : page.getCurrentPage()) {
AadUserConversationMember member = (AadUserConversationMember) m;
Expand Down Expand Up @@ -925,8 +927,6 @@ public MicrosoftMembersCollection getTeamMembers(String id, MicrosoftUserIdentif
if (builder == null) break;
page = builder.buildRequest().get();
}

getCache().put(CACHE_MEMBERS, ret);
} catch (MicrosoftCredentialsException e) {
throw e;
} catch (Exception ex) {
Expand Down Expand Up @@ -1063,6 +1063,77 @@ public boolean addOwnerToTeam(String userId, String teamId) throws MicrosoftCred
return true;
}

@Override
public SynchronizationStatus addUsersToTeamOrGroup(String teamId, List<MicrosoftUser> members, SynchronizationStatus status, LinkedList<String> roles) throws MicrosoftCredentialsException {
boolean res = false;
String dataKey = roles.contains(MicrosoftUser.OWNER) ? "ownerId" : "memberId";

ConversationMemberCollectionRequest postMembers = graphClient.teams(teamId).members()
.buildRequest();

final int MAX_RETRY = 2;
final int MAX_PER_REQUEST = 20;
final int MAX_REQUESTS = members.size() / MAX_PER_REQUEST;

for (int i = 0; i <= MAX_REQUESTS; i++) {
List<MicrosoftUser> pendingMembers = members.subList(i * MAX_PER_REQUEST, Math.min(MAX_PER_REQUEST * (i +1 ), members.size()));
List<MicrosoftUser> successMembers = new LinkedList<>();

int retryCount = 0;
while (!pendingMembers.isEmpty() && retryCount < MAX_RETRY) {
BatchRequestContent batchRequestContent = new BatchRequestContent();

members.forEach(member -> {
ConversationMember memberToAdd = new ConversationMember();

memberToAdd.oDataType = "#microsoft.graph.aadUserConversationMember";
memberToAdd.roles = roles;
memberToAdd.additionalDataManager().put("[email protected]", new JsonPrimitive("https://graph.microsoft.com/v1.0/users('" + member.getId() + "')"));

batchRequestContent.addBatchRequestStep(postMembers, HttpMethod.POST, memberToAdd);
});

BatchResponseContent responseContent = getGraphClient().batch().buildRequest().post(batchRequestContent);
HashMap<String, ?> membersResponse = parseBatchResponse(responseContent, members);

successMembers.addAll((List<MicrosoftUser>) membersResponse.get("success"));
pendingMembers = (List<MicrosoftUser>) membersResponse.get("failed");
List<Map<String, ?>> errors = (List<Map<String, ?>>) membersResponse.get("errors");
handleMicrosoftExceptions(errors);
retryCount++;
}

for (MicrosoftUser pendingMember : pendingMembers) {
if (!res && status != SynchronizationStatus.ERROR) {
//once ERROR status is set, do not check it again
status = (pendingMember != null && pendingMember.isGuest()) ? SynchronizationStatus.ERROR_GUEST : SynchronizationStatus.ERROR;
}

// save log add member
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_ADD_MEMBER)
.status((pendingMember != null && pendingMember.isGuest()) ? MicrosoftLog.Status.OK : MicrosoftLog.Status.KO)
.addData("teamId", teamId)
.addData(dataKey, pendingMember != null ? pendingMember.getId() : "null")
.build());

}

successMembers.forEach(member -> {
// save log add member
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_ADD_MEMBER)
.status(MicrosoftLog.Status.OK)
.addData("teamId", teamId)
.addData(dataKey, member.getId())
.build());
});

}

return status;
}

@Override
public boolean removeUserFromGroup(String userId, String groupId) throws MicrosoftCredentialsException {
try {
Expand Down Expand Up @@ -1375,6 +1446,9 @@ public List<MicrosoftChannel> createChannels(List<org.sakaiproject.site.api.Grou
}

switch(listToProcess.get(0).getClass().getSimpleName()) {
case "MicrosoftUser":
resultMap = parseBatchResponseToMicrosoftUser(responseContent,(List<MicrosoftUser>) listToProcess);
break;
case "BaseGroup":
resultMap = parseBatchResponseToMicrosoftChannel(responseContent, listToProcess);
break;
Expand All @@ -1386,6 +1460,53 @@ public List<MicrosoftChannel> createChannels(List<org.sakaiproject.site.api.Grou
return resultMap;
}

private HashMap<String,?> parseBatchResponseToMicrosoftUser(BatchResponseContent responseContent, List<MicrosoftUser> listToProcess) {
HashMap<String, Object> responseMap = new HashMap<>();

Map<String, MicrosoftUser> successRequests =
responseContent.responses.stream().filter(r -> r.status <= 299).collect(Collectors.toList())
.stream().map(r -> {
Map.Entry<String, MicrosoftUser> entry = new AbstractMap.SimpleEntry<>(
r.body.getAsJsonObject().get("userId").getAsString(),
listToProcess.stream().filter(user -> user.getId().equals(r.body.getAsJsonObject().get("userId").getAsString())).findFirst().orElse(null)
);
return entry;
}).collect(Collectors.toMap(
Map.Entry::getKey,
Map.Entry::getValue
));

List<MicrosoftUser> pendingRequests = listToProcess.stream()
.filter(user -> !successRequests.containsKey(user.getId()))
.collect(Collectors.toList());

List<Map<String, ?>> errors = responseContent.responses.stream()
.filter(r -> r.status > 299)
.map(r -> {
String code, innerError;
try {
code = r.body.getAsJsonObject().get("error").getAsJsonObject().get("code").getAsString();
innerError = r.body.getAsJsonObject().get("error").getAsJsonObject().get("innerError").getAsJsonObject().get("code").getAsString();
} catch (Exception e) {
code = "Failure";
innerError = "Failure";
}
return Map.of(
"status", r.status,
"retryAfter", r.headers.containsKey("Retry-After") ? r.headers.get("Retry-After") : 5,
"code", code,
"innerError", innerError);
})
.collect(Collectors.toList());


responseMap.put("success", new ArrayList<>(successRequests.values()));
responseMap.put("failed", pendingRequests);
responseMap.put("errors", errors);

return responseMap;
}

private HashMap<String, ?> parseBatchResponseToMicrosoftTeam(BatchResponseContent responseContent, List<?> listToProcess) {
HashMap<String, Object> responseMap = new HashMap<>();

Expand Down Expand Up @@ -1642,6 +1763,126 @@ public boolean addOwnerToChannel(String userId, String teamId, String channelId)
return true;
}

public SynchronizationStatus addUsersToChannel(SiteSynchronization ss, GroupSynchronization gs, List<MicrosoftUser> members, SynchronizationStatus status, LinkedList<String> roles) throws MicrosoftCredentialsException {
String teamId = ss.getTeamId();
String channelId = gs.getChannelId();
boolean res = false;

ConversationMemberCollectionRequest postMembers = graphClient.teams(teamId).channels(channelId).members()
.buildRequest();

final int MAX_RETRY = 2;
final int MAX_PER_REQUEST = 20;
final int MAX_REQUESTS = members.size() / MAX_PER_REQUEST;

for (int i = 0; i <= MAX_REQUESTS; i++) {
List<MicrosoftUser> pendingMembers = members.subList(i * MAX_PER_REQUEST, Math.min(MAX_PER_REQUEST * (i +1 ), members.size()));
List<MicrosoftUser> successMembers = new LinkedList<>();

int retryCount = 0;
while (!pendingMembers.isEmpty() && retryCount < MAX_RETRY) {
BatchRequestContent batchRequestContent = new BatchRequestContent();

members.forEach(member -> {
ConversationMember memberToAdd = new ConversationMember();

memberToAdd.oDataType = "#microsoft.graph.aadUserConversationMember";
memberToAdd.roles = roles;
memberToAdd.additionalDataManager().put("[email protected]", new JsonPrimitive("https://graph.microsoft.com/v1.0/users('" + member.getId() + "')"));

batchRequestContent.addBatchRequestStep(postMembers, HttpMethod.POST, memberToAdd);
});

BatchResponseContent responseContent = getGraphClient().batch().buildRequest().post(batchRequestContent);

HashMap<String, ?> membersResponse = parseBatchResponse(responseContent, members);

successMembers.addAll((List<MicrosoftUser>) membersResponse.get("success"));
pendingMembers = (List<MicrosoftUser>) membersResponse.get("failed");
List<Map<String, ?>> errors = (List<Map<String, ?>>) membersResponse.get("errors");
handleMicrosoftExceptions(errors);
retryCount++;
}

for (MicrosoftUser pendingMember : pendingMembers) {
if (status != SynchronizationStatus.ERROR) {
//once ERROR status is set, do not check it again
status = (pendingMember != null && pendingMember.isGuest()) ? SynchronizationStatus.ERROR_GUEST : SynchronizationStatus.ERROR;
}

//save log
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_USER_ADDED_TO_CHANNEL)
.status(MicrosoftLog.Status.KO)
.addData("email", pendingMember.getEmail())
.addData("microsoftUserId", pendingMember.getId())
.addData("siteId", ss.getSiteId())
.addData("teamId", ss.getTeamId())
.addData("groupId", gs.getGroupId())
.addData("channelId", gs.getChannelId())
.addData("owner", Boolean.toString(roles.contains(MicrosoftUser.OWNER) && !pendingMember.isGuest()))
.addData("guest", Boolean.toString(pendingMember.isGuest()))
.build());

}

successMembers.forEach(member -> {
//save log
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_USER_ADDED_TO_CHANNEL)
.status(MicrosoftLog.Status.OK)
.addData("email", member.getEmail())
.addData("microsoftUserId", member.getId())
.addData("siteId", ss.getSiteId())
.addData("teamId", ss.getTeamId())
.addData("groupId", gs.getGroupId())
.addData("channelId", gs.getChannelId())
.addData("owner", Boolean.toString(roles.contains(MicrosoftUser.OWNER) && !member.isGuest()))
.addData("guest", Boolean.toString(member.isGuest()))
.build());
});

}

return status;
}

private void handleMicrosoftExceptions(List<Map<String,?>> errors) {
if(!errors.isEmpty()) {

if(errors.stream().anyMatch(e -> e.containsValue(429))) {
Map<String, ?> error = errors.stream().filter(e -> e.containsValue(429)).findFirst().get();
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_TOO_MANY_REQUESTS)
.addData("Status", error.get("status").toString())
.addData("Code", error.get("code").toString())
.addData("RetryAfter", error.get("retryAfter").toString())
.addData("InnerError", error.get("innerError").toString())
.build());
int retryAfter = Integer.parseInt(error.get("retryAfter").toString());

try {
Thread.sleep(retryAfter * 1000L);
} catch (InterruptedException ignored) {
}
} else if (errors.stream().anyMatch(e -> e.containsValue(404))) {
Map<String, ?> error = errors.stream().filter(e -> e.containsValue(404)).findFirst().get();
microsoftLoggingRepository.save(MicrosoftLog.builder()
.event(MicrosoftLog.EVENT_USER_NOT_FOUND_ON_TEAM)
.addData("Status", error.get("status").toString())
.addData("Code", error.get("code").toString())
.addData("RetryAfter", error.get("retryAfter").toString())
.addData("InnerError", error.get("innerError").toString())
.build());
int retryAfter = Integer.parseInt(error.get("retryAfter").toString());
try {
Thread.sleep(retryAfter * 1000L);
} catch (InterruptedException ignored) {
}
}
}
}

@Override
public boolean removeMemberFromChannel(String memberId, String teamId, String channelId) throws MicrosoftCredentialsException {
try {
Expand Down
Loading

0 comments on commit 8af4765

Please sign in to comment.