From fcc7e11e37c9b8138a03fcc4acae68f60bbfee98 Mon Sep 17 00:00:00 2001 From: ankitpatnaik-atlan Date: Thu, 26 Dec 2024 18:06:38 +0530 Subject: [PATCH 1/7] mesh-299: enrich audit logs --- .../atlas/model/audit/EntityAuditEventV2.java | 19 +++++++ .../model/instance/AtlasEntityHeader.java | 10 ++++ .../audit/ESBasedAuditRepository.java | 49 ++++++++++++++++++- 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java index 9a4b03df73..c7eb29fe27 100644 --- a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java +++ b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java @@ -135,6 +135,7 @@ public static EntityAuditActionV2 fromString(String strValue) { private Map detail; private AtlasEntityHeader entityDetail; private Map headers; + private List linkedEntities; public EntityAuditEventV2() { } @@ -250,6 +251,19 @@ public void setHeaders(Map headers) { this.headers = headers; } + public List getLinkedEntities() { + return linkedEntities; + } + + public void setLinkedEntities(List linkedEntities) { + this.linkedEntities = linkedEntities; + } + + @JsonIgnore + public boolean hasLinkedEntities() { + return linkedEntities != null && !linkedEntities.isEmpty(); + } + @JsonIgnore public String getEntityDefinitionString() { if (entity != null) { @@ -315,6 +329,11 @@ public String toString() { sb.append(", detail=").append(detail); sb.append(", created=").append(created); sb.append(", headers=").append(headers); + + if (hasLinkedEntities()) { + sb.append(", linkedEntities=").append(linkedEntities); + } + sb.append('}'); return sb.toString(); diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index 13820f5255..53c9cecf9a 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -74,6 +74,8 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private Integer depth = null; private Integer traversalOrder = null; private Integer finishTime = null; + private String name = null; + private String typeName = null; private Map collapse = null; @@ -280,6 +282,14 @@ public void setCollapse(Map collapse) { this.collapse = collapse; } + public String getName(){ return name; } + + public void setName(String name){ this.name = name; } + + public String getTypeName(){ return typeName; } + + public void setTypeName(){ this.typeName = typeName; } + @Override public StringBuilder toString(StringBuilder sb) { if (sb == null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index cbab135606..52119921e1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -29,6 +29,7 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.type.AtlasType; import org.apache.commons.collections.MapUtils; import org.apache.commons.configuration.Configuration; @@ -47,6 +48,9 @@ import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import javax.inject.Inject; import javax.inject.Singleton; @@ -60,6 +64,7 @@ import java.util.*; import static java.nio.charset.Charset.defaultCharset; +import static org.apache.atlas.repository.Constants.DOMAIN_GUIDS; import static org.springframework.util.StreamUtils.copyToString; /** @@ -94,12 +99,22 @@ public class ESBasedAuditRepository extends AbstractStorageBasedAuditRepository private RestClient lowLevelClient; private final Configuration configuration; + private EntityGraphRetriever entityGraphRetriever; + private AtlasGraph graph; + private AtlasTypeRegistry typeRegistry; @Inject public ESBasedAuditRepository(Configuration configuration) { this.configuration = configuration; } - + + @Inject + public ESBasedAuditRepository(Configuration configuration, AtlasGraph graph, AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityGraphRetriever) { + this.configuration = configuration; + this.graph = graph; + this.typeRegistry = typeRegistry; + this.entityGraphRetriever = entityGraphRetriever; + } @Override public void putEventsV1(List events) throws AtlasException { @@ -243,6 +258,28 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr eventKey = event.getEntityId() + ":" + event.getTimestamp(); } + Map detail = event.getDetail(); + if (detail != null && detail.containsKey("attributes")) { + Map attributes = (Map) detail.get("attributes"); + List domainGUIDs = (List) attributes.get(DOMAIN_GUIDS); + + if (domainGUIDs != null && !domainGUIDs.isEmpty()) { + List linkedEntityList = new ArrayList<>(); + for (String domainGUID: domainGUIDs) { + try { + AtlasEntityHeader domainEntityHeader = fetchAtlasEntityHeader(domainGUID); + if (domainEntityHeader != null) { + source.put("domainDetails", domainEntityHeader); + linkedEntityList.add(domainEntityHeader); + } + } catch (Exception e) { + throw new AtlasBaseException(e); + } + } + event.setLinkedEntities(linkedEntityList); + } + } + event.setHeaders((Map) source.get("headers")); event.setEventKey(eventKey); @@ -258,6 +295,16 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr return searchResult; } + private AtlasEntityHeader fetchAtlasEntityHeader(String domainGUID) throws AtlasBaseException { + try { + EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry); + + return entityRetriever.toAtlasEntityHeader(domainGUID); + } catch (Exception e) { + throw new AtlasBaseException(e); + } + } + private String performSearchOnIndex(String queryString) throws IOException { HttpEntity entity = new NStringEntity(queryString, ContentType.APPLICATION_JSON); String endPoint = INDEX_NAME + "/_search"; From f58935dcd1f948f455207a62dd92ebdfc3002ccd Mon Sep 17 00:00:00 2001 From: ankitpatnaik-atlan Date: Thu, 26 Dec 2024 20:21:35 +0530 Subject: [PATCH 2/7] mesh-299-audit: corrections --- .../apache/atlas/model/instance/AtlasEntityHeader.java | 10 ---------- .../atlas/repository/audit/ESBasedAuditRepository.java | 1 - 2 files changed, 11 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index 53c9cecf9a..13820f5255 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -74,8 +74,6 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private Integer depth = null; private Integer traversalOrder = null; private Integer finishTime = null; - private String name = null; - private String typeName = null; private Map collapse = null; @@ -282,14 +280,6 @@ public void setCollapse(Map collapse) { this.collapse = collapse; } - public String getName(){ return name; } - - public void setName(String name){ this.name = name; } - - public String getTypeName(){ return typeName; } - - public void setTypeName(){ this.typeName = typeName; } - @Override public StringBuilder toString(StringBuilder sb) { if (sb == null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index 52119921e1..e5dc76b34f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -269,7 +269,6 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr try { AtlasEntityHeader domainEntityHeader = fetchAtlasEntityHeader(domainGUID); if (domainEntityHeader != null) { - source.put("domainDetails", domainEntityHeader); linkedEntityList.add(domainEntityHeader); } } catch (Exception e) { From ad8292d5dd155a1f737a2e37c2be07f1d199a2f5 Mon Sep 17 00:00:00 2001 From: ankitpatnaik-atlan Date: Fri, 27 Dec 2024 11:34:26 +0530 Subject: [PATCH 3/7] mesh-299-audit: code-corrections in constructor --- .../audit/ESBasedAuditRepository.java | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index e5dc76b34f..d0328dab0d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -48,8 +48,6 @@ import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import javax.inject.Inject; @@ -100,19 +98,10 @@ public class ESBasedAuditRepository extends AbstractStorageBasedAuditRepository private RestClient lowLevelClient; private final Configuration configuration; private EntityGraphRetriever entityGraphRetriever; - private AtlasGraph graph; - private AtlasTypeRegistry typeRegistry; @Inject - public ESBasedAuditRepository(Configuration configuration) { + public ESBasedAuditRepository(Configuration configuration, EntityGraphRetriever entityGraphRetriever) { this.configuration = configuration; - } - - @Inject - public ESBasedAuditRepository(Configuration configuration, AtlasGraph graph, AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityGraphRetriever) { - this.configuration = configuration; - this.graph = graph; - this.typeRegistry = typeRegistry; this.entityGraphRetriever = entityGraphRetriever; } @@ -271,7 +260,7 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr if (domainEntityHeader != null) { linkedEntityList.add(domainEntityHeader); } - } catch (Exception e) { + } catch (AtlasBaseException e) { throw new AtlasBaseException(e); } } @@ -295,11 +284,10 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr } private AtlasEntityHeader fetchAtlasEntityHeader(String domainGUID) throws AtlasBaseException { - try { - EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry); - - return entityRetriever.toAtlasEntityHeader(domainGUID); - } catch (Exception e) { + try { + AtlasEntityHeader entityHeader = entityGraphRetriever.toAtlasEntityHeader(domainGUID); + return entityHeader; + } catch (AtlasBaseException e) { throw new AtlasBaseException(e); } } From 76c87273290d24f5269fabd60f3d71a8a07e8f0b Mon Sep 17 00:00:00 2001 From: ankitpatnaik-atlan Date: Mon, 30 Dec 2024 12:46:14 +0530 Subject: [PATCH 4/7] mesh-299-audit: dynamic guid check --- .../audit/ESBasedAuditRepository.java | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index d0328dab0d..97574c9d7d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -88,6 +88,7 @@ public class ESBasedAuditRepository extends AbstractStorageBasedAuditRepository private static final String DETAIL = "detail"; private static final String ENTITY = "entity"; private static final String bulkMetadata = String.format("{ \"index\" : { \"_index\" : \"%s\" } }%n", INDEX_NAME); + private static final Set guidKeys = new HashSet<>(Arrays.asList(DOMAIN_GUIDS)); /* * created → event creation time @@ -250,20 +251,28 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr Map detail = event.getDetail(); if (detail != null && detail.containsKey("attributes")) { Map attributes = (Map) detail.get("attributes"); - List domainGUIDs = (List) attributes.get(DOMAIN_GUIDS); - - if (domainGUIDs != null && !domainGUIDs.isEmpty()) { - List linkedEntityList = new ArrayList<>(); - for (String domainGUID: domainGUIDs) { - try { - AtlasEntityHeader domainEntityHeader = fetchAtlasEntityHeader(domainGUID); - if (domainEntityHeader != null) { - linkedEntityList.add(domainEntityHeader); + List linkedEntityList = new ArrayList<>(); + + for (Map.Entry entry: attributes.entrySet()) { + if (guidKeys.contains(entry.getKey()) && entry.getValue() instanceof List) { + List guids = (List) entry.getValue(); + + if (guids != null && !guids.isEmpty()){ + for (String guid: guids){ + try { + AtlasEntityHeader entityHeader = fetchAtlasEntityHeader(guid); + if (entityHeader != null) { + linkedEntityList.add(entityHeader); + } + } catch (AtlasBaseException e) { + throw new AtlasBaseException(e); + } } - } catch (AtlasBaseException e) { - throw new AtlasBaseException(e); } } + } + + if(!linkedEntityList.isEmpty()){ event.setLinkedEntities(linkedEntityList); } } From 838421ca29a417e5802d62444ddd0985b351d464 Mon Sep 17 00:00:00 2001 From: ankitpatnaik-atlan Date: Mon, 30 Dec 2024 13:14:50 +0530 Subject: [PATCH 5/7] mesh-299-audit: removed unrequired null check --- .../apache/atlas/repository/audit/ESBasedAuditRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index 97574c9d7d..03cb77a76f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -257,7 +257,7 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr if (guidKeys.contains(entry.getKey()) && entry.getValue() instanceof List) { List guids = (List) entry.getValue(); - if (guids != null && !guids.isEmpty()){ + if (!guids.isEmpty()){ for (String guid: guids){ try { AtlasEntityHeader entityHeader = fetchAtlasEntityHeader(guid); From c9d41aeb7eb3cf4d22d48b4713f12f21f6166f5f Mon Sep 17 00:00:00 2001 From: ankitpatnaik-atlan Date: Tue, 31 Dec 2024 15:38:38 +0530 Subject: [PATCH 6/7] mesh-299-audit: variable name change and changes in check --- .../atlas/repository/audit/ESBasedAuditRepository.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index 03cb77a76f..2b0c802b83 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -88,7 +88,7 @@ public class ESBasedAuditRepository extends AbstractStorageBasedAuditRepository private static final String DETAIL = "detail"; private static final String ENTITY = "entity"; private static final String bulkMetadata = String.format("{ \"index\" : { \"_index\" : \"%s\" } }%n", INDEX_NAME); - private static final Set guidKeys = new HashSet<>(Arrays.asList(DOMAIN_GUIDS)); + private static final Set linkedAttributes = new HashSet<>(Arrays.asList(DOMAIN_GUIDS)); /* * created → event creation time @@ -254,10 +254,10 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr List linkedEntityList = new ArrayList<>(); for (Map.Entry entry: attributes.entrySet()) { - if (guidKeys.contains(entry.getKey()) && entry.getValue() instanceof List) { + if (linkedAttributes.contains(entry.getKey())) { List guids = (List) entry.getValue(); - if (!guids.isEmpty()){ + if (guids != null && !guids.isEmpty()){ for (String guid: guids){ try { AtlasEntityHeader entityHeader = fetchAtlasEntityHeader(guid); From 074c3fc8d3531bfb146f6705beb3951542db7521 Mon Sep 17 00:00:00 2001 From: ankitpatnaik-atlan Date: Fri, 10 Jan 2025 18:13:07 +0530 Subject: [PATCH 7/7] mesh-299: changed the position of linkedentity attribute to root --- .../atlas/model/audit/EntityAuditEventV2.java | 17 ------------ .../model/audit/EntityAuditSearchResult.java | 7 +++++ .../audit/ESBasedAuditRepository.java | 26 +++++++++++-------- 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java index c7eb29fe27..199292160a 100644 --- a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java +++ b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java @@ -135,7 +135,6 @@ public static EntityAuditActionV2 fromString(String strValue) { private Map detail; private AtlasEntityHeader entityDetail; private Map headers; - private List linkedEntities; public EntityAuditEventV2() { } @@ -251,19 +250,6 @@ public void setHeaders(Map headers) { this.headers = headers; } - public List getLinkedEntities() { - return linkedEntities; - } - - public void setLinkedEntities(List linkedEntities) { - this.linkedEntities = linkedEntities; - } - - @JsonIgnore - public boolean hasLinkedEntities() { - return linkedEntities != null && !linkedEntities.isEmpty(); - } - @JsonIgnore public String getEntityDefinitionString() { if (entity != null) { @@ -330,9 +316,6 @@ public String toString() { sb.append(", created=").append(created); sb.append(", headers=").append(headers); - if (hasLinkedEntities()) { - sb.append(", linkedEntities=").append(linkedEntities); - } sb.append('}'); diff --git a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditSearchResult.java b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditSearchResult.java index a71cdf41db..f69edf1c7a 100644 --- a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditSearchResult.java +++ b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditSearchResult.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.atlas.model.Clearable; +import org.apache.atlas.model.instance.AtlasEntityHeader; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -30,6 +31,7 @@ public EntityAuditSearchResult() { } private Map aggregations; private int count; private int totalCount; + private Map linkedEntities; public List getEntityAudits() { return entityAudits; @@ -63,6 +65,10 @@ public void setTotalCount(int totalCount) { this.totalCount = totalCount; } + public Map getLinkedEntities() { return linkedEntities; } + + public void setLinkedEntities(Map linkedEntities) { this.linkedEntities = linkedEntities; } + @Override public boolean equals(Object o) { if (this == o) { return true; } @@ -85,6 +91,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("EntityAuditSearchResult{"); sb.append("entityAudits='").append(entityAudits).append('\''); sb.append(", aggregations='").append(aggregations).append('\''); + sb.append(", linkedEntities='").append(linkedEntities).append('\''); sb.append(", count=").append(count); sb.append(", totalCount=").append(totalCount); sb.append('}'); diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index 2b0c802b83..51e0012ff2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -225,6 +225,12 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr Map responseMap = AtlasType.fromJson(responseString, Map.class); Map hits_0 = (Map) responseMap.get("hits"); List hits_1 = (List) hits_0.get("hits"); + Map existingLinkedEntities = searchResult.getLinkedEntities(); + + if (existingLinkedEntities == null) { + existingLinkedEntities = new HashMap<>(); + } + for (LinkedHashMap hit : hits_1) { Map source = (Map) hit.get("_source"); String entityGuid = (String) source.get(ENTITYID); @@ -251,7 +257,6 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr Map detail = event.getDetail(); if (detail != null && detail.containsKey("attributes")) { Map attributes = (Map) detail.get("attributes"); - List linkedEntityList = new ArrayList<>(); for (Map.Entry entry: attributes.entrySet()) { if (linkedAttributes.contains(entry.getKey())) { @@ -259,22 +264,20 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr if (guids != null && !guids.isEmpty()){ for (String guid: guids){ - try { - AtlasEntityHeader entityHeader = fetchAtlasEntityHeader(guid); - if (entityHeader != null) { - linkedEntityList.add(entityHeader); + if(!existingLinkedEntities.containsKey(guid)){ + try { + AtlasEntityHeader entityHeader = fetchAtlasEntityHeader(guid); + if (entityHeader != null) { + existingLinkedEntities.put(guid, entityHeader); + } + } catch (AtlasBaseException e) { + throw new AtlasBaseException(e); } - } catch (AtlasBaseException e) { - throw new AtlasBaseException(e); } } } } } - - if(!linkedEntityList.isEmpty()){ - event.setLinkedEntities(linkedEntityList); - } } event.setHeaders((Map) source.get("headers")); @@ -286,6 +289,7 @@ private EntityAuditSearchResult getResultFromResponse(String responseString) thr Map countObject = (Map) hits_0.get("total"); int totalCount = (int) countObject.get("value"); searchResult.setEntityAudits(entityAudits); + searchResult.setLinkedEntities(existingLinkedEntities); searchResult.setAggregations(aggregationsMap); searchResult.setTotalCount(totalCount); searchResult.setCount(entityAudits.size());