From 5067eeb329d777900d433fe6985823bf0882bdd2 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 7 Nov 2024 15:26:43 +0530 Subject: [PATCH 001/241] DG-1907 Validate Channel Link --- .../AccessControlPreProcessor.java | 79 +++++++++++++++++++ .../accesscontrol/PersonaPreProcessor.java | 3 +- .../accesscontrol/PurposePreProcessor.java | 4 +- .../repository/util/AccessControlUtils.java | 2 + 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/AccessControlPreProcessor.java diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/AccessControlPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/AccessControlPreProcessor.java new file mode 100644 index 0000000000..eecb799b9e --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/AccessControlPreProcessor.java @@ -0,0 +1,79 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_CHANNEL_LINK; + +public class AccessControlPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AccessControlPreProcessor.class); + + // Regex reference - https://atlanhq.atlassian.net/browse/DG-1907?focusedCommentId=270252 + private static final Pattern REGEX_SLACK_CHANNEL_LINK = Pattern.compile("^https://(?\\w+\\.slack\\.com)/archives/(?C\\w{8,})(?:/p(?\\d{10}))?$"); + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("AccessControlPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + + switch (operation) { + case CREATE: + processCreateAccessControlAsset(entity); + break; + case UPDATE: + processUpdateAccessControlAsset(context, entity); + break; + } + } + + private void processCreateAccessControlAsset(AtlasEntity entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateAccessControlAsset"); + + validateChannelLink(entity); + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void processUpdateAccessControlAsset(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateAccessControlAsset"); + + validateChannelLink(entity); + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void validateChannelLink(AtlasEntity entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("validateChannelLink"); + + if (entity.hasAttribute(ATTR_CHANNEL_LINK)) { + String channelLink = (String) entity.getAttribute(ATTR_CHANNEL_LINK); + + if (StringUtils.isNotEmpty(channelLink)) { + Matcher channelLinkMatcher = REGEX_SLACK_CHANNEL_LINK.matcher(channelLink); + + if (!channelLinkMatcher.matches()) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Please provide a valid URL for " + ATTR_CHANNEL_LINK); + } + } + } + + RequestContext.get().endMetricRecord(metricRecorder); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index 222bca0962..8a2c8b1945 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -71,7 +71,7 @@ import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; import static org.apache.atlas.repository.util.AccessControlUtils.validateNoPoliciesAttached; -public class PersonaPreProcessor implements PreProcessor { +public class PersonaPreProcessor extends AccessControlPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(PersonaPreProcessor.class); protected final AtlasGraph graph; @@ -100,6 +100,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co if (LOG.isDebugEnabled()) { LOG.debug("PersonaPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); } + super.processAttributes(entityStruct, context, operation); AtlasEntity entity = (AtlasEntity) entityStruct; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java index 1bb311e84e..ca8fcb803f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java @@ -62,7 +62,7 @@ import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; import static org.apache.atlas.repository.util.AccessControlUtils.validateUniquenessByTags; -public class PurposePreProcessor implements PreProcessor { +public class PurposePreProcessor extends AccessControlPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(PurposePreProcessor.class); private final AtlasGraph graph; @@ -90,6 +90,8 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co LOG.debug("PurposePreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); } + super.processAttributes(entityStruct, context, operation); + AtlasEntity entity = (AtlasEntity) entityStruct; switch (operation) { diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index 68ee5318ed..7101868e30 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -81,6 +81,8 @@ public final class AccessControlUtils { public static final String POLICY_TYPE_ALLOW = "allow"; public static final String POLICY_TYPE_DENY = "deny"; + public static final String ATTR_CHANNEL_LINK = "channelLink"; + public static final String ACCESS_READ_PURPOSE_METADATA = "entity-read"; public static final String ACCESS_READ_PERSONA_METADATA = "persona-asset-read"; public static final String ACCESS_READ_PERSONA_GLOSSARY = "persona-glossary-read"; From f227a6895a137b0a3892b1d14fce6bc7f5074b76 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 11 Nov 2024 13:54:04 +0530 Subject: [PATCH 002/241] Added secure in ATLASSESSIONID Cookie. --- .github/workflows/maven.yml | 4 ++-- webapp/src/main/webapp/WEB-INF/web.xml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..3af25b1ea5 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - lineageondemand + - dg1908 jobs: build: @@ -64,7 +64,7 @@ jobs: - name: Build with Maven run: | branch_name=${{ steps.get_branch.outputs.branch }} - if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]] + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'dg1908' ]] then echo "build without dashboard" chmod +x ./build.sh && ./build.sh build_without_dashboard diff --git a/webapp/src/main/webapp/WEB-INF/web.xml b/webapp/src/main/webapp/WEB-INF/web.xml index 07092d62eb..590901b279 100755 --- a/webapp/src/main/webapp/WEB-INF/web.xml +++ b/webapp/src/main/webapp/WEB-INF/web.xml @@ -129,6 +129,7 @@ ATLASSESSIONID true + true From a7d650aadc6902434311d1fa2e7f39ad38ed3631 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Tue, 12 Nov 2024 01:03:14 +0530 Subject: [PATCH 003/241] fix: Added 20s timeout for lineage traversal --- .../lineage/AtlasLineageOnDemandInfo.java | 9 + .../atlas/discovery/EntityLineageService.java | 163 +++++++++++------- .../atlas/discovery/TimeoutTracker.java | 17 ++ 3 files changed, 122 insertions(+), 67 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/discovery/TimeoutTracker.java diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java index 3a86a8a963..9ddcfe8f2d 100644 --- a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java +++ b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java @@ -33,6 +33,7 @@ public class AtlasLineageOnDemandInfo implements Serializable { private LineageOnDemandRequest lineageOnDemandPayload; private boolean upstreamEntityLimitReached; private boolean downstreamEntityLimitReached; + private boolean traversalTimedOut; public AtlasLineageOnDemandInfo() { } @@ -135,6 +136,14 @@ public void setDownstreamEntityLimitReached(boolean downstreamEntityLimitReached this.downstreamEntityLimitReached = downstreamEntityLimitReached; } + public boolean isTraversalTimedOut() { + return traversalTimedOut; + } + + public void setTraversalTimedOut(boolean traversalTimedOut) { + this.traversalTimedOut = traversalTimedOut; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index a873105a6d..17758bded6 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -298,12 +298,13 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag AtomicInteger inputEntitiesTraversed = new AtomicInteger(0); AtomicInteger outputEntitiesTraversed = new AtomicInteger(0); AtomicInteger traversalOrder = new AtomicInteger(1); + TimeoutTracker tracker = new TimeoutTracker(50); // 20 seconds timeout if (isDataSet) { AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, true, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed, traversalOrder); + traverseEdgesOnDemand(datasetVertex, true, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed, traversalOrder, tracker); if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed, traversalOrder); + traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed, traversalOrder, tracker); AtlasEntityHeader baseEntityHeader = entityRetriever.toAtlasEntityHeader(datasetVertex, atlasLineageOnDemandContext.getAttributes()); setGraphTraversalMetadata(level, traversalOrder, baseEntityHeader); ret.getGuidEntityMap().put(guid, baseEntityHeader); @@ -312,11 +313,11 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag // make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1' if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_INPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, true, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed, traversalOrder); + traverseEdgesOnDemand(processEdges, true, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed, traversalOrder, tracker); } if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_OUTPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, false, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed, traversalOrder); + traverseEdgesOnDemand(processEdges, false, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed, traversalOrder, tracker); } } RequestContext.get().endMetricRecord(metricRecorder); @@ -329,7 +330,7 @@ private static void setGraphTraversalMetadata(int level, AtomicInteger traversal baseEntityHeader.setFinishTime(traversalOrder.get()); } - private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder) throws AtlasBaseException { + private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder, TimeoutTracker timeoutTracker) throws AtlasBaseException { AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; int nextLevel = isInput ? level - 1: level + 1; @@ -346,7 +347,7 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI } boolean isInputEdge = processEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); - if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, new HashSet<>())) { + if (handleHorizontalAndVerticalPagination(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, new HashSet<>(), timeoutTracker)) { break; } else { addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); @@ -360,66 +361,44 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI ret.getRelationsOnDemand().put(inGuid, new LineageInfoOnDemand(inGuidLineageConstrains)); } - traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, nextLevel, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); + traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, nextLevel, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder, timeoutTracker); } } - private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder) throws AtlasBaseException { + private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder, TimeoutTracker timeoutTracker) throws AtlasBaseException { if (isEntityTraversalLimitReached(entitiesTraversed)) return; if (depth != 0) { // base condition of recursion for depth AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("traverseEdgesOnDemand"); - AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; - int nextLevel = isInput ? level - 1: level + 1; - // keep track of visited vertices to avoid circular loop - visitedVertices.add(getId(datasetVertex)); + try { + int nextLevel = isInput ? level - 1 : level + 1; + // keep track of visited vertices to avoid circular loop + visitedVertices.add(getId(datasetVertex)); - AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesIn = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesIn"); - Iterator incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); - RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesIn); + AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesIn = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesIn"); + Iterator incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); + RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesIn); - while (incomingEdges.hasNext()) { - AtlasEdge incomingEdge = incomingEdges.next(); - AtlasVertex processVertex = incomingEdge.getOutVertex(); + while (incomingEdges.hasNext()) { + AtlasEdge incomingEdge = incomingEdges.next(); + AtlasVertex processVertex = incomingEdge.getOutVertex(); - if (!vertexMatchesEvaluation(processVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(incomingEdge, atlasLineageOnDemandContext)) { - continue; - } - - if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { - continue; - } - - if (incrementAndCheckIfRelationsLimitReached(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, visitedVertices)) { - LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(baseGuid); - if (entityOnDemandInfo == null) - continue; - if (isInput ? entityOnDemandInfo.isInputRelationsReachedLimit() : entityOnDemandInfo.isOutputRelationsReachedLimit()) - break; - else + if (!vertexMatchesEvaluation(processVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(incomingEdge, atlasLineageOnDemandContext)) { continue; - } else { - addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, level, traversalOrder); - } - - AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); - Iterator outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); - RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesOut); - - while (outgoingEdges.hasNext()) { - AtlasEdge outgoingEdge = outgoingEdges.next(); - AtlasVertex entityVertex = outgoingEdge.getInVertex(); + } - if (!vertexMatchesEvaluation(entityVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(outgoingEdge, atlasLineageOnDemandContext)) { - continue; + if (timeoutTracker.hasTimedOut()) { + executeCircuitBreaker(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + return; } - if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { + if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { continue; } - if (incrementAndCheckIfRelationsLimitReached(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, visitedVertices)) { - String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); - LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(processGuid); + + boolean isRelationsLimitReached = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + if (isRelationsLimitReached) { + LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(baseGuid); if (entityOnDemandInfo == null) continue; if (isInput ? entityOnDemandInfo.isInputRelationsReachedLimit() : entityOnDemandInfo.isOutputRelationsReachedLimit()) @@ -427,24 +406,69 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); - entitiesTraversed.incrementAndGet(); - traversalOrder.incrementAndGet(); - if (isEntityTraversalLimitReached(entitiesTraversed)) - setEntityLimitReachedFlag(isInput, ret); + addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, level, traversalOrder); } - if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { - traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); // execute inner depth - AtlasEntityHeader traversedEntity = ret.getGuidEntityMap().get(AtlasGraphUtilsV2.getIdFromVertex(entityVertex)); - traversedEntity.setFinishTime(traversalOrder.get()); + + AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); + Iterator outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); + RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesOut); + + while (outgoingEdges.hasNext()) { + AtlasEdge outgoingEdge = outgoingEdges.next(); + AtlasVertex entityVertex = outgoingEdge.getInVertex(); + + if (!vertexMatchesEvaluation(entityVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(outgoingEdge, atlasLineageOnDemandContext)) { + continue; + } + + if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { + continue; + } + boolean stopTraversal = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + // If timeout occurred or entity limit reached, set pagination flags again for consistency and stop + if (timeoutTracker.hasTimedOut()) { + handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + break; + } + if (stopTraversal) { + String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); + LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(processGuid); + + if (entityOnDemandInfo == null) + continue; + if (isInput ? entityOnDemandInfo.isInputRelationsReachedLimit() : entityOnDemandInfo.isOutputRelationsReachedLimit()) + break; + else + continue; + } else { + addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); + entitiesTraversed.incrementAndGet(); + traversalOrder.incrementAndGet(); + + if (isEntityTraversalLimitReached(entitiesTraversed)) + setEntityLimitReachedFlag(isInput, ret); + } + if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { + traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder, timeoutTracker); // execute inner depth + AtlasEntityHeader traversedEntity = ret.getGuidEntityMap().get(AtlasGraphUtilsV2.getIdFromVertex(entityVertex)); + if (traversedEntity != null) + traversedEntity.setFinishTime(traversalOrder.get()); + } } } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); } - - RequestContext.get().endMetricRecord(metricRecorder); } } + + private void executeCircuitBreaker(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, Set visitedVertices, TimeoutTracker timeoutTracker) { + // Set timeout flags on the entity + handleHorizontalAndVerticalPagination(atlasEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + ret.setTraversalTimedOut(true); + } + private static void setEntityLimitReachedFlag(boolean isInput, AtlasLineageOnDemandInfo ret) { if (isInput) ret.setUpstreamEntityLimitReached(true); @@ -684,7 +708,7 @@ private static String getId(AtlasVertex vertex) { return vertex.getIdForDisplay(); } - private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasLineageOnDemandInfo.LineageDirection direction, Set visitedVertices) { + private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, Set visitedVertices, TimeoutTracker timeoutTracker) { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("incrementAndCheckIfRelationsLimitReached"); AtlasVertex inVertex = isInput ? atlasEdge.getOutVertex() : atlasEdge.getInVertex(); @@ -698,9 +722,9 @@ private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, bo LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints); LineageInfoOnDemand outLineageInfo = ret.getRelationsOnDemand().containsKey(outGuid) ? ret.getRelationsOnDemand().get(outGuid) : new LineageInfoOnDemand(outGuidLineageConstraints); - setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, inVertex, inGuid, outVertex, outGuid, inLineageInfo, outLineageInfo, visitedVertices); + setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, depth, entitiesTraversed, inVertex, outVertex, inLineageInfo, outLineageInfo, visitedVertices, timeoutTracker); - boolean hasRelationsLimitReached = setVerticalPaginationFlags(entitiesTraversed, inLineageInfo, outLineageInfo); + boolean hasRelationsLimitReached = setVerticalPaginationFlags(entitiesTraversed, inLineageInfo, outLineageInfo, timeoutTracker); if (!hasRelationsLimitReached) { ret.getRelationsOnDemand().put(inGuid, inLineageInfo); ret.getRelationsOnDemand().put(outGuid, outLineageInfo); @@ -710,8 +734,13 @@ private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, bo return hasRelationsLimitReached; } - private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo) { + private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, TimeoutTracker timeoutTracker) { boolean hasRelationsLimitReached = false; + if (timeoutTracker.hasTimedOut()) { + inLineageInfo.setHasMoreInputs(true); + outLineageInfo.setHasMoreOutputs(true); + } + if (inLineageInfo.isInputRelationsReachedLimit() || outLineageInfo.isOutputRelationsReachedLimit() || isEntityTraversalLimitReached(entitiesTraversed)) { inLineageInfo.setHasMoreInputs(true); outLineageInfo.setHasMoreOutputs(true); @@ -725,10 +754,10 @@ private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, Line return hasRelationsLimitReached; } - private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, String inGuid, AtlasVertex outVertex, String outGuid, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set visitedVertices) { + private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, AtlasVertex outVertex, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set visitedVertices, TimeoutTracker timeoutTracker) { boolean isOutVertexVisited = visitedVertices.contains(getId(outVertex)); boolean isInVertexVisited = visitedVertices.contains(getId(inVertex)); - if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1) { // is the vertex a leaf? + if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1 || timeoutTracker.hasTimedOut()) { // If traversal has to stop, set pagination flags if (isInput && ! isOutVertexVisited) setHasUpstream(atlasLineageOnDemandContext, outVertex, outLineageInfo); else if (!isInput && ! isInVertexVisited) diff --git a/repository/src/main/java/org/apache/atlas/discovery/TimeoutTracker.java b/repository/src/main/java/org/apache/atlas/discovery/TimeoutTracker.java new file mode 100644 index 0000000000..3f7fce2553 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/TimeoutTracker.java @@ -0,0 +1,17 @@ +package org.apache.atlas.discovery; + +public class TimeoutTracker { + + private final long startTime; + private final long timeoutMillis; + + public TimeoutTracker(long timeoutMillis) { + this.startTime = System.currentTimeMillis(); + this.timeoutMillis = timeoutMillis; + } + + public boolean hasTimedOut() { + return (System.currentTimeMillis() - startTime) > timeoutMillis; + } + +} From 750a0ab1f4ce7c763a2c0f2bcb0327acec56ced6 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Tue, 12 Nov 2024 13:42:33 +0530 Subject: [PATCH 004/241] build fix --- .../java/org/apache/atlas/discovery/EntityLineageService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 17758bded6..b0e3f63fe8 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -522,6 +522,7 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices, lineageParentsForEntityMap, lineageChildrenForEntityMap); continue; } + if (checkOffsetAndSkipEntity(lineageListContext, ret)) { skippedVertices.add(currentGUID); enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices, lineageParentsForEntityMap, lineageChildrenForEntityMap); From b15906f7e4276ea167521af10b904ffa7e42f1c4 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Tue, 12 Nov 2024 16:52:19 +0530 Subject: [PATCH 005/241] fix --- .../java/org/apache/atlas/discovery/EntityLineageService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index a873105a6d..0c5850f241 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -543,6 +543,7 @@ private void enqueueNeighbours(AtlasVertex currentVertex, boolean isDataset, RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdges); while (edges.hasNext()) { + AtlasEdge currentEdge = edges.next(); if (!lineageListContext.evaluateTraversalFilter(currentEdge)) continue; From 3a21da7046356b9639e7f71ad3a7a8b87f34e051 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Tue, 12 Nov 2024 17:13:42 +0530 Subject: [PATCH 006/241] ci fix --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..84d3ddfe43 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -77,7 +77,7 @@ jobs: shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git refs/heads/${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" id: get_version - name: Set up Buildx From 2aa74f085a9c56ca1343ac7e6aff92219d344c01 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Tue, 12 Nov 2024 18:50:09 +0530 Subject: [PATCH 007/241] Added 15s timeout --- .../org/apache/atlas/discovery/EntityLineageService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index b0e3f63fe8..918d3167b4 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -81,7 +81,7 @@ @Service public class EntityLineageService implements AtlasLineageService { private static final Logger LOG = LoggerFactory.getLogger(EntityLineageService.class); - + private static final long LINEAGE_TRAVERSAL_TIMEOUT_MILLIS = 15000; private static final String PROCESS_INPUTS_EDGE = "__Process.inputs"; private static final String PROCESS_OUTPUTS_EDGE = "__Process.outputs"; private static final String COLUMNS = "columns"; @@ -298,7 +298,7 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag AtomicInteger inputEntitiesTraversed = new AtomicInteger(0); AtomicInteger outputEntitiesTraversed = new AtomicInteger(0); AtomicInteger traversalOrder = new AtomicInteger(1); - TimeoutTracker tracker = new TimeoutTracker(50); // 20 seconds timeout + TimeoutTracker tracker = new TimeoutTracker(LINEAGE_TRAVERSAL_TIMEOUT_MILLIS); if (isDataSet) { AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) @@ -723,6 +723,7 @@ private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boole LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints); LineageInfoOnDemand outLineageInfo = ret.getRelationsOnDemand().containsKey(outGuid) ? ret.getRelationsOnDemand().get(outGuid) : new LineageInfoOnDemand(outGuidLineageConstraints); + setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, depth, entitiesTraversed, inVertex, outVertex, inLineageInfo, outLineageInfo, visitedVertices, timeoutTracker); boolean hasRelationsLimitReached = setVerticalPaginationFlags(entitiesTraversed, inLineageInfo, outLineageInfo, timeoutTracker); From 17b7ec97540a424720b71c57a50eba744ab6e90b Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Tue, 12 Nov 2024 22:22:53 +0530 Subject: [PATCH 008/241] Timeout - response clean up handling --- .../atlas/discovery/EntityLineageService.java | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 918d3167b4..7e5fd20992 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -387,17 +387,18 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i continue; } - if (timeoutTracker.hasTimedOut()) { - executeCircuitBreaker(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - return; - } - if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { continue; } - boolean isRelationsLimitReached = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - if (isRelationsLimitReached) { + boolean stopProcessIteration = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + // If timeout occurred or entity limit reached, set pagination flags again for consistency and stop + if (timeoutTracker.hasTimedOut()) { + handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + break; + } + + if (stopProcessIteration) { LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(baseGuid); if (entityOnDemandInfo == null) continue; @@ -424,13 +425,15 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { continue; } - boolean stopTraversal = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - // If timeout occurred or entity limit reached, set pagination flags again for consistency and stop + boolean stopDatasetIteration = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + // If timeout occurred or entity limit reached, set pagination flags again for consistency if (timeoutTracker.hasTimedOut()) { handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - break; + ret.setTraversalTimedOut(true); + return; } - if (stopTraversal) { + + if (stopDatasetIteration) { String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(processGuid); @@ -447,6 +450,11 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i if (isEntityTraversalLimitReached(entitiesTraversed)) setEntityLimitReachedFlag(isInput, ret); + if (timeoutTracker.hasTimedOut()) { + handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + ret.setTraversalTimedOut(true); + return; + } } if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder, timeoutTracker); // execute inner depth From 0e531548d95511901828917fe459c93b317798ab Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Tue, 12 Nov 2024 22:38:03 +0530 Subject: [PATCH 009/241] Timeout - response clean up handling --- .../apache/atlas/discovery/EntityLineageService.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 7e5fd20992..a881539741 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -366,7 +366,7 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI } private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder, TimeoutTracker timeoutTracker) throws AtlasBaseException { - if (isEntityTraversalLimitReached(entitiesTraversed)) + if (isEntityTraversalLimitReached(entitiesTraversed) || timeoutTracker.hasTimedOut()) return; if (depth != 0) { // base condition of recursion for depth AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("traverseEdgesOnDemand"); @@ -461,6 +461,10 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i AtlasEntityHeader traversedEntity = ret.getGuidEntityMap().get(AtlasGraphUtilsV2.getIdFromVertex(entityVertex)); if (traversedEntity != null) traversedEntity.setFinishTime(traversalOrder.get()); + if (timeoutTracker.hasTimedOut()) { + ret.setTraversalTimedOut(true); + return; + } } } } @@ -471,12 +475,6 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i } - private void executeCircuitBreaker(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, Set visitedVertices, TimeoutTracker timeoutTracker) { - // Set timeout flags on the entity - handleHorizontalAndVerticalPagination(atlasEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - ret.setTraversalTimedOut(true); - } - private static void setEntityLimitReachedFlag(boolean isInput, AtlasLineageOnDemandInfo ret) { if (isInput) ret.setUpstreamEntityLimitReached(true); From 34b4f6d7727d1bad638ca673f31529e927c10e3e Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 13 Nov 2024 00:25:29 +0530 Subject: [PATCH 010/241] Timeout - response clean up handling --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 +- .../java/org/apache/atlas/discovery/EntityLineageService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 7fd081b9e3..5fe8a5e96f 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -94,7 +94,7 @@ public enum AtlasConfiguration { GRAPH_TRAVERSAL_PARALLELISM("atlas.graph.traverse.bucket.size",10), LINEAGE_ON_DEMAND_ENABLED("atlas.lineage.on.demand.enabled", true), LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT("atlas.lineage.on.demand.default.node.count", 3), - LINEAGE_MAX_NODE_COUNT("atlas.lineage.max.node.count", 100), + LINEAGE_MAX_NODE_COUNT("atlas.lineage.max.node.count", 1000), SUPPORTED_RELATIONSHIP_EVENTS("atlas.notification.relationships.filter", "asset_readme,asset_links"), diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index a881539741..58d2717272 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -81,7 +81,7 @@ @Service public class EntityLineageService implements AtlasLineageService { private static final Logger LOG = LoggerFactory.getLogger(EntityLineageService.class); - private static final long LINEAGE_TRAVERSAL_TIMEOUT_MILLIS = 15000; + private static final long LINEAGE_TRAVERSAL_TIMEOUT_MILLIS = 4; private static final String PROCESS_INPUTS_EDGE = "__Process.inputs"; private static final String PROCESS_OUTPUTS_EDGE = "__Process.outputs"; private static final String COLUMNS = "columns"; From f31e7ef78dedd9061b99e293407851b6c7712263 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 13 Nov 2024 00:26:40 +0530 Subject: [PATCH 011/241] Timeout - response clean up handling --- .../java/org/apache/atlas/discovery/EntityLineageService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 58d2717272..66591daa9d 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -81,7 +81,7 @@ @Service public class EntityLineageService implements AtlasLineageService { private static final Logger LOG = LoggerFactory.getLogger(EntityLineageService.class); - private static final long LINEAGE_TRAVERSAL_TIMEOUT_MILLIS = 4; + private static final long LINEAGE_TRAVERSAL_TIMEOUT_MILLIS = 4000; private static final String PROCESS_INPUTS_EDGE = "__Process.inputs"; private static final String PROCESS_OUTPUTS_EDGE = "__Process.outputs"; private static final String COLUMNS = "columns"; From 3202acdc84e87180869847119bb994e369741dd1 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 13 Nov 2024 02:33:43 +0530 Subject: [PATCH 012/241] fixes --- .../atlas/discovery/EntityLineageService.java | 63 ++++++++----------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 66591daa9d..c70edb0bde 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -347,7 +347,7 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI } boolean isInputEdge = processEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); - if (handleHorizontalAndVerticalPagination(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, new HashSet<>(), timeoutTracker)) { + if (handleHorizontalAndVerticalPagination(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, new HashSet<>(), false)) { break; } else { addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); @@ -381,8 +381,14 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i while (incomingEdges.hasNext()) { AtlasEdge incomingEdge = incomingEdges.next(); - AtlasVertex processVertex = incomingEdge.getOutVertex(); + boolean isTimedOut = timeoutTracker.hasTimedOut(); + if (isTimedOut) { + handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); + ret.setTraversalTimedOut(true); + break; + } + AtlasVertex processVertex = incomingEdge.getOutVertex(); if (!vertexMatchesEvaluation(processVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(incomingEdge, atlasLineageOnDemandContext)) { continue; } @@ -391,13 +397,8 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i continue; } - boolean stopProcessIteration = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + boolean stopProcessIteration = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); // If timeout occurred or entity limit reached, set pagination flags again for consistency and stop - if (timeoutTracker.hasTimedOut()) { - handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - break; - } - if (stopProcessIteration) { LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(baseGuid); if (entityOnDemandInfo == null) @@ -416,8 +417,14 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i while (outgoingEdges.hasNext()) { AtlasEdge outgoingEdge = outgoingEdges.next(); - AtlasVertex entityVertex = outgoingEdge.getInVertex(); + isTimedOut = timeoutTracker.hasTimedOut(); + if (isTimedOut) { + handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); + ret.setTraversalTimedOut(true); + break; + } + AtlasVertex entityVertex = outgoingEdge.getInVertex(); if (!vertexMatchesEvaluation(entityVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(outgoingEdge, atlasLineageOnDemandContext)) { continue; } @@ -425,14 +432,8 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { continue; } - boolean stopDatasetIteration = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - // If timeout occurred or entity limit reached, set pagination flags again for consistency - if (timeoutTracker.hasTimedOut()) { - handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - ret.setTraversalTimedOut(true); - return; - } + boolean stopDatasetIteration = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); if (stopDatasetIteration) { String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(processGuid); @@ -450,21 +451,12 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i if (isEntityTraversalLimitReached(entitiesTraversed)) setEntityLimitReachedFlag(isInput, ret); - if (timeoutTracker.hasTimedOut()) { - handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - ret.setTraversalTimedOut(true); - return; - } } if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder, timeoutTracker); // execute inner depth AtlasEntityHeader traversedEntity = ret.getGuidEntityMap().get(AtlasGraphUtilsV2.getIdFromVertex(entityVertex)); if (traversedEntity != null) traversedEntity.setFinishTime(traversalOrder.get()); - if (timeoutTracker.hasTimedOut()) { - ret.setTraversalTimedOut(true); - return; - } } } } @@ -715,7 +707,7 @@ private static String getId(AtlasVertex vertex) { return vertex.getIdForDisplay(); } - private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, Set visitedVertices, TimeoutTracker timeoutTracker) { + private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, Set visitedVertices, boolean isTimedOut) { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("incrementAndCheckIfRelationsLimitReached"); AtlasVertex inVertex = isInput ? atlasEdge.getOutVertex() : atlasEdge.getInVertex(); @@ -729,10 +721,9 @@ private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boole LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints); LineageInfoOnDemand outLineageInfo = ret.getRelationsOnDemand().containsKey(outGuid) ? ret.getRelationsOnDemand().get(outGuid) : new LineageInfoOnDemand(outGuidLineageConstraints); + setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, depth, entitiesTraversed, inVertex, outVertex, inLineageInfo, outLineageInfo, visitedVertices, isTimedOut); - setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, depth, entitiesTraversed, inVertex, outVertex, inLineageInfo, outLineageInfo, visitedVertices, timeoutTracker); - - boolean hasRelationsLimitReached = setVerticalPaginationFlags(entitiesTraversed, inLineageInfo, outLineageInfo, timeoutTracker); + boolean hasRelationsLimitReached = setVerticalPaginationFlags(inLineageInfo, outLineageInfo, isTimedOut); if (!hasRelationsLimitReached) { ret.getRelationsOnDemand().put(inGuid, inLineageInfo); ret.getRelationsOnDemand().put(outGuid, outLineageInfo); @@ -742,14 +733,14 @@ private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boole return hasRelationsLimitReached; } - private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, TimeoutTracker timeoutTracker) { - boolean hasRelationsLimitReached = false; - if (timeoutTracker.hasTimedOut()) { + private boolean setVerticalPaginationFlags(LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, boolean isTimedOut) { + if (isTimedOut) { inLineageInfo.setHasMoreInputs(true); outLineageInfo.setHasMoreOutputs(true); + return true; } - - if (inLineageInfo.isInputRelationsReachedLimit() || outLineageInfo.isOutputRelationsReachedLimit() || isEntityTraversalLimitReached(entitiesTraversed)) { + boolean hasRelationsLimitReached = false; + if (inLineageInfo.isInputRelationsReachedLimit() || outLineageInfo.isOutputRelationsReachedLimit()) { inLineageInfo.setHasMoreInputs(true); outLineageInfo.setHasMoreOutputs(true); hasRelationsLimitReached = true; @@ -762,10 +753,10 @@ private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, Line return hasRelationsLimitReached; } - private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, AtlasVertex outVertex, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set visitedVertices, TimeoutTracker timeoutTracker) { + private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, AtlasVertex outVertex, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set visitedVertices, boolean isTimedOut) { boolean isOutVertexVisited = visitedVertices.contains(getId(outVertex)); boolean isInVertexVisited = visitedVertices.contains(getId(inVertex)); - if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1 || timeoutTracker.hasTimedOut()) { // If traversal has to stop, set pagination flags + if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1 || isTimedOut) { // If traversal has to stop, set pagination flags if (isInput && ! isOutVertexVisited) setHasUpstream(atlasLineageOnDemandContext, outVertex, outLineageInfo); else if (!isInput && ! isInVertexVisited) From 558bf81cc6848c65dbcdb0172a90bbec09800fdd Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 13 Nov 2024 02:38:17 +0530 Subject: [PATCH 013/241] fixes to timeout algo --- .../atlas/discovery/EntityLineageService.java | 63 ++++++++----------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index a881539741..16c3ec816a 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -347,7 +347,7 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI } boolean isInputEdge = processEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); - if (handleHorizontalAndVerticalPagination(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, new HashSet<>(), timeoutTracker)) { + if (handleHorizontalAndVerticalPagination(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, new HashSet<>(), false)) { break; } else { addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); @@ -381,8 +381,14 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i while (incomingEdges.hasNext()) { AtlasEdge incomingEdge = incomingEdges.next(); - AtlasVertex processVertex = incomingEdge.getOutVertex(); + boolean isTimedOut = timeoutTracker.hasTimedOut(); + if (isTimedOut) { + handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); + ret.setTraversalTimedOut(true); + break; + } + AtlasVertex processVertex = incomingEdge.getOutVertex(); if (!vertexMatchesEvaluation(processVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(incomingEdge, atlasLineageOnDemandContext)) { continue; } @@ -391,13 +397,8 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i continue; } - boolean stopProcessIteration = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); + boolean stopProcessIteration = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); // If timeout occurred or entity limit reached, set pagination flags again for consistency and stop - if (timeoutTracker.hasTimedOut()) { - handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - break; - } - if (stopProcessIteration) { LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(baseGuid); if (entityOnDemandInfo == null) @@ -416,8 +417,14 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i while (outgoingEdges.hasNext()) { AtlasEdge outgoingEdge = outgoingEdges.next(); - AtlasVertex entityVertex = outgoingEdge.getInVertex(); + isTimedOut = timeoutTracker.hasTimedOut(); + if (isTimedOut) { + handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); + ret.setTraversalTimedOut(true); + break; + } + AtlasVertex entityVertex = outgoingEdge.getInVertex(); if (!vertexMatchesEvaluation(entityVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(outgoingEdge, atlasLineageOnDemandContext)) { continue; } @@ -425,14 +432,8 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { continue; } - boolean stopDatasetIteration = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - // If timeout occurred or entity limit reached, set pagination flags again for consistency - if (timeoutTracker.hasTimedOut()) { - handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - ret.setTraversalTimedOut(true); - return; - } + boolean stopDatasetIteration = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); if (stopDatasetIteration) { String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(processGuid); @@ -450,21 +451,12 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i if (isEntityTraversalLimitReached(entitiesTraversed)) setEntityLimitReachedFlag(isInput, ret); - if (timeoutTracker.hasTimedOut()) { - handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, timeoutTracker); - ret.setTraversalTimedOut(true); - return; - } } if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder, timeoutTracker); // execute inner depth AtlasEntityHeader traversedEntity = ret.getGuidEntityMap().get(AtlasGraphUtilsV2.getIdFromVertex(entityVertex)); if (traversedEntity != null) traversedEntity.setFinishTime(traversalOrder.get()); - if (timeoutTracker.hasTimedOut()) { - ret.setTraversalTimedOut(true); - return; - } } } } @@ -715,7 +707,7 @@ private static String getId(AtlasVertex vertex) { return vertex.getIdForDisplay(); } - private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, Set visitedVertices, TimeoutTracker timeoutTracker) { + private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, Set visitedVertices, boolean isTimedOut) { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("incrementAndCheckIfRelationsLimitReached"); AtlasVertex inVertex = isInput ? atlasEdge.getOutVertex() : atlasEdge.getInVertex(); @@ -729,10 +721,9 @@ private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boole LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints); LineageInfoOnDemand outLineageInfo = ret.getRelationsOnDemand().containsKey(outGuid) ? ret.getRelationsOnDemand().get(outGuid) : new LineageInfoOnDemand(outGuidLineageConstraints); + setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, depth, entitiesTraversed, inVertex, outVertex, inLineageInfo, outLineageInfo, visitedVertices, isTimedOut); - setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, depth, entitiesTraversed, inVertex, outVertex, inLineageInfo, outLineageInfo, visitedVertices, timeoutTracker); - - boolean hasRelationsLimitReached = setVerticalPaginationFlags(entitiesTraversed, inLineageInfo, outLineageInfo, timeoutTracker); + boolean hasRelationsLimitReached = setVerticalPaginationFlags(inLineageInfo, outLineageInfo, isTimedOut); if (!hasRelationsLimitReached) { ret.getRelationsOnDemand().put(inGuid, inLineageInfo); ret.getRelationsOnDemand().put(outGuid, outLineageInfo); @@ -742,14 +733,14 @@ private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boole return hasRelationsLimitReached; } - private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, TimeoutTracker timeoutTracker) { - boolean hasRelationsLimitReached = false; - if (timeoutTracker.hasTimedOut()) { + private boolean setVerticalPaginationFlags(LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, boolean isTimedOut) { + if (isTimedOut) { inLineageInfo.setHasMoreInputs(true); outLineageInfo.setHasMoreOutputs(true); + return true; } - - if (inLineageInfo.isInputRelationsReachedLimit() || outLineageInfo.isOutputRelationsReachedLimit() || isEntityTraversalLimitReached(entitiesTraversed)) { + boolean hasRelationsLimitReached = false; + if (inLineageInfo.isInputRelationsReachedLimit() || outLineageInfo.isOutputRelationsReachedLimit()) { inLineageInfo.setHasMoreInputs(true); outLineageInfo.setHasMoreOutputs(true); hasRelationsLimitReached = true; @@ -762,10 +753,10 @@ private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, Line return hasRelationsLimitReached; } - private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, AtlasVertex outVertex, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set visitedVertices, TimeoutTracker timeoutTracker) { + private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, AtlasVertex outVertex, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set visitedVertices, boolean isTimedOut) { boolean isOutVertexVisited = visitedVertices.contains(getId(outVertex)); boolean isInVertexVisited = visitedVertices.contains(getId(inVertex)); - if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1 || timeoutTracker.hasTimedOut()) { // If traversal has to stop, set pagination flags + if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1 || isTimedOut) { // If traversal has to stop, set pagination flags if (isInput && ! isOutVertexVisited) setHasUpstream(atlasLineageOnDemandContext, outVertex, outLineageInfo); else if (!isInput && ! isInVertexVisited) From 505150a6df2a6763d81aa0148c340dba2c13006f Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 13 Nov 2024 02:47:27 +0530 Subject: [PATCH 014/241] moved pagination logic before timeouts --- .../atlas/discovery/EntityLineageService.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 16c3ec816a..9ee18d61a4 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -381,6 +381,11 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i while (incomingEdges.hasNext()) { AtlasEdge incomingEdge = incomingEdges.next(); + + if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { + continue; + } + boolean isTimedOut = timeoutTracker.hasTimedOut(); if (isTimedOut) { handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); @@ -393,10 +398,6 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i continue; } - if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { - continue; - } - boolean stopProcessIteration = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); // If timeout occurred or entity limit reached, set pagination flags again for consistency and stop if (stopProcessIteration) { @@ -417,6 +418,11 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i while (outgoingEdges.hasNext()) { AtlasEdge outgoingEdge = outgoingEdges.next(); + + if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { + continue; + } + isTimedOut = timeoutTracker.hasTimedOut(); if (isTimedOut) { handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); @@ -429,10 +435,6 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i continue; } - if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { - continue; - } - boolean stopDatasetIteration = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); if (stopDatasetIteration) { String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); From 0587eb1f5b64ba1d004950698c52535088245cb2 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 13 Nov 2024 02:48:35 +0530 Subject: [PATCH 015/241] moved pagination logic before timeouts --- .../atlas/discovery/EntityLineageService.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index c70edb0bde..9872d8103c 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -381,6 +381,11 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i while (incomingEdges.hasNext()) { AtlasEdge incomingEdge = incomingEdges.next(); + + if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { + continue; + } + boolean isTimedOut = timeoutTracker.hasTimedOut(); if (isTimedOut) { handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); @@ -393,10 +398,6 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i continue; } - if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { - continue; - } - boolean stopProcessIteration = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); // If timeout occurred or entity limit reached, set pagination flags again for consistency and stop if (stopProcessIteration) { @@ -417,6 +418,11 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i while (outgoingEdges.hasNext()) { AtlasEdge outgoingEdge = outgoingEdges.next(); + + if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { + continue; + } + isTimedOut = timeoutTracker.hasTimedOut(); if (isTimedOut) { handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); @@ -429,10 +435,6 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i continue; } - if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { - continue; - } - boolean stopDatasetIteration = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); if (stopDatasetIteration) { String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); From d9b46dda4e73e45e67252742d717bec4ac8931b1 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 13 Nov 2024 18:13:47 +0530 Subject: [PATCH 016/241] fixed traversal timeout --- .../org/apache/atlas/AtlasConfiguration.java | 3 +- .../lineage/AtlasLineageOnDemandInfo.java | 10 +- .../AtlasLineageOnDemandContext.java | 9 + .../atlas/discovery/EntityLineageService.java | 195 +++++++++--------- .../atlas/discovery/TimeoutChecker.java | 15 ++ .../atlas/discovery/TimeoutTracker.java | 17 -- 6 files changed, 128 insertions(+), 121 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/discovery/TimeoutChecker.java delete mode 100644 repository/src/main/java/org/apache/atlas/discovery/TimeoutTracker.java diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 7fd081b9e3..25138e409a 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -94,7 +94,8 @@ public enum AtlasConfiguration { GRAPH_TRAVERSAL_PARALLELISM("atlas.graph.traverse.bucket.size",10), LINEAGE_ON_DEMAND_ENABLED("atlas.lineage.on.demand.enabled", true), LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT("atlas.lineage.on.demand.default.node.count", 3), - LINEAGE_MAX_NODE_COUNT("atlas.lineage.max.node.count", 100), + LINEAGE_MAX_NODE_COUNT("atlas.lineage.max.node.count", 50), + LINEAGE_TIMEOUT_MS("atlas.lineage.max.timeout.ms", 15000), SUPPORTED_RELATIONSHIP_EVENTS("atlas.notification.relationships.filter", "asset_readme,asset_links"), diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java index 9ddcfe8f2d..7d2a64c8c1 100644 --- a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java +++ b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java @@ -33,7 +33,7 @@ public class AtlasLineageOnDemandInfo implements Serializable { private LineageOnDemandRequest lineageOnDemandPayload; private boolean upstreamEntityLimitReached; private boolean downstreamEntityLimitReached; - private boolean traversalTimedOut; + private boolean timeoutOccurred; public AtlasLineageOnDemandInfo() { } @@ -136,12 +136,12 @@ public void setDownstreamEntityLimitReached(boolean downstreamEntityLimitReached this.downstreamEntityLimitReached = downstreamEntityLimitReached; } - public boolean isTraversalTimedOut() { - return traversalTimedOut; + public void setTimeoutOccurred(boolean timeoutOccurred) { + this.timeoutOccurred = timeoutOccurred; } - public void setTraversalTimedOut(boolean traversalTimedOut) { - this.traversalTimedOut = traversalTimedOut; + public boolean isTimeoutOccurred() { + return timeoutOccurred; } @Override diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageOnDemandContext.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageOnDemandContext.java index 5509684855..d35a56b568 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageOnDemandContext.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageOnDemandContext.java @@ -23,6 +23,7 @@ public class AtlasLineageOnDemandContext { private Set attributes; private Set relationAttributes; private LineageOnDemandBaseParams defaultParams; + private TimeoutChecker timeoutChecker; public AtlasLineageOnDemandContext(LineageOnDemandRequest lineageOnDemandRequest, AtlasTypeRegistry typeRegistry) { this.constraints = lineageOnDemandRequest.getConstraints(); @@ -81,6 +82,14 @@ public void setDefaultParams(LineageOnDemandBaseParams defaultParams) { this.defaultParams = defaultParams; } + public TimeoutChecker getTimeoutChecker() { + return timeoutChecker; + } + + public void setTimeoutChecker(TimeoutChecker timeoutChecker) { + this.timeoutChecker = timeoutChecker; + } + protected Predicate constructInMemoryPredicate(AtlasTypeRegistry typeRegistry, SearchParameters.FilterCriteria filterCriteria) { LineageSearchProcessor lineageSearchProcessor = new LineageSearchProcessor(); return lineageSearchProcessor.constructInMemoryPredicate(typeRegistry, filterCriteria); diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 9ee18d61a4..9dd862b333 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -81,13 +81,14 @@ @Service public class EntityLineageService implements AtlasLineageService { private static final Logger LOG = LoggerFactory.getLogger(EntityLineageService.class); - private static final long LINEAGE_TRAVERSAL_TIMEOUT_MILLIS = 15000; + private static final String PROCESS_INPUTS_EDGE = "__Process.inputs"; private static final String PROCESS_OUTPUTS_EDGE = "__Process.outputs"; private static final String COLUMNS = "columns"; private static final boolean LINEAGE_USING_GREMLIN = AtlasConfiguration.LINEAGE_USING_GREMLIN.getBoolean(); private static final Integer DEFAULT_LINEAGE_MAX_NODE_COUNT = 9000; private static final int LINEAGE_ON_DEMAND_DEFAULT_DEPTH = 3; + private static final long LINEAGE_TIMEOUT_MS = AtlasConfiguration.LINEAGE_TIMEOUT_MS.getLong(); private static final String SEPARATOR = "->"; private final AtlasGraph graph; @@ -298,13 +299,14 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag AtomicInteger inputEntitiesTraversed = new AtomicInteger(0); AtomicInteger outputEntitiesTraversed = new AtomicInteger(0); AtomicInteger traversalOrder = new AtomicInteger(1); - TimeoutTracker tracker = new TimeoutTracker(LINEAGE_TRAVERSAL_TIMEOUT_MILLIS); + TimeoutChecker timeoutChecker = new TimeoutChecker(LINEAGE_TIMEOUT_MS); + atlasLineageOnDemandContext.setTimeoutChecker(timeoutChecker); if (isDataSet) { AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, true, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed, traversalOrder, tracker); + traverseEdgesOnDemand(datasetVertex, true, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed, traversalOrder); if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed, traversalOrder, tracker); + traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed, traversalOrder); AtlasEntityHeader baseEntityHeader = entityRetriever.toAtlasEntityHeader(datasetVertex, atlasLineageOnDemandContext.getAttributes()); setGraphTraversalMetadata(level, traversalOrder, baseEntityHeader); ret.getGuidEntityMap().put(guid, baseEntityHeader); @@ -313,11 +315,11 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag // make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1' if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_INPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, true, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed, traversalOrder, tracker); + traverseEdgesOnDemand(processEdges, true, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed, traversalOrder); } if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_OUTPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, false, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed, traversalOrder, tracker); + traverseEdgesOnDemand(processEdges, false, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed, traversalOrder); } } RequestContext.get().endMetricRecord(metricRecorder); @@ -330,7 +332,7 @@ private static void setGraphTraversalMetadata(int level, AtomicInteger traversal baseEntityHeader.setFinishTime(traversalOrder.get()); } - private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder, TimeoutTracker timeoutTracker) throws AtlasBaseException { + private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder) throws AtlasBaseException { AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; int nextLevel = isInput ? level - 1: level + 1; @@ -347,7 +349,7 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI } boolean isInputEdge = processEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); - if (handleHorizontalAndVerticalPagination(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, new HashSet<>(), false)) { + if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, new HashSet<>())) { break; } else { addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); @@ -361,47 +363,86 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI ret.getRelationsOnDemand().put(inGuid, new LineageInfoOnDemand(inGuidLineageConstrains)); } - traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, nextLevel, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder, timeoutTracker); + traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, nextLevel, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); } } - private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder, TimeoutTracker timeoutTracker) throws AtlasBaseException { - if (isEntityTraversalLimitReached(entitiesTraversed) || timeoutTracker.hasTimedOut()) + private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder) throws AtlasBaseException { + // Get timeout checker from context or create new one + TimeoutChecker timeoutChecker = atlasLineageOnDemandContext.getTimeoutChecker(); + // Check timeout before starting traversal + if (timeoutChecker.hasTimedOut()) { + handleTimeout(ret); + return; + } + + if (isEntityTraversalLimitReached(entitiesTraversed)) return; if (depth != 0) { // base condition of recursion for depth AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("traverseEdgesOnDemand"); - try { - int nextLevel = isInput ? level - 1 : level + 1; - // keep track of visited vertices to avoid circular loop - visitedVertices.add(getId(datasetVertex)); + AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; + int nextLevel = isInput ? level - 1: level + 1; + // keep track of visited vertices to avoid circular loop + visitedVertices.add(getId(datasetVertex)); + + AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesIn = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesIn"); + Iterator incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); + RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesIn); + + while (incomingEdges.hasNext()) { + // Check timeout periodically + if (timeoutChecker.hasTimedOut()) { + handleTimeout(ret); + return; + } - AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesIn = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesIn"); - Iterator incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); - RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesIn); + AtlasEdge incomingEdge = incomingEdges.next(); + AtlasVertex processVertex = incomingEdge.getOutVertex(); - while (incomingEdges.hasNext()) { - AtlasEdge incomingEdge = incomingEdges.next(); + if (!vertexMatchesEvaluation(processVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(incomingEdge, atlasLineageOnDemandContext)) { + continue; + } - if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { - continue; - } + if (checkForOffset(incomingEdge, datasetVertex, atlasLineageOnDemandContext, ret)) { + continue; + } - boolean isTimedOut = timeoutTracker.hasTimedOut(); - if (isTimedOut) { - handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); - ret.setTraversalTimedOut(true); + if (incrementAndCheckIfRelationsLimitReached(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, visitedVertices)) { + LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(baseGuid); + if (entityOnDemandInfo == null) + continue; + if (isInput ? entityOnDemandInfo.isInputRelationsReachedLimit() : entityOnDemandInfo.isOutputRelationsReachedLimit()) break; + else + continue; + } else { + addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, level, traversalOrder); + } + + AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); + Iterator outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); + RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesOut); + + while (outgoingEdges.hasNext()) { + // Check timeout in inner loop as well + if (timeoutChecker.hasTimedOut()) { + handleTimeout(ret); + return; } - AtlasVertex processVertex = incomingEdge.getOutVertex(); - if (!vertexMatchesEvaluation(processVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(incomingEdge, atlasLineageOnDemandContext)) { + AtlasEdge outgoingEdge = outgoingEdges.next(); + AtlasVertex entityVertex = outgoingEdge.getInVertex(); + + if (!vertexMatchesEvaluation(entityVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(outgoingEdge, atlasLineageOnDemandContext)) { continue; } - boolean stopProcessIteration = handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); - // If timeout occurred or entity limit reached, set pagination flags again for consistency and stop - if (stopProcessIteration) { - LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(baseGuid); + if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { + continue; + } + if (incrementAndCheckIfRelationsLimitReached(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, visitedVertices)) { + String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); + LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(processGuid); if (entityOnDemandInfo == null) continue; if (isInput ? entityOnDemandInfo.isInputRelationsReachedLimit() : entityOnDemandInfo.isOutputRelationsReachedLimit()) @@ -409,65 +450,29 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, level, traversalOrder); + addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); + entitiesTraversed.incrementAndGet(); + traversalOrder.incrementAndGet(); + if (isEntityTraversalLimitReached(entitiesTraversed)) + setEntityLimitReachedFlag(isInput, ret); } - - AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); - Iterator outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); - RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesOut); - - while (outgoingEdges.hasNext()) { - AtlasEdge outgoingEdge = outgoingEdges.next(); - - if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { - continue; - } - - isTimedOut = timeoutTracker.hasTimedOut(); - if (isTimedOut) { - handleHorizontalAndVerticalPagination(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); - ret.setTraversalTimedOut(true); - break; - } - - AtlasVertex entityVertex = outgoingEdge.getInVertex(); - if (!vertexMatchesEvaluation(entityVertex, atlasLineageOnDemandContext) || !edgeMatchesEvaluation(outgoingEdge, atlasLineageOnDemandContext)) { - continue; - } - - boolean stopDatasetIteration = handleHorizontalAndVerticalPagination(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, visitedVertices, isTimedOut); - if (stopDatasetIteration) { - String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); - LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(processGuid); - - if (entityOnDemandInfo == null) - continue; - if (isInput ? entityOnDemandInfo.isInputRelationsReachedLimit() : entityOnDemandInfo.isOutputRelationsReachedLimit()) - break; - else - continue; - } else { - addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); - entitiesTraversed.incrementAndGet(); - traversalOrder.incrementAndGet(); - - if (isEntityTraversalLimitReached(entitiesTraversed)) - setEntityLimitReachedFlag(isInput, ret); - } - if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { - traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder, timeoutTracker); // execute inner depth - AtlasEntityHeader traversedEntity = ret.getGuidEntityMap().get(AtlasGraphUtilsV2.getIdFromVertex(entityVertex)); - if (traversedEntity != null) - traversedEntity.setFinishTime(traversalOrder.get()); - } + if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { + traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); // execute inner depth + AtlasEntityHeader traversedEntity = ret.getGuidEntityMap().get(AtlasGraphUtilsV2.getIdFromVertex(entityVertex)); + if (traversedEntity != null) + traversedEntity.setFinishTime(traversalOrder.get()); } } - } finally { - RequestContext.get().endMetricRecord(metricRecorder); } + + RequestContext.get().endMetricRecord(metricRecorder); } } + private void handleTimeout(AtlasLineageOnDemandInfo ret) { + ret.setTimeoutOccurred(true); + LOG.warn("Lineage traversal timed out after {} ms", LINEAGE_TIMEOUT_MS); + } private static void setEntityLimitReachedFlag(boolean isInput, AtlasLineageOnDemandInfo ret) { if (isInput) @@ -522,7 +527,6 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices, lineageParentsForEntityMap, lineageChildrenForEntityMap); continue; } - if (checkOffsetAndSkipEntity(lineageListContext, ret)) { skippedVertices.add(currentGUID); enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices, lineageParentsForEntityMap, lineageChildrenForEntityMap); @@ -709,7 +713,7 @@ private static String getId(AtlasVertex vertex) { return vertex.getIdForDisplay(); } - private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, Set visitedVertices, boolean isTimedOut) { + private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasLineageOnDemandInfo.LineageDirection direction, Set visitedVertices) { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("incrementAndCheckIfRelationsLimitReached"); AtlasVertex inVertex = isInput ? atlasEdge.getOutVertex() : atlasEdge.getInVertex(); @@ -723,9 +727,9 @@ private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boole LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints); LineageInfoOnDemand outLineageInfo = ret.getRelationsOnDemand().containsKey(outGuid) ? ret.getRelationsOnDemand().get(outGuid) : new LineageInfoOnDemand(outGuidLineageConstraints); - setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, depth, entitiesTraversed, inVertex, outVertex, inLineageInfo, outLineageInfo, visitedVertices, isTimedOut); + setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, inVertex, inGuid, outVertex, outGuid, inLineageInfo, outLineageInfo, visitedVertices); - boolean hasRelationsLimitReached = setVerticalPaginationFlags(inLineageInfo, outLineageInfo, isTimedOut); + boolean hasRelationsLimitReached = setVerticalPaginationFlags(entitiesTraversed, inLineageInfo, outLineageInfo); if (!hasRelationsLimitReached) { ret.getRelationsOnDemand().put(inGuid, inLineageInfo); ret.getRelationsOnDemand().put(outGuid, outLineageInfo); @@ -735,14 +739,9 @@ private boolean handleHorizontalAndVerticalPagination(AtlasEdge atlasEdge, boole return hasRelationsLimitReached; } - private boolean setVerticalPaginationFlags(LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, boolean isTimedOut) { - if (isTimedOut) { - inLineageInfo.setHasMoreInputs(true); - outLineageInfo.setHasMoreOutputs(true); - return true; - } + private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo) { boolean hasRelationsLimitReached = false; - if (inLineageInfo.isInputRelationsReachedLimit() || outLineageInfo.isOutputRelationsReachedLimit()) { + if (inLineageInfo.isInputRelationsReachedLimit() || outLineageInfo.isOutputRelationsReachedLimit() || isEntityTraversalLimitReached(entitiesTraversed)) { inLineageInfo.setHasMoreInputs(true); outLineageInfo.setHasMoreOutputs(true); hasRelationsLimitReached = true; @@ -755,10 +754,10 @@ private boolean setVerticalPaginationFlags(LineageInfoOnDemand inLineageInfo, Li return hasRelationsLimitReached; } - private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, AtlasVertex outVertex, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set visitedVertices, boolean isTimedOut) { + private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, String inGuid, AtlasVertex outVertex, String outGuid, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set visitedVertices) { boolean isOutVertexVisited = visitedVertices.contains(getId(outVertex)); boolean isInVertexVisited = visitedVertices.contains(getId(inVertex)); - if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1 || isTimedOut) { // If traversal has to stop, set pagination flags + if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1) { // is the vertex a leaf? if (isInput && ! isOutVertexVisited) setHasUpstream(atlasLineageOnDemandContext, outVertex, outLineageInfo); else if (!isInput && ! isInVertexVisited) diff --git a/repository/src/main/java/org/apache/atlas/discovery/TimeoutChecker.java b/repository/src/main/java/org/apache/atlas/discovery/TimeoutChecker.java new file mode 100644 index 0000000000..74147d9d1a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/TimeoutChecker.java @@ -0,0 +1,15 @@ +package org.apache.atlas.discovery; + +public class TimeoutChecker { + private final long startTime; + private final long timeoutMs; + + public TimeoutChecker(long timeoutMs) { + this.startTime = System.currentTimeMillis(); + this.timeoutMs = timeoutMs; + } + + public boolean hasTimedOut() { + return System.currentTimeMillis() - startTime > timeoutMs; + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/discovery/TimeoutTracker.java b/repository/src/main/java/org/apache/atlas/discovery/TimeoutTracker.java deleted file mode 100644 index 3f7fce2553..0000000000 --- a/repository/src/main/java/org/apache/atlas/discovery/TimeoutTracker.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.apache.atlas.discovery; - -public class TimeoutTracker { - - private final long startTime; - private final long timeoutMillis; - - public TimeoutTracker(long timeoutMillis) { - this.startTime = System.currentTimeMillis(); - this.timeoutMillis = timeoutMillis; - } - - public boolean hasTimedOut() { - return (System.currentTimeMillis() - startTime) > timeoutMillis; - } - -} From ad3edbc68d2400b0eb60c8a2cf38ff3f5af7bd47 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 13 Nov 2024 18:14:25 +0530 Subject: [PATCH 017/241] fixed traversal timeout --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 25138e409a..4ed55c0ed0 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -94,7 +94,7 @@ public enum AtlasConfiguration { GRAPH_TRAVERSAL_PARALLELISM("atlas.graph.traverse.bucket.size",10), LINEAGE_ON_DEMAND_ENABLED("atlas.lineage.on.demand.enabled", true), LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT("atlas.lineage.on.demand.default.node.count", 3), - LINEAGE_MAX_NODE_COUNT("atlas.lineage.max.node.count", 50), + LINEAGE_MAX_NODE_COUNT("atlas.lineage.max.node.count", 100), LINEAGE_TIMEOUT_MS("atlas.lineage.max.timeout.ms", 15000), SUPPORTED_RELATIONSHIP_EVENTS("atlas.notification.relationships.filter", "asset_readme,asset_links"), From 1df7000731d9f06c699d5e4b8f8400a80cc1dd4c Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 13 Nov 2024 19:17:23 +0530 Subject: [PATCH 018/241] resolved conflicts --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 1 + 1 file changed, 1 insertion(+) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 4ed55c0ed0..6f425c3958 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -114,6 +114,7 @@ public enum AtlasConfiguration { HERACLES_API_SERVER_URL("atlas.heracles.api.service.url", "http://heracles-service.heracles.svc.cluster.local"), INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), + ENABLE_ASYNC_INDEXSEARCH("atlas.indexsearch.async.enable", true), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), From c9e87268ea60b6213469c44cb4c45f37f2c1dc82 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 15 Nov 2024 11:52:43 +0530 Subject: [PATCH 019/241] PLT-2568 use all hosts available for es for connection pooling, node status visibility and retry --- .../repository/graphdb/janus/AtlasElasticsearchDatabase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java index aac96cba48..ab1086ac00 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java @@ -86,7 +86,7 @@ public static RestClient getLowLevelClient() { try { List httpHosts = getHttpHosts(); - RestClientBuilder builder = RestClient.builder(httpHosts.get(0)); + RestClientBuilder builder = RestClient.builder(httpHosts.toArray(new HttpHost[0])); builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setKeepAliveStrategy(((httpResponse, httpContext) -> 3600000))); builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder .setConnectTimeout(AtlasConfiguration.INDEX_CLIENT_CONNECTION_TIMEOUT.getInt()) From e3b229559b35a1cc87037411df1e641c138505f7 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 15 Nov 2024 19:41:45 +0530 Subject: [PATCH 020/241] mesh-280: validation for dataProductAssetDSL attribute --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 1 + .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index e5171fde92..de42d0ae1a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -70,6 +70,7 @@ public class PreProcessorUtils { public static final String INPUT_PORT_GUIDS_ATTR = "daapInputPortGuids"; public static final String DAAP_STATUS_ATTR = "daapStatus"; public static final String DAAP_ARCHIVED_STATUS = "Archived"; + public static final String DAAP_ASSET_DSL_ATTR = "dataProductAssetsDSL"; //Migration Constants public static final String MIGRATION_TYPE_PREFIX = "MIGRATION:"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index d8aee05974..458a5c695f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -83,6 +83,10 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws entity.removeAttribute(OUTPUT_PORT_GUIDS_ATTR); entity.removeAttribute(INPUT_PORT_GUIDS_ATTR); + if(!entity.hasAttribute(DAAP_ASSET_DSL_ATTR) && entity.getAttribute(DAAP_ASSET_DSL_ATTR) == null){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "DataProductAssetDSL attribute is mandatory for DataProduct creation"); + } + if (parentDomainObject == null) { throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Cannot create a Product without a Domain Relationship"); } else { From 305a385dd7c287f26c8a4ee288f6a68e9f8aea0e Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Mon, 18 Nov 2024 13:31:07 +0530 Subject: [PATCH 021/241] PLT-2568 brotli compression for all endpoints --- webapp/pom.xml | 6 ++ .../web/filters/BrotliCompressionFilter.java | 59 +++++++++++++++++++ .../web/filters/BrotliResponseWrapper.java | 54 +++++++++++++++++ webapp/src/main/webapp/WEB-INF/web.xml | 10 ++++ 4 files changed, 129 insertions(+) create mode 100644 webapp/src/main/java/org/apache/atlas/web/filters/BrotliCompressionFilter.java create mode 100644 webapp/src/main/java/org/apache/atlas/web/filters/BrotliResponseWrapper.java diff --git a/webapp/pom.xml b/webapp/pom.xml index 8783276f6d..05a97f200a 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -585,6 +585,12 @@ compile + + com.aayushatharva.brotli4j + brotli4j + 1.17.0 + + diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/BrotliCompressionFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/BrotliCompressionFilter.java new file mode 100644 index 0000000000..f83b0b595e --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/filters/BrotliCompressionFilter.java @@ -0,0 +1,59 @@ +package org.apache.atlas.web.filters; + +import com.aayushatharva.brotli4j.Brotli4jLoader; +import com.aayushatharva.brotli4j.encoder.Encoder; +import com.aayushatharva.brotli4j.encoder.Encoder.Parameters; + +import javax.servlet.*; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +public class BrotliCompressionFilter implements Filter { + + @Override + public void init(FilterConfig filterConfig) { + Brotli4jLoader.ensureAvailability(); + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + + // Ensure request and response are HttpServletRequest and HttpServletResponse + if (request instanceof HttpServletRequest && response instanceof HttpServletResponse) { + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + + String acceptEncoding = httpRequest.getHeader("Accept-Encoding"); + + // Check if the client supports Brotli compression + if (acceptEncoding != null && acceptEncoding.contains("br")) { + // Wrap the response with a Brotli compression wrapper + BrotliResponseWrapper responseWrapper = new BrotliResponseWrapper(httpResponse); + chain.doFilter(request, responseWrapper); + + // Compress the response content with Brotli + byte[] uncompressedData = responseWrapper.getOutputStreamData(); + Parameters params = new Parameters().setQuality(6); // Set Brotli quality level + byte[] compressedOutput = Encoder.compress(uncompressedData, params); + + // Write Brotli-compressed data to the actual response + httpResponse.setHeader("Content-Encoding", "br"); + httpResponse.setContentLength(compressedOutput.length); + httpResponse.getOutputStream().write(compressedOutput); + } else { + // Proceed without compression + chain.doFilter(request, response); + } + } else { + // Proceed without compression if not HTTP + chain.doFilter(request, response); + } + } + + @Override + public void destroy() { + // Optional: Add cleanup logic here if needed + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/BrotliResponseWrapper.java b/webapp/src/main/java/org/apache/atlas/web/filters/BrotliResponseWrapper.java new file mode 100644 index 0000000000..e8cdbb860b --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/filters/BrotliResponseWrapper.java @@ -0,0 +1,54 @@ +package org.apache.atlas.web.filters; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponseWrapper; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; + +public class BrotliResponseWrapper extends HttpServletResponseWrapper { + + private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + private ServletOutputStream servletOutputStream; + private PrintWriter writer; + + public BrotliResponseWrapper(HttpServletResponse response) { + super(response); + } + + @Override + public ServletOutputStream getOutputStream() throws IOException { + if (servletOutputStream == null) { + servletOutputStream = new ServletOutputStream() { + @Override + public void write(int b) { + outputStream.write(b); + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public void setWriteListener(javax.servlet.WriteListener listener) { + // No-op for this example + } + }; + } + return servletOutputStream; + } + + @Override + public PrintWriter getWriter() throws IOException { + if (writer == null) { + writer = new PrintWriter(outputStream); + } + return writer; + } + + public byte[] getOutputStreamData() { + return outputStream.toByteArray(); + } +} diff --git a/webapp/src/main/webapp/WEB-INF/web.xml b/webapp/src/main/webapp/WEB-INF/web.xml index 07092d62eb..efb26298e2 100755 --- a/webapp/src/main/webapp/WEB-INF/web.xml +++ b/webapp/src/main/webapp/WEB-INF/web.xml @@ -114,6 +114,16 @@ /api/atlas/admin/status + + brotliFilter + org.apache.atlas.web.filters.BrotliCompressionFilter + + + + brotliFilter + /* + + org.springframework.web.context.request.RequestContextListener From 22def50c617a3229ebebf0410eccc1805e2ea693 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Wed, 21 Aug 2024 18:22:46 +0530 Subject: [PATCH 022/241] Merge pull request #3395 from atlanhq/ns/DG-1709-refresh-delta-policies DG-1709 | Delta based policies refresh --- .../atlas/plugin/model/RangerPolicyDelta.java | 4 + .../policyengine/RangerPolicyRepository.java | 34 +-- .../plugin/service/RangerBasePlugin.java | 1 + .../atlas/plugin/util/PolicyRefresher.java | 38 ++- .../plugin/util/RangerPolicyDeltaUtil.java | 12 +- .../atlas/plugin/util/ServicePolicies.java | 2 +- .../CachePolicyTransformerImpl.java | 237 ++++++++++++++++-- .../apache/atlas/ApplicationProperties.java | 1 + .../org/apache/atlas/web/rest/AuthREST.java | 40 ++- 9 files changed, 299 insertions(+), 70 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index adfe24af1d..def6c6c0dd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -73,6 +73,9 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol @JsonIgnore public Long getPolicyId() { return policy != null ? policy.getId() : null; } + @JsonIgnore + public String getPolicyGuid() { return policy != null ? policy.getGuid() : null; } + @JsonIgnore public String getZoneName() { return policy != null ? policy.getZoneName() : null; } @@ -94,6 +97,7 @@ public String toString() { + ", serviceType:" + getServiceType() + ", policyType:" + getPolicyType() + ", policyId:[" + getPolicyId() + "]" + + ", policyGuid:[" + getPolicyGuid() + "]" + ", policy:[" + policy +"]"; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java index 5b3b469fb0..4324ccb2cd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java @@ -90,7 +90,7 @@ enum AuditModeEnum { private List dataMaskPolicyEvaluators; private List rowFilterPolicyEvaluators; private final List auditPolicyEvaluators; - private Map policyEvaluatorsMap; + private Map policyEvaluatorsMap; private boolean isContextEnrichersShared = false; private boolean isPreCleaned = false; @@ -654,9 +654,9 @@ public List getLikelyMatchPolicyEvaluators(RangerAccessRe } - public Map getPolicyEvaluatorsMap() { return policyEvaluatorsMap; } + public Map getPolicyEvaluatorsMap() { return policyEvaluatorsMap; } - RangerPolicyEvaluator getPolicyEvaluator(Long id) { + RangerPolicyEvaluator getPolicyEvaluator(String id) { return policyEvaluatorsMap.get(id); } @@ -1252,17 +1252,17 @@ private void removeEvaluatorFromTrie(RangerPolicyEvaluator oldEvaluator, RangerR } } - private Map createPolicyEvaluatorsMap() { - Map tmpPolicyEvaluatorMap = new HashMap<>(); + private Map createPolicyEvaluatorsMap() { + Map tmpPolicyEvaluatorMap = new HashMap<>(); for (RangerPolicyEvaluator evaluator : getPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } for (RangerPolicyEvaluator evaluator : getDataMaskPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } for (RangerPolicyEvaluator evaluator : getRowFilterPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } return tmpPolicyEvaluatorMap; @@ -1294,7 +1294,7 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) { } if (!RangerPolicy.POLICY_TYPE_AUDIT.equals(policy.getPolicyType())) { - policyEvaluatorsMap.put(policy.getId(), ret); + policyEvaluatorsMap.put(policy.getGuid(), ret); } } } @@ -1306,22 +1306,22 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) { return ret; } - private void removePolicy(Long id) { + private void removePolicy(String guid) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerPolicyRepository.removePolicy(" + id +")"); + LOG.debug("==> RangerPolicyRepository.removePolicy(" + guid +")"); } Iterator iterator = policies.iterator(); while (iterator.hasNext()) { - if (id.equals(iterator.next().getId())) { + if (guid.equals(iterator.next().getGuid())) { iterator.remove(); //break; } } - policyEvaluatorsMap.remove(id); + policyEvaluatorsMap.remove(guid); if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerPolicyRepository.removePolicy(" + id +")"); + LOG.debug("<== RangerPolicyRepository.removePolicy(" + guid +")"); } } @@ -1355,13 +1355,13 @@ private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) { } private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) { - + LOG.info("PolicyDelta: RangerPolicyRepository.update is called, policyGuid: "+delta.getPolicyGuid()); if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")"); } Integer changeType = delta.getChangeType(); String policyType = delta.getPolicyType(); - Long policyId = delta.getPolicyId(); + String policyId = delta.getPolicyGuid(); RangerPolicy policy = delta.getPolicy(); @@ -1472,7 +1472,7 @@ private void updateResourceTrie(List deltas) { for (RangerPolicyDelta delta : deltas) { final Integer changeType = delta.getChangeType(); final String serviceType = delta.getServiceType(); - final Long policyId = delta.getPolicyId(); + final String policyId = delta.getPolicyGuid(); final String policyType = delta.getPolicyType(); if (!serviceType.equals(this.serviceDef.getName())) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index b224cccc7e..fb44584e44 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -379,6 +379,7 @@ public void setPolicies(ServicePolicies policies) { RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine; newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies); + //TODO: this looks like a mistake, second arg should be servicePolicies which has the applied delta } if (newPolicyEngine != null) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index aae09a7d26..b18906dde4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -21,8 +21,12 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; +import org.apache.atlas.repository.audit.ESBasedAuditRepository; +import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,6 +44,8 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import static org.apache.atlas.ApplicationProperties.DELTA_BASED_REFRESH; + public class PolicyRefresher extends Thread { private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); @@ -64,6 +70,9 @@ public class PolicyRefresher extends Thread { private long lastActivationTimeInMillis; private boolean policiesSetInPlugin; private boolean serviceDefSetInPlugin; + private Configuration atlasConfig; + private boolean enableDeltaBasedRefresh; + private ESBasedAuditRepository auditRepository; public PolicyRefresher(RangerBasePlugin plugIn) { @@ -104,6 +113,16 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.userStoreProvider = new RangerUserStoreProvider(getServiceType(), appId, getServiceName(), atlasAuthAdminClient, cacheDir, pluginConfig); this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); + try { + this.atlasConfig = ApplicationProperties.get(); + this.auditRepository = new ESBasedAuditRepository(atlasConfig); + this.auditRepository.start(); + this.enableDeltaBasedRefresh = this.atlasConfig.getBoolean(DELTA_BASED_REFRESH, false); + } catch (AtlasException e) { + LOG.error("PolicyDelta: Error while reading atlas configuration", e); + this.enableDeltaBasedRefresh = false; + } + setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); if(LOG.isDebugEnabled()) { @@ -316,13 +335,20 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { - if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null) { RangerRESTUtils restUtils = new RangerRESTUtils(); - CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); - - svcPolicies = transformer.getPolicies(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis); + CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); + if (lastUpdatedTiemInMillis == -1) { + svcPolicies = transformer.getPoliciesAll(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis, null); + } else if (this.enableDeltaBasedRefresh) { + svcPolicies = transformer.getPoliciesDelta(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis); + } else { + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); + } } else { svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index e3cb0f3697..b92ae004d4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -79,7 +79,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List applyDeltas(List policies, List deltas, String compo for (RangerPolicyDelta delta : deltas) { final Integer changeType = delta.getChangeType(); - final Long policyId = delta.getPolicyId(); + final String policyGuid = delta.getPolicyGuid(); if (changeType == null) { isValid = false; @@ -171,7 +167,7 @@ public static boolean isValidDeltas(List deltas, String compo && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { isValid = false; - } else if (policyId == null) { + } else if (policyGuid == null) { isValid = false; } else { final String serviceType = delta.getServiceType(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 547349c8f5..1a4dd81f29 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -390,7 +390,7 @@ static public TagPolicies copyHeader(TagPolicies source, String componentService return ret; } - public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { + public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { ServicePolicies ret = copyHeader(servicePolicies); List oldResourcePolicies = policyEngine.getResourcePolicies(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 4c6b9621fa..f092a488a6 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -18,14 +18,19 @@ package org.apache.atlas.policytransformer; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; +import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; +import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.util.ServicePolicies; import org.apache.atlas.plugin.model.RangerPolicy; import org.apache.atlas.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; @@ -37,6 +42,7 @@ import org.apache.atlas.plugin.model.RangerServiceDef; import org.apache.atlas.plugin.model.RangerValiditySchedule; import org.apache.atlas.plugin.util.ServicePolicies.TagPolicies; +import org.apache.atlas.repository.audit.ESBasedAuditRepository; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -53,20 +59,11 @@ import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.apache.atlas.repository.Constants.NAME; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.SERVICE_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; @@ -116,11 +113,15 @@ public class CachePolicyTransformerImpl { private PurposeCachePolicyTransformer purposeTransformer; private AtlasEntityHeader service; + private final ESBasedAuditRepository auditRepository; + + private final Map auditEventToDeltaChangeType; @Inject - public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry, ESBasedAuditRepository auditRepository) throws AtlasBaseException { this.graph = new AtlasJanusGraph(); this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); + this.auditRepository = auditRepository; personaTransformer = new PersonaCachePolicyTransformer(entityRetriever); purposeTransformer = new PurposeCachePolicyTransformer(entityRetriever); @@ -131,13 +132,81 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa LOG.error("Failed to initialize discoveryService"); throw new AtlasBaseException(e.getCause()); } + + this.auditEventToDeltaChangeType = new HashMap<>(); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPolicies(String serviceName, String pluginId, Long lastUpdatedTime) { + public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Long lastUpdatedTime) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); + + ServicePolicies servicePolicies = new ServicePolicies(); + + try { + servicePolicies.setServiceName(serviceName); + + service = getServiceEntity(serviceName); + servicePolicies.setPolicyVersion(-1L); + servicePolicies.setPolicyUpdateTime(new Date()); + + if (service != null) { + servicePolicies.setServiceName(serviceName); + servicePolicies.setServiceId(service.getGuid()); + + String serviceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, serviceName); + servicePolicies.setServiceDef(getResourceAsObject(serviceDefName, RangerServiceDef.class)); + + List policiesDelta = getServicePoliciesDelta(service, 250, lastUpdatedTime); + servicePolicies.setPolicyDeltas(policiesDelta); + + + //Process tag based policies + String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); + if (StringUtils.isNotEmpty(tagServiceName)) { + AtlasEntityHeader tagService = getServiceEntity(tagServiceName); + + if (tagService != null) { + List tagRangerPolicies = getServicePolicies(tagService, 0, null); + + TagPolicies tagPolicies = new TagPolicies(); + + tagPolicies.setServiceName(tagServiceName); + tagPolicies.setPolicyUpdateTime(new Date()); + tagPolicies.setServiceId(tagService.getGuid()); + tagPolicies.setPolicyVersion(-1L); + + String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); + tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); + + servicePolicies.setTagPolicies(tagPolicies); + servicePolicies.getTagPolicies().setPolicies(tagRangerPolicies); + LOG.info("PolicyDelta: {}: Found tag policies - {}", serviceName, tagRangerPolicies.size()); + } + } + + + + LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); + LOG.info("PolicyDelta: Found and set {} policies as delta and {} tag policies", servicePolicies.getPolicyDeltas().size(), servicePolicies.getTagPolicies().getPolicies().size()); + } + + } catch (Exception e) { + LOG.error("PolicyDelta: {}: ERROR in getPoliciesDelta {}: {}", serviceName, e.getMessage(), e); + return null; + } + + RequestContext.get().endMetricRecord(recorder); + return servicePolicies; + } + + + public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime, Date latestEditTime) { //TODO: return only if updated AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPolicies." + serviceName); @@ -151,7 +220,7 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las servicePolicies.setPolicyUpdateTime(new Date()); if (service != null) { - List allPolicies = getServicePolicies(service, 250); + List allPolicies = getServicePolicies(service, 250, latestEditTime); servicePolicies.setServiceName(serviceName); servicePolicies.setServiceId(service.getGuid()); @@ -165,7 +234,7 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las AtlasEntityHeader tagService = getServiceEntity(tagServiceName); if (tagService != null) { - allPolicies.addAll(getServicePolicies(tagService, 0)); + allPolicies.addAll(getServicePolicies(tagService, 0, latestEditTime)); TagPolicies tagPolicies = new TagPolicies(); @@ -195,7 +264,7 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las } } catch (Exception e) { - LOG.error("ERROR in getPolicies {}: ", e); + LOG.error("ERROR in getPolicies: ", e); return null; } @@ -203,14 +272,30 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las return servicePolicies; } - private List getServicePolicies(AtlasEntityHeader service, int batchSize) throws AtlasBaseException, IOException { + private List getServicePolicies(AtlasEntityHeader service, int batchSize, Date latestEditTime) throws AtlasBaseException, IOException, InterruptedException { List servicePolicies = new ArrayList<>(); + List atlasPolicies = new ArrayList<>(); String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); - List atlasPolicies = getAtlasPolicies(serviceName, batchSize); + int maxAttempts = 5; + int sleepFor = 500; + for (int attempt = 0; attempt <= maxAttempts; attempt++) { + try { + atlasPolicies = getAtlasPolicies(serviceName, batchSize, latestEditTime, new ArrayList<>()); + break; + } catch (AtlasBaseException e) { + LOG.error("ES_SYNC_FIX: {}: ERROR in getServicePolicies: {}", serviceName, e.getMessage()); + TimeUnit.MILLISECONDS.sleep(sleepFor); + if (attempt == maxAttempts) { + throw e; + } + sleepFor *= 2; + } + } + LOG.info("ES_SYNC_FIX: {}: Moving to transform policies, size: {}", serviceName, atlasPolicies.size()); if (CollectionUtils.isNotEmpty(atlasPolicies)) { //transform policies servicePolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -218,6 +303,83 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } + private List getServicePoliciesDelta(AtlasEntityHeader service, int batchSize, Long lastUpdatedTime) throws AtlasBaseException, IOException { + + String serviceName = (String) service.getAttribute("name"); + String serviceType = (String) service.getAttribute("authServiceType"); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getServicePoliciesWithDelta." + serviceName); + + List policyDeltas = new ArrayList<>(); + + // TODO: when getServicePolicies (without delta) is removed, merge the pagination for audit logs and policy fetch into one + List auditEvents = queryPoliciesAuditLogs(serviceName, lastUpdatedTime, batchSize); + Map policiesWithChangeType = new HashMap<>(); + for (EntityAuditEventV2 event : auditEvents) { + if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policiesWithChangeType.containsKey(event.getEntityId())) { + policiesWithChangeType.put(event.getEntityId(), event.getAction()); + } + } + LOG.info("PolicyDelta: {}: Total audit logs found = {}, events for {} ({}) = {}", serviceName, auditEvents.size(), POLICY_ENTITY_TYPE, policiesWithChangeType.size(), policiesWithChangeType); + if (policiesWithChangeType.isEmpty()) { + return policyDeltas; + } + + ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, null, policyGuids); + + List rangerPolicies = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(atlasPolicies)) { + rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); + } + + for (RangerPolicy policy : rangerPolicies) { + Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getGuid())); + RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); + policyDeltas.add(delta); + } + LOG.info("PolicyDelta: {}: atlas policies found = {}, delta created = {}", serviceName, atlasPolicies.size(), policyDeltas.size()); + RequestContext.get().endMetricRecord(recorder); + + return policyDeltas; + } + + private List queryPoliciesAuditLogs(String serviceName, Long afterTime, int batchSize) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.queryPoliciesAuditLogs." + serviceName); + + List entityUpdateToWatch = new ArrayList<>(); + entityUpdateToWatch.add(POLICY_ENTITY_TYPE); + entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); + + AuditSearchParams parameters = new AuditSearchParams(); + Map dsl = getMap("size", batchSize); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); + afterTime = afterTime == -1 ? 0 : afterTime; + mustClauseList.add(getMap("range", getMap("created", getMap("gte", afterTime)))); + + List> sortList = new ArrayList<>(0); + sortList.add(getMap("created", getMap("order", "desc"))); + dsl.put("sort", sortList); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + parameters.setDsl(dsl); + + List events = new ArrayList<>(); + try { + EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + } catch (AtlasBaseException e) { + LOG.error("ERROR in queryPoliciesAuditLogs while fetching entity audits {}: ", e.getMessage(), e); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + return events; + } + private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, String serviceType, String serviceName) throws IOException, AtlasBaseException { @@ -417,13 +579,13 @@ private List getPolicyConditions(AtlasEntityHeader at return null; } - List conditions = (List) atlasPolicy.getAttribute("policyConditions"); + List> conditions = (List>) atlasPolicy.getAttribute("policyConditions"); - for (AtlasStruct condition : conditions) { + for (HashMap condition : conditions) { RangerPolicyItemCondition rangerCondition = new RangerPolicyItemCondition(); - rangerCondition.setType((String) condition.getAttribute("policyConditionType")); - rangerCondition.setValues((List) condition.getAttribute("policyConditionValues")); + rangerCondition.setType((String) condition.get("policyConditionType")); + rangerCondition.setValues((List) condition.get("policyConditionValues")); ret.add(rangerCondition); } @@ -452,7 +614,7 @@ private List getPolicyValiditySchedule(AtlasEntityHeader return ret; } - private List getAtlasPolicies(String serviceName, int batchSize) throws AtlasBaseException { + private List getAtlasPolicies(String serviceName, int batchSize, Date latestEditTime, List policyGuids) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl."+service+".getAtlasPolicies"); List ret = new ArrayList<>(); @@ -481,8 +643,13 @@ private List getAtlasPolicies(String serviceName, int batchSi List> mustClauseList = new ArrayList<>(); mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); + if (!policyGuids.isEmpty()) { + mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); + } + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); List sortList = new ArrayList<>(0); @@ -509,13 +676,33 @@ private List getAtlasPolicies(String serviceName, int batchSi List headers = discoveryService.directIndexSearch(indexSearchParams).getEntities(); if (headers != null) { ret.addAll(headers); + LOG.info("ES_SYNC_FIX: {}: ======= Found result with {} policies", serviceName, headers.size()); } else { found = false; + LOG.info("ES_SYNC_FIX: {}: ======= Found result with null policies", serviceName); } from += size; } while (found && ret.size() % size == 0); + if (Objects.equals(serviceName, "atlas")) { + boolean latestEditFound = false; + Date latestEditTimeAvailable = null; + for (AtlasEntityHeader entity : ret) { + // LOG.info("ES_SYNC_FIX: {}: Looping on returned policies: {}, size: {}", serviceName, entity.getDisplayText(), ret.size()); + if (latestEditTime == null || entity.getUpdateTime().compareTo(latestEditTime) >= 0) { + LOG.info("ES_SYNC_FIX: {}: Found latest policy: {}, latestEditTime: {}, found policy time: {}", serviceName, entity.getDisplayText(), latestEditTime, entity.getUpdateTime()); + latestEditFound = true; + break; + } + latestEditTimeAvailable = entity.getUpdateTime(); + // LOG.info("ES_SYNC_FIX: {}: Checked for latest edit, entity: {}, latestEditTimeAvailable: {}", serviceName, entity.getDisplayText(), latestEditTimeAvailable); + } + if (latestEditTime != null && !latestEditFound) { + LOG.info("ES_SYNC_FIX: {}: Latest edit not found yet, policies: {}, latestEditTime: {}, latestEditTimeAvailable: {}", serviceName, ret.size(), latestEditTime, latestEditTimeAvailable); + throw new AtlasBaseException("Latest edit not found yet"); + } + } } finally { RequestContext.get().endMetricRecord(recorder); diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java index 78b487a8dd..dc198ba2fa 100644 --- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java +++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -70,6 +70,7 @@ public final class ApplicationProperties extends PropertiesConfiguration { public static final boolean DEFAULT_INDEX_RECOVERY = true; public static final AtlasRunMode DEFAULT_ATLAS_RUN_MODE = AtlasRunMode.PROD; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; + public static final String DELTA_BASED_REFRESH = "atlas.authorizer.enable.delta_based_refresh"; public static final SimpleEntry DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true"); public static final SimpleEntry DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20"); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index c5868cbfa3..a8a535cc7c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -21,6 +21,7 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.plugin.util.KeycloakUserStore; @@ -51,10 +52,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; @@ -152,11 +150,12 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); } - if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { + Long latestEditTime = getLastEditTime(serviceName, lastUpdatedTime); + if (latestEditTime == null) { return null; } - ServicePolicies ret = policyTransformer.getPolicies(serviceName, pluginId, lastUpdatedTime); + ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime, new Date(latestEditTime)); updateLastSync(serviceName); @@ -184,7 +183,7 @@ private void updateLastSync(String serviceName) { } } - private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { + private Long getLastEditTime(String serviceName, long lastUpdatedTime) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.isPolicyUpdated." + serviceName); List entityUpdateToWatch = new ArrayList<>(); @@ -199,26 +198,41 @@ private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + List> sortList = new ArrayList<>(); + sortList.add(getMap("created", "desc")); + dsl.put("sort", sortList); + parameters.setDsl(dsl); + Long lastEditTime = 0L; // this timestamp is used to verify if the found policies are synced with any policy create or update op on cassandra try { EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); - - if (result == null || CollectionUtils.isEmpty(result.getEntityAudits())) { - return false; + if (result != null) { + if (!CollectionUtils.isEmpty(result.getEntityAudits())) { + EntityAuditEventV2 lastAuditLog = result.getEntityAudits().get(0); + if (!EntityAuditEventV2.EntityAuditActionV2.getDeleteActions().contains(lastAuditLog.getAction()) && + lastAuditLog.getTypeName().equals(POLICY_ENTITY_TYPE) + ) { + lastEditTime = lastAuditLog.getTimestamp(); + } else { + LOG.info("ES_SYNC_FIX: {}: found delete action, so ignoring the last edit time: {}", serviceName, lastAuditLog.getTimestamp()); + } + } else { + lastEditTime = null; // no edits found + } } } catch (AtlasBaseException e) { LOG.error("ERROR in getPoliciesIfUpdated while fetching entity audits {}: ", e.getMessage()); - return true; } finally { RequestContext.get().endMetricRecord(recorder); + LOG.info("Last edit time for service {} is {}, dsl: {}", serviceName, lastEditTime, dsl); } - return true; + return lastEditTime; } private Map getMap(String key, Object value) { From bfd0386a076485298e3eb9b25500ff7755fd5a15 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 18 Nov 2024 17:02:35 +0530 Subject: [PATCH 023/241] Remove time based policy fetch retry during ES sync --- .../atlas/plugin/util/PolicyRefresher.java | 6 +-- .../CachePolicyTransformerImpl.java | 54 +++---------------- .../org/apache/atlas/web/rest/AuthREST.java | 40 +++++--------- 3 files changed, 23 insertions(+), 77 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b18906dde4..617dc10e7f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -340,12 +340,10 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); if (lastUpdatedTiemInMillis == -1) { svcPolicies = transformer.getPoliciesAll(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis, null); + restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); } else if (this.enableDeltaBasedRefresh) { svcPolicies = transformer.getPoliciesDelta(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis); + restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); } else { svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index f092a488a6..4b04b6856d 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -18,7 +18,6 @@ package org.apache.atlas.policytransformer; -import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; @@ -60,7 +59,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.*; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.*; @@ -172,7 +170,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon AtlasEntityHeader tagService = getServiceEntity(tagServiceName); if (tagService != null) { - List tagRangerPolicies = getServicePolicies(tagService, 0, null); + List tagRangerPolicies = getServicePolicies(tagService, 0); TagPolicies tagPolicies = new TagPolicies(); @@ -206,7 +204,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon } - public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime, Date latestEditTime) { + public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime) { //TODO: return only if updated AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPolicies." + serviceName); @@ -220,7 +218,7 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long servicePolicies.setPolicyUpdateTime(new Date()); if (service != null) { - List allPolicies = getServicePolicies(service, 250, latestEditTime); + List allPolicies = getServicePolicies(service, 250); servicePolicies.setServiceName(serviceName); servicePolicies.setServiceId(service.getGuid()); @@ -234,7 +232,7 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long AtlasEntityHeader tagService = getServiceEntity(tagServiceName); if (tagService != null) { - allPolicies.addAll(getServicePolicies(tagService, 0, latestEditTime)); + allPolicies.addAll(getServicePolicies(tagService, 0)); TagPolicies tagPolicies = new TagPolicies(); @@ -272,30 +270,14 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long return servicePolicies; } - private List getServicePolicies(AtlasEntityHeader service, int batchSize, Date latestEditTime) throws AtlasBaseException, IOException, InterruptedException { + private List getServicePolicies(AtlasEntityHeader service, int batchSize) throws AtlasBaseException, IOException { List servicePolicies = new ArrayList<>(); - List atlasPolicies = new ArrayList<>(); String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, new ArrayList<>()); - int maxAttempts = 5; - int sleepFor = 500; - for (int attempt = 0; attempt <= maxAttempts; attempt++) { - try { - atlasPolicies = getAtlasPolicies(serviceName, batchSize, latestEditTime, new ArrayList<>()); - break; - } catch (AtlasBaseException e) { - LOG.error("ES_SYNC_FIX: {}: ERROR in getServicePolicies: {}", serviceName, e.getMessage()); - TimeUnit.MILLISECONDS.sleep(sleepFor); - if (attempt == maxAttempts) { - throw e; - } - sleepFor *= 2; - } - } - LOG.info("ES_SYNC_FIX: {}: Moving to transform policies, size: {}", serviceName, atlasPolicies.size()); if (CollectionUtils.isNotEmpty(atlasPolicies)) { //transform policies servicePolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -325,7 +307,7 @@ private List getServicePoliciesDelta(AtlasEntityHeader servic } ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); - List atlasPolicies = getAtlasPolicies(serviceName, batchSize, null, policyGuids); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, policyGuids); List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { @@ -614,7 +596,7 @@ private List getPolicyValiditySchedule(AtlasEntityHeader return ret; } - private List getAtlasPolicies(String serviceName, int batchSize, Date latestEditTime, List policyGuids) throws AtlasBaseException { + private List getAtlasPolicies(String serviceName, int batchSize, List policyGuids) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl."+service+".getAtlasPolicies"); List ret = new ArrayList<>(); @@ -676,33 +658,13 @@ private List getAtlasPolicies(String serviceName, int batchSi List headers = discoveryService.directIndexSearch(indexSearchParams).getEntities(); if (headers != null) { ret.addAll(headers); - LOG.info("ES_SYNC_FIX: {}: ======= Found result with {} policies", serviceName, headers.size()); } else { found = false; - LOG.info("ES_SYNC_FIX: {}: ======= Found result with null policies", serviceName); } from += size; } while (found && ret.size() % size == 0); - if (Objects.equals(serviceName, "atlas")) { - boolean latestEditFound = false; - Date latestEditTimeAvailable = null; - for (AtlasEntityHeader entity : ret) { - // LOG.info("ES_SYNC_FIX: {}: Looping on returned policies: {}, size: {}", serviceName, entity.getDisplayText(), ret.size()); - if (latestEditTime == null || entity.getUpdateTime().compareTo(latestEditTime) >= 0) { - LOG.info("ES_SYNC_FIX: {}: Found latest policy: {}, latestEditTime: {}, found policy time: {}", serviceName, entity.getDisplayText(), latestEditTime, entity.getUpdateTime()); - latestEditFound = true; - break; - } - latestEditTimeAvailable = entity.getUpdateTime(); - // LOG.info("ES_SYNC_FIX: {}: Checked for latest edit, entity: {}, latestEditTimeAvailable: {}", serviceName, entity.getDisplayText(), latestEditTimeAvailable); - } - if (latestEditTime != null && !latestEditFound) { - LOG.info("ES_SYNC_FIX: {}: Latest edit not found yet, policies: {}, latestEditTime: {}, latestEditTimeAvailable: {}", serviceName, ret.size(), latestEditTime, latestEditTimeAvailable); - throw new AtlasBaseException("Latest edit not found yet"); - } - } } finally { RequestContext.get().endMetricRecord(recorder); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index a8a535cc7c..bb945ffa97 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -21,7 +21,6 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AuditSearchParams; -import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.plugin.util.KeycloakUserStore; @@ -52,7 +51,10 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; @@ -150,12 +152,11 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); } - Long latestEditTime = getLastEditTime(serviceName, lastUpdatedTime); - if (latestEditTime == null) { + if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; } - ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime, new Date(latestEditTime)); + ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); updateLastSync(serviceName); @@ -183,7 +184,7 @@ private void updateLastSync(String serviceName) { } } - private Long getLastEditTime(String serviceName, long lastUpdatedTime) { + private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.isPolicyUpdated." + serviceName); List entityUpdateToWatch = new ArrayList<>(); @@ -198,41 +199,26 @@ private Long getLastEditTime(String serviceName, long lastUpdatedTime) { mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - List> sortList = new ArrayList<>(); - sortList.add(getMap("created", "desc")); - dsl.put("sort", sortList); - parameters.setDsl(dsl); - Long lastEditTime = 0L; // this timestamp is used to verify if the found policies are synced with any policy create or update op on cassandra try { EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); - if (result != null) { - if (!CollectionUtils.isEmpty(result.getEntityAudits())) { - EntityAuditEventV2 lastAuditLog = result.getEntityAudits().get(0); - if (!EntityAuditEventV2.EntityAuditActionV2.getDeleteActions().contains(lastAuditLog.getAction()) && - lastAuditLog.getTypeName().equals(POLICY_ENTITY_TYPE) - ) { - lastEditTime = lastAuditLog.getTimestamp(); - } else { - LOG.info("ES_SYNC_FIX: {}: found delete action, so ignoring the last edit time: {}", serviceName, lastAuditLog.getTimestamp()); - } - } else { - lastEditTime = null; // no edits found - } + + if (result == null || CollectionUtils.isEmpty(result.getEntityAudits())) { + return false; } } catch (AtlasBaseException e) { LOG.error("ERROR in getPoliciesIfUpdated while fetching entity audits {}: ", e.getMessage()); + return true; } finally { RequestContext.get().endMetricRecord(recorder); - LOG.info("Last edit time for service {} is {}, dsl: {}", serviceName, lastEditTime, dsl); } - return lastEditTime; + return true; } private Map getMap(String key, Object value) { From 34b5496f8e3b40bfa85077ad337b2e948ab04f6e Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 18 Nov 2024 17:05:28 +0530 Subject: [PATCH 024/241] Add missing changes for policy conditons --- .../policytransformer/CachePolicyTransformerImpl.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 4b04b6856d..3a97d0144e 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -29,6 +29,7 @@ import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.util.ServicePolicies; import org.apache.atlas.plugin.model.RangerPolicy; @@ -561,13 +562,13 @@ private List getPolicyConditions(AtlasEntityHeader at return null; } - List> conditions = (List>) atlasPolicy.getAttribute("policyConditions"); + List conditions = (List) atlasPolicy.getAttribute("policyConditions"); - for (HashMap condition : conditions) { + for (AtlasStruct condition : conditions) { RangerPolicyItemCondition rangerCondition = new RangerPolicyItemCondition(); - rangerCondition.setType((String) condition.get("policyConditionType")); - rangerCondition.setValues((List) condition.get("policyConditionValues")); + rangerCondition.setType((String) condition.getAttribute("policyConditionType")); + rangerCondition.setValues((List) condition.getAttribute("policyConditionValues")); ret.add(rangerCondition); } From 064cf1066f87c75f16bbdee1953f4311d4fa73da Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Mon, 18 Nov 2024 17:38:09 +0530 Subject: [PATCH 025/241] PLT-2748 use multiple ES hosts --- .../repository/graphdb/janus/AtlasElasticsearchDatabase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java index aac96cba48..ab1086ac00 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchDatabase.java @@ -86,7 +86,7 @@ public static RestClient getLowLevelClient() { try { List httpHosts = getHttpHosts(); - RestClientBuilder builder = RestClient.builder(httpHosts.get(0)); + RestClientBuilder builder = RestClient.builder(httpHosts.toArray(new HttpHost[0])); builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setKeepAliveStrategy(((httpResponse, httpContext) -> 3600000))); builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder .setConnectTimeout(AtlasConfiguration.INDEX_CLIENT_CONNECTION_TIMEOUT.getInt()) From 44394126fb7bbd4d9fd35b66bcea905a0bcd8cf4 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Wed, 13 Nov 2024 20:18:37 +0530 Subject: [PATCH 026/241] PLT-2568 optimise vertex fetch from janusgraph --- .../discovery/EntityDiscoveryService.java | 156 +++++++++++------- 1 file changed, 97 insertions(+), 59 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 5c66f77e05..5ceafec5af 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -70,6 +70,9 @@ import javax.script.ScriptEngine; import javax.script.ScriptException; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.*; @@ -84,6 +87,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class); private static final String DEFAULT_SORT_ATTRIBUTE_NAME = "name"; + private static final int AVAILABLEPROCESSORS = Runtime.getRuntime().availableProcessors(); + private static final ForkJoinPool CUSTOMTHREADPOOL = new ForkJoinPool(AVAILABLEPROCESSORS/2); // Use half of available cores private final AtlasGraph graph; private final EntityGraphRetriever entityRetriever; @@ -1071,78 +1076,111 @@ public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) thro } } + @SuppressWarnings("rawtypes") private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { SearchParams searchParams = ret.getSearchParameters(); - try { - if(LOG.isDebugEnabled()){ - LOG.debug("Preparing search results for ({})", ret.getSearchParameters()); - } - Iterator iterator = indexQueryResult.getIterator(); - boolean showSearchScore = searchParams.getShowSearchScore(); - if (iterator == null) { - return; - } + boolean showSearchScore = searchParams.getShowSearchScore(); + List results = new ArrayList<>(); - while (iterator.hasNext()) { - Result result = iterator.next(); - AtlasVertex vertex = result.getVertex(); - - if (vertex == null) { - LOG.warn("vertex in null"); - continue; - } + // Collect results for batch processing + Iterator iterator = indexQueryResult.getIterator(); + while (iterator != null && iterator.hasNext()) { + results.add(iterator.next()); + } - AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); - if(RequestContext.get().includeClassifications()){ - header.setClassifications(entityRetriever.getAllClassifications(vertex)); - } - if (showSearchScore) { - ret.addEntityScore(header.getGuid(), result.getScore()); - } - if (fetchCollapsedResults) { - Map collapse = new HashMap<>(); - - Set collapseKeys = result.getCollapseKeys(); - for (String collapseKey : collapseKeys) { - AtlasSearchResult collapseRet = new AtlasSearchResult(); - collapseRet.setSearchParameters(ret.getSearchParameters()); - - Set collapseResultAttributes = new HashSet<>(); - if (searchParams.getCollapseAttributes() != null) { - collapseResultAttributes.addAll(searchParams.getCollapseAttributes()); - } else { - collapseResultAttributes = resultAttributes; + // Batch fetch vertices + List vertices = results.stream() + .map(Result::getVertex) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + // Use ConcurrentHashMap for thread-safe access + ConcurrentHashMap headers = new ConcurrentHashMap<>(); + + // Run vertex processing in limited parallel threads + CompletableFuture.runAsync(() -> CUSTOMTHREADPOOL.submit(() -> + vertices.parallelStream().forEach(vertex -> { + String guid = vertex.getProperty("guid", String.class); + headers.computeIfAbsent(guid, k -> { + try { + AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); + if (RequestContext.get().includeClassifications()) { + header.setClassifications(entityRetriever.getAllClassifications(vertex)); + } + return header; + } catch (AtlasBaseException e) { + throw new RuntimeException(e); } + }); + }) + ).join(), CUSTOMTHREADPOOL); - if (searchParams.getCollapseRelationAttributes() != null) { - RequestContext.get().getRelationAttrsForSearch().clear(); - RequestContext.get().setRelationAttrsForSearch(searchParams.getCollapseRelationAttributes()); - } + // Process results and handle collapse in parallel + results.parallelStream().forEach(result -> { + AtlasVertex vertex = result.getVertex(); + if (vertex == null) return; - DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); - collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); - prepareSearchResult(collapseRet, indexQueryCollapsedResult, collapseResultAttributes, false); + String guid = vertex.getProperty("guid", String.class); + AtlasEntityHeader header = headers.get(guid); - collapseRet.setSearchParameters(null); - collapse.put(collapseKey, collapseRet); - } - if (!collapse.isEmpty()) { - header.setCollapse(collapse); - } + if (showSearchScore) { + ret.addEntityScore(header.getGuid(), result.getScore()); + } + + if (fetchCollapsedResults) { + Map collapse; + try { + collapse = processCollapseResults(result, searchParams, resultAttributes); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); } - if (searchParams.getShowSearchMetadata()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); - ret.addSort(header.getGuid(), result.getSort()); - } else if (searchParams.getShowHighlights()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); + if (!collapse.isEmpty()) { + header.setCollapse(collapse); } + } - ret.addEntity(header); + if (searchParams.getShowSearchMetadata()) { + ret.addHighlights(header.getGuid(), result.getHighLights()); + ret.addSort(header.getGuid(), result.getSort()); + } else if (searchParams.getShowHighlights()) { + ret.addHighlights(header.getGuid(), result.getHighLights()); } - } catch (Exception e) { - throw e; + + ret.addEntity(header); + }); + + if (!searchParams.getEnableFullRestriction()) { + scrubSearchResults(ret, searchParams.getSuppressLogs()); } - scrubSearchResults(ret, searchParams.getSuppressLogs()); + } + + // Non-recursive collapse processing + private Map processCollapseResults(Result result, SearchParams searchParams, Set resultAttributes) throws AtlasBaseException { + Map collapse = new HashMap<>(); + Set collapseKeys = result.getCollapseKeys(); + + for (String collapseKey : collapseKeys) { + AtlasSearchResult collapseRet = new AtlasSearchResult(); + collapseRet.setSearchParameters(searchParams); + Set collapseResultAttributes = new HashSet<>(Optional.ofNullable(searchParams.getCollapseAttributes()).orElse(resultAttributes)); + DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); + collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); + + // Directly iterate over collapse vertices + Iterator iterator = indexQueryCollapsedResult.getIterator(); + while (iterator != null && iterator.hasNext()) { + Result collapseResult = iterator.next(); + AtlasVertex collapseVertex = collapseResult.getVertex(); + if (collapseVertex == null) continue; + + AtlasEntityHeader collapseHeader = entityRetriever.toAtlasEntityHeader(collapseVertex, collapseResultAttributes); + collapseRet.addEntity(collapseHeader); + } + + collapse.put(collapseKey, collapseRet); + } + + return collapse; } private Map getMap(String key, Object value) { From 4fc2f43e2a99b30f6231480d729664968fe3ca2f Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 16:22:44 +0530 Subject: [PATCH 027/241] Fix property name --- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 5ceafec5af..9303ab9170 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1100,7 +1100,7 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i // Run vertex processing in limited parallel threads CompletableFuture.runAsync(() -> CUSTOMTHREADPOOL.submit(() -> vertices.parallelStream().forEach(vertex -> { - String guid = vertex.getProperty("guid", String.class); + String guid = vertex.getProperty("__guid", String.class); headers.computeIfAbsent(guid, k -> { try { AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); @@ -1120,7 +1120,7 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i AtlasVertex vertex = result.getVertex(); if (vertex == null) return; - String guid = vertex.getProperty("guid", String.class); + String guid = vertex.getProperty("__guid", String.class); AtlasEntityHeader header = headers.get(guid); if (showSearchScore) { From a90209633b7abc044112e14cfe75489ea3a31914 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 16:23:54 +0530 Subject: [PATCH 028/241] add null cehck --- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 9303ab9170..c5cc8019dc 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1146,7 +1146,9 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i ret.addHighlights(header.getGuid(), result.getHighLights()); } - ret.addEntity(header); + if (header != null) { + ret.addEntity(header); + } }); if (!searchParams.getEnableFullRestriction()) { From f29738a35b5d79400b2430ff327ebde363be85ee Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 16:31:50 +0530 Subject: [PATCH 029/241] remove submit call --- .../discovery/EntityDiscoveryService.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index c5cc8019dc..f7a76485c8 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1098,22 +1098,20 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i ConcurrentHashMap headers = new ConcurrentHashMap<>(); // Run vertex processing in limited parallel threads - CompletableFuture.runAsync(() -> CUSTOMTHREADPOOL.submit(() -> - vertices.parallelStream().forEach(vertex -> { - String guid = vertex.getProperty("__guid", String.class); - headers.computeIfAbsent(guid, k -> { - try { - AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); - if (RequestContext.get().includeClassifications()) { - header.setClassifications(entityRetriever.getAllClassifications(vertex)); - } - return header; - } catch (AtlasBaseException e) { - throw new RuntimeException(e); - } - }); - }) - ).join(), CUSTOMTHREADPOOL); + CompletableFuture.runAsync(() -> vertices.parallelStream().forEach(vertex -> { + String guid = vertex.getProperty("__guid", String.class); + headers.computeIfAbsent(guid, k -> { + try { + AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); + if (RequestContext.get().includeClassifications()) { + header.setClassifications(entityRetriever.getAllClassifications(vertex)); + } + return header; + } catch (AtlasBaseException e) { + throw new RuntimeException("Failed to process vertex with GUID: " + guid, e); + } + }); + }), CUSTOMTHREADPOOL).join(); // Process results and handle collapse in parallel results.parallelStream().forEach(result -> { From 59bee7e843e5f802a6e5d2ec53fed068f8218334 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 17:36:05 +0530 Subject: [PATCH 030/241] handle cocurrent modification exception --- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index f7a76485c8..1622cd3ea4 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1096,6 +1096,7 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i // Use ConcurrentHashMap for thread-safe access ConcurrentHashMap headers = new ConcurrentHashMap<>(); + ConcurrentHashMap entitiesSet = new ConcurrentHashMap<>(); // Run vertex processing in limited parallel threads CompletableFuture.runAsync(() -> vertices.parallelStream().forEach(vertex -> { @@ -1145,10 +1146,11 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i } if (header != null) { - ret.addEntity(header); + entitiesSet.put(header.getGuid(), header); } }); + ret.setEntities(new ArrayList<>(entitiesSet.values())); if (!searchParams.getEnableFullRestriction()) { scrubSearchResults(ret, searchParams.getSuppressLogs()); } From 7bb10377c256ea51de5e5bd644df3a40dcc4cccc Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 18:27:06 +0530 Subject: [PATCH 031/241] update libraries --- .../apache/atlas/model/discovery/AtlasSearchResult.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java index 96bc0dc587..963e850636 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java @@ -31,13 +31,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.LinkedHashMap; +import java.util.*; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; From 15cf5f6c82a26445d01962498d05c9aeae14df12 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 18:41:52 +0530 Subject: [PATCH 032/241] remove scrub results --- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 1622cd3ea4..7e9cbb9890 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1149,11 +1149,6 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i entitiesSet.put(header.getGuid(), header); } }); - - ret.setEntities(new ArrayList<>(entitiesSet.values())); - if (!searchParams.getEnableFullRestriction()) { - scrubSearchResults(ret, searchParams.getSuppressLogs()); - } } // Non-recursive collapse processing From 6f5c1cf97b28b7240402a431c89021f370949878 Mon Sep 17 00:00:00 2001 From: aarshi0301 <151714378+aarshi0301@users.noreply.github.com> Date: Mon, 18 Nov 2024 20:05:10 +0530 Subject: [PATCH 033/241] Revert "NOC-87266 | Janusgraph optimisation" --- .../model/discovery/AtlasSearchResult.java | 8 +- .../discovery/EntityDiscoveryService.java | 151 +++++++----------- 2 files changed, 65 insertions(+), 94 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java index 963e850636..96bc0dc587 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java @@ -31,7 +31,13 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.LinkedHashMap; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 7e9cbb9890..5c66f77e05 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -70,9 +70,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptException; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.*; @@ -87,8 +84,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class); private static final String DEFAULT_SORT_ATTRIBUTE_NAME = "name"; - private static final int AVAILABLEPROCESSORS = Runtime.getRuntime().availableProcessors(); - private static final ForkJoinPool CUSTOMTHREADPOOL = new ForkJoinPool(AVAILABLEPROCESSORS/2); // Use half of available cores private final AtlasGraph graph; private final EntityGraphRetriever entityRetriever; @@ -1076,108 +1071,78 @@ public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) thro } } - @SuppressWarnings("rawtypes") private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { SearchParams searchParams = ret.getSearchParameters(); - boolean showSearchScore = searchParams.getShowSearchScore(); - List results = new ArrayList<>(); + try { + if(LOG.isDebugEnabled()){ + LOG.debug("Preparing search results for ({})", ret.getSearchParameters()); + } + Iterator iterator = indexQueryResult.getIterator(); + boolean showSearchScore = searchParams.getShowSearchScore(); + if (iterator == null) { + return; + } - // Collect results for batch processing - Iterator iterator = indexQueryResult.getIterator(); - while (iterator != null && iterator.hasNext()) { - results.add(iterator.next()); - } + while (iterator.hasNext()) { + Result result = iterator.next(); + AtlasVertex vertex = result.getVertex(); - // Batch fetch vertices - List vertices = results.stream() - .map(Result::getVertex) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - // Use ConcurrentHashMap for thread-safe access - ConcurrentHashMap headers = new ConcurrentHashMap<>(); - ConcurrentHashMap entitiesSet = new ConcurrentHashMap<>(); - - // Run vertex processing in limited parallel threads - CompletableFuture.runAsync(() -> vertices.parallelStream().forEach(vertex -> { - String guid = vertex.getProperty("__guid", String.class); - headers.computeIfAbsent(guid, k -> { - try { - AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); - if (RequestContext.get().includeClassifications()) { - header.setClassifications(entityRetriever.getAllClassifications(vertex)); - } - return header; - } catch (AtlasBaseException e) { - throw new RuntimeException("Failed to process vertex with GUID: " + guid, e); + if (vertex == null) { + LOG.warn("vertex in null"); + continue; } - }); - }), CUSTOMTHREADPOOL).join(); - // Process results and handle collapse in parallel - results.parallelStream().forEach(result -> { - AtlasVertex vertex = result.getVertex(); - if (vertex == null) return; + AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); + if(RequestContext.get().includeClassifications()){ + header.setClassifications(entityRetriever.getAllClassifications(vertex)); + } + if (showSearchScore) { + ret.addEntityScore(header.getGuid(), result.getScore()); + } + if (fetchCollapsedResults) { + Map collapse = new HashMap<>(); + + Set collapseKeys = result.getCollapseKeys(); + for (String collapseKey : collapseKeys) { + AtlasSearchResult collapseRet = new AtlasSearchResult(); + collapseRet.setSearchParameters(ret.getSearchParameters()); + + Set collapseResultAttributes = new HashSet<>(); + if (searchParams.getCollapseAttributes() != null) { + collapseResultAttributes.addAll(searchParams.getCollapseAttributes()); + } else { + collapseResultAttributes = resultAttributes; + } - String guid = vertex.getProperty("__guid", String.class); - AtlasEntityHeader header = headers.get(guid); + if (searchParams.getCollapseRelationAttributes() != null) { + RequestContext.get().getRelationAttrsForSearch().clear(); + RequestContext.get().setRelationAttrsForSearch(searchParams.getCollapseRelationAttributes()); + } - if (showSearchScore) { - ret.addEntityScore(header.getGuid(), result.getScore()); - } + DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); + collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); + prepareSearchResult(collapseRet, indexQueryCollapsedResult, collapseResultAttributes, false); - if (fetchCollapsedResults) { - Map collapse; - try { - collapse = processCollapseResults(result, searchParams, resultAttributes); - } catch (AtlasBaseException e) { - throw new RuntimeException(e); + collapseRet.setSearchParameters(null); + collapse.put(collapseKey, collapseRet); + } + if (!collapse.isEmpty()) { + header.setCollapse(collapse); + } } - if (!collapse.isEmpty()) { - header.setCollapse(collapse); + if (searchParams.getShowSearchMetadata()) { + ret.addHighlights(header.getGuid(), result.getHighLights()); + ret.addSort(header.getGuid(), result.getSort()); + } else if (searchParams.getShowHighlights()) { + ret.addHighlights(header.getGuid(), result.getHighLights()); } - } - if (searchParams.getShowSearchMetadata()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); - ret.addSort(header.getGuid(), result.getSort()); - } else if (searchParams.getShowHighlights()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); + ret.addEntity(header); } - - if (header != null) { - entitiesSet.put(header.getGuid(), header); - } - }); - } - - // Non-recursive collapse processing - private Map processCollapseResults(Result result, SearchParams searchParams, Set resultAttributes) throws AtlasBaseException { - Map collapse = new HashMap<>(); - Set collapseKeys = result.getCollapseKeys(); - - for (String collapseKey : collapseKeys) { - AtlasSearchResult collapseRet = new AtlasSearchResult(); - collapseRet.setSearchParameters(searchParams); - Set collapseResultAttributes = new HashSet<>(Optional.ofNullable(searchParams.getCollapseAttributes()).orElse(resultAttributes)); - DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); - collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); - - // Directly iterate over collapse vertices - Iterator iterator = indexQueryCollapsedResult.getIterator(); - while (iterator != null && iterator.hasNext()) { - Result collapseResult = iterator.next(); - AtlasVertex collapseVertex = collapseResult.getVertex(); - if (collapseVertex == null) continue; - - AtlasEntityHeader collapseHeader = entityRetriever.toAtlasEntityHeader(collapseVertex, collapseResultAttributes); - collapseRet.addEntity(collapseHeader); - } - - collapse.put(collapseKey, collapseRet); + } catch (Exception e) { + throw e; } - - return collapse; + scrubSearchResults(ret, searchParams.getSuppressLogs()); } private Map getMap(String key, Object value) { From d20be85bc0b435c909dcf99235372cdfa7e7f689 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 18 Nov 2024 23:13:13 +0530 Subject: [PATCH 034/241] feat: add client origin tag in prometheus --- .../apache/atlas/service/metrics/MetricsRegistry.java | 2 +- .../service/metrics/MetricsRegistryServiceImpl.java | 5 +++-- .../main/java/org/apache/atlas/RequestContext.java | 11 ++++++++++- .../org/apache/atlas/web/filters/AuditFilter.java | 2 ++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java index 8974155487..60c1553f9c 100644 --- a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java @@ -8,7 +8,7 @@ public interface MetricsRegistry { - void collect(String requestId, String requestUri, AtlasPerfMetrics metrics); + void collect(String requestId, String requestUri, AtlasPerfMetrics metrics, String clientOrigin); void collectApplicationMetrics(String requestId, String requestUri, List applicationMetrics); diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java index 19171325e2..e11fc5725f 100644 --- a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java @@ -30,6 +30,7 @@ public class MetricsRegistryServiceImpl implements MetricsRegistry { private static final String NAME = "name"; private static final String URI = "uri"; + private static final String ORIGIN = "origin"; private static final String METHOD_DIST_SUMMARY = "method_dist_summary"; private static final String APPLICATION_LEVEL_METRICS_SUMMARY = "application_level_metrics_summary"; private static final double[] PERCENTILES = {0.99}; @@ -43,7 +44,7 @@ public MetricsRegistryServiceImpl() throws AtlasException { } @Override - public void collect(String requestId, String requestUri, AtlasPerfMetrics metrics) { + public void collect(String requestId, String requestUri, AtlasPerfMetrics metrics, String clientOrigin) { try { if (!ApplicationProperties.get().getBoolean(METHOD_LEVEL_METRICS_ENABLE, false)) { return; @@ -52,7 +53,7 @@ public void collect(String requestId, String requestUri, AtlasPerfMetrics metric for (String name : this.filteredMethods) { if(metrics.hasMetric(name)) { AtlasPerfMetrics.Metric metric = metrics.getMetric(name); - Timer.builder(METHOD_DIST_SUMMARY).tags(Tags.of(NAME, metric.getName(), URI, requestUri)).publishPercentiles(PERCENTILES) + Timer.builder(METHOD_DIST_SUMMARY).tags(Tags.of(NAME, metric.getName(), URI, requestUri, ORIGIN, clientOrigin)).publishPercentiles(PERCENTILES) .register(getMeterRegistry()).record(metric.getTotalTimeMSecs(), TimeUnit.MILLISECONDS); } } diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 814a4cb450..a2dad1b68d 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -80,6 +80,7 @@ public class RequestContext { private Set userGroups; private String clientIPAddress; private List forwardedAddresses; + private String clientOrigin; private DeleteType deleteType = DeleteType.DEFAULT; private boolean isPurgeRequested = false; private int maxAttempts = 1; @@ -178,7 +179,7 @@ public void clearCache() { if (metrics != null && !metrics.isEmpty()) { METRICS.debug(metrics.toString()); if (Objects.nonNull(this.metricsRegistry)){ - this.metricsRegistry.collect(traceId, this.requestUri, metrics); + this.metricsRegistry.collect(traceId, this.requestUri, metrics, this.getClientOrigin()); } metrics.clear(); } @@ -744,6 +745,14 @@ public void setIncludeClassificationNames(boolean includeClassificationNames) { this.includeClassificationNames = includeClassificationNames; } + public String getClientOrigin() { + return clientOrigin; + } + + public void setClientOrigin(String clientOrigin) { + this.clientOrigin = clientOrigin; + } + public class EntityGuidPair { private final Object entity; private final String guid; diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java index d3cb21b94d..53a8213be9 100755 --- a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java @@ -90,6 +90,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha final Set userGroups = AtlasAuthorizationUtils.getCurrentUserGroups(); final String deleteType = httpRequest.getParameter("deleteType"); final boolean skipFailedEntities = Boolean.parseBoolean(httpRequest.getParameter("skipFailedEntities")); + final String clientOrigin = httpRequest.getHeader("X-Atlan-Client-Origin"); try { currentThread.setName(formatName(oldName, internalRequestId)); @@ -103,6 +104,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha requestContext.setCreateShellEntityForNonExistingReference(createShellEntityForNonExistingReference); requestContext.setForwardedAddresses(AtlasAuthorizationUtils.getForwardedAddressesFromRequest(httpRequest)); requestContext.setSkipFailedEntities(skipFailedEntities); + requestContext.setClientOrigin(httpRequest.getHeader("X-Atlan-Client-Origin")); requestContext.setMetricRegistry(metricsRegistry); MDC.put(TRACE_ID, internalRequestId); MDC.put(X_ATLAN_REQUEST_ID, ofNullable(httpRequest.getHeader(X_ATLAN_REQUEST_ID)).orElse(EMPTY)); From 364c4aa20edbcf59a57a47d5c7803029519ac633 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Wed, 13 Nov 2024 20:18:37 +0530 Subject: [PATCH 035/241] PLT-2568 optimise vertex fetch from janusgraph --- .../discovery/EntityDiscoveryService.java | 156 +++++++++++------- 1 file changed, 97 insertions(+), 59 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 5c66f77e05..5ceafec5af 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -70,6 +70,9 @@ import javax.script.ScriptEngine; import javax.script.ScriptException; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.*; @@ -84,6 +87,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class); private static final String DEFAULT_SORT_ATTRIBUTE_NAME = "name"; + private static final int AVAILABLEPROCESSORS = Runtime.getRuntime().availableProcessors(); + private static final ForkJoinPool CUSTOMTHREADPOOL = new ForkJoinPool(AVAILABLEPROCESSORS/2); // Use half of available cores private final AtlasGraph graph; private final EntityGraphRetriever entityRetriever; @@ -1071,78 +1076,111 @@ public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) thro } } + @SuppressWarnings("rawtypes") private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { SearchParams searchParams = ret.getSearchParameters(); - try { - if(LOG.isDebugEnabled()){ - LOG.debug("Preparing search results for ({})", ret.getSearchParameters()); - } - Iterator iterator = indexQueryResult.getIterator(); - boolean showSearchScore = searchParams.getShowSearchScore(); - if (iterator == null) { - return; - } + boolean showSearchScore = searchParams.getShowSearchScore(); + List results = new ArrayList<>(); - while (iterator.hasNext()) { - Result result = iterator.next(); - AtlasVertex vertex = result.getVertex(); - - if (vertex == null) { - LOG.warn("vertex in null"); - continue; - } + // Collect results for batch processing + Iterator iterator = indexQueryResult.getIterator(); + while (iterator != null && iterator.hasNext()) { + results.add(iterator.next()); + } - AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); - if(RequestContext.get().includeClassifications()){ - header.setClassifications(entityRetriever.getAllClassifications(vertex)); - } - if (showSearchScore) { - ret.addEntityScore(header.getGuid(), result.getScore()); - } - if (fetchCollapsedResults) { - Map collapse = new HashMap<>(); - - Set collapseKeys = result.getCollapseKeys(); - for (String collapseKey : collapseKeys) { - AtlasSearchResult collapseRet = new AtlasSearchResult(); - collapseRet.setSearchParameters(ret.getSearchParameters()); - - Set collapseResultAttributes = new HashSet<>(); - if (searchParams.getCollapseAttributes() != null) { - collapseResultAttributes.addAll(searchParams.getCollapseAttributes()); - } else { - collapseResultAttributes = resultAttributes; + // Batch fetch vertices + List vertices = results.stream() + .map(Result::getVertex) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + // Use ConcurrentHashMap for thread-safe access + ConcurrentHashMap headers = new ConcurrentHashMap<>(); + + // Run vertex processing in limited parallel threads + CompletableFuture.runAsync(() -> CUSTOMTHREADPOOL.submit(() -> + vertices.parallelStream().forEach(vertex -> { + String guid = vertex.getProperty("guid", String.class); + headers.computeIfAbsent(guid, k -> { + try { + AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); + if (RequestContext.get().includeClassifications()) { + header.setClassifications(entityRetriever.getAllClassifications(vertex)); + } + return header; + } catch (AtlasBaseException e) { + throw new RuntimeException(e); } + }); + }) + ).join(), CUSTOMTHREADPOOL); - if (searchParams.getCollapseRelationAttributes() != null) { - RequestContext.get().getRelationAttrsForSearch().clear(); - RequestContext.get().setRelationAttrsForSearch(searchParams.getCollapseRelationAttributes()); - } + // Process results and handle collapse in parallel + results.parallelStream().forEach(result -> { + AtlasVertex vertex = result.getVertex(); + if (vertex == null) return; - DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); - collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); - prepareSearchResult(collapseRet, indexQueryCollapsedResult, collapseResultAttributes, false); + String guid = vertex.getProperty("guid", String.class); + AtlasEntityHeader header = headers.get(guid); - collapseRet.setSearchParameters(null); - collapse.put(collapseKey, collapseRet); - } - if (!collapse.isEmpty()) { - header.setCollapse(collapse); - } + if (showSearchScore) { + ret.addEntityScore(header.getGuid(), result.getScore()); + } + + if (fetchCollapsedResults) { + Map collapse; + try { + collapse = processCollapseResults(result, searchParams, resultAttributes); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); } - if (searchParams.getShowSearchMetadata()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); - ret.addSort(header.getGuid(), result.getSort()); - } else if (searchParams.getShowHighlights()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); + if (!collapse.isEmpty()) { + header.setCollapse(collapse); } + } - ret.addEntity(header); + if (searchParams.getShowSearchMetadata()) { + ret.addHighlights(header.getGuid(), result.getHighLights()); + ret.addSort(header.getGuid(), result.getSort()); + } else if (searchParams.getShowHighlights()) { + ret.addHighlights(header.getGuid(), result.getHighLights()); } - } catch (Exception e) { - throw e; + + ret.addEntity(header); + }); + + if (!searchParams.getEnableFullRestriction()) { + scrubSearchResults(ret, searchParams.getSuppressLogs()); } - scrubSearchResults(ret, searchParams.getSuppressLogs()); + } + + // Non-recursive collapse processing + private Map processCollapseResults(Result result, SearchParams searchParams, Set resultAttributes) throws AtlasBaseException { + Map collapse = new HashMap<>(); + Set collapseKeys = result.getCollapseKeys(); + + for (String collapseKey : collapseKeys) { + AtlasSearchResult collapseRet = new AtlasSearchResult(); + collapseRet.setSearchParameters(searchParams); + Set collapseResultAttributes = new HashSet<>(Optional.ofNullable(searchParams.getCollapseAttributes()).orElse(resultAttributes)); + DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); + collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); + + // Directly iterate over collapse vertices + Iterator iterator = indexQueryCollapsedResult.getIterator(); + while (iterator != null && iterator.hasNext()) { + Result collapseResult = iterator.next(); + AtlasVertex collapseVertex = collapseResult.getVertex(); + if (collapseVertex == null) continue; + + AtlasEntityHeader collapseHeader = entityRetriever.toAtlasEntityHeader(collapseVertex, collapseResultAttributes); + collapseRet.addEntity(collapseHeader); + } + + collapse.put(collapseKey, collapseRet); + } + + return collapse; } private Map getMap(String key, Object value) { From 8c7ca544b2e93e9832b56db07037c97692920816 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 08:16:33 +0530 Subject: [PATCH 036/241] remove method --- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 5ceafec5af..6d0374c7ea 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1148,10 +1148,6 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i ret.addEntity(header); }); - - if (!searchParams.getEnableFullRestriction()) { - scrubSearchResults(ret, searchParams.getSuppressLogs()); - } } // Non-recursive collapse processing From 3df42f5c1ec76811c94e30112986940dedacc845 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 16:22:44 +0530 Subject: [PATCH 037/241] Fix property name --- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 6d0374c7ea..93d89a4640 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1100,7 +1100,7 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i // Run vertex processing in limited parallel threads CompletableFuture.runAsync(() -> CUSTOMTHREADPOOL.submit(() -> vertices.parallelStream().forEach(vertex -> { - String guid = vertex.getProperty("guid", String.class); + String guid = vertex.getProperty("__guid", String.class); headers.computeIfAbsent(guid, k -> { try { AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); @@ -1120,7 +1120,7 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i AtlasVertex vertex = result.getVertex(); if (vertex == null) return; - String guid = vertex.getProperty("guid", String.class); + String guid = vertex.getProperty("__guid", String.class); AtlasEntityHeader header = headers.get(guid); if (showSearchScore) { From 0bb62fca5c9305bb4e7583eac8dbea2efd2255be Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 16:23:54 +0530 Subject: [PATCH 038/241] add null cehck --- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 93d89a4640..94ff39616a 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1146,7 +1146,9 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i ret.addHighlights(header.getGuid(), result.getHighLights()); } - ret.addEntity(header); + if (header != null) { + ret.addEntity(header); + } }); } From 9b966a61859a6a861c0d75c4f007cc48e644f0ae Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 16:31:50 +0530 Subject: [PATCH 039/241] remove submit call --- .../discovery/EntityDiscoveryService.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 94ff39616a..669dc3ea3a 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1098,22 +1098,20 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i ConcurrentHashMap headers = new ConcurrentHashMap<>(); // Run vertex processing in limited parallel threads - CompletableFuture.runAsync(() -> CUSTOMTHREADPOOL.submit(() -> - vertices.parallelStream().forEach(vertex -> { - String guid = vertex.getProperty("__guid", String.class); - headers.computeIfAbsent(guid, k -> { - try { - AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); - if (RequestContext.get().includeClassifications()) { - header.setClassifications(entityRetriever.getAllClassifications(vertex)); - } - return header; - } catch (AtlasBaseException e) { - throw new RuntimeException(e); - } - }); - }) - ).join(), CUSTOMTHREADPOOL); + CompletableFuture.runAsync(() -> vertices.parallelStream().forEach(vertex -> { + String guid = vertex.getProperty("__guid", String.class); + headers.computeIfAbsent(guid, k -> { + try { + AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); + if (RequestContext.get().includeClassifications()) { + header.setClassifications(entityRetriever.getAllClassifications(vertex)); + } + return header; + } catch (AtlasBaseException e) { + throw new RuntimeException("Failed to process vertex with GUID: " + guid, e); + } + }); + }), CUSTOMTHREADPOOL).join(); // Process results and handle collapse in parallel results.parallelStream().forEach(result -> { From b0530b36b73774e3ff566112faa3e4a262aa6237 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 17:36:05 +0530 Subject: [PATCH 040/241] handle cocurrent modification exception --- .../apache/atlas/discovery/EntityDiscoveryService.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 669dc3ea3a..1622cd3ea4 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1096,6 +1096,7 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i // Use ConcurrentHashMap for thread-safe access ConcurrentHashMap headers = new ConcurrentHashMap<>(); + ConcurrentHashMap entitiesSet = new ConcurrentHashMap<>(); // Run vertex processing in limited parallel threads CompletableFuture.runAsync(() -> vertices.parallelStream().forEach(vertex -> { @@ -1145,9 +1146,14 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i } if (header != null) { - ret.addEntity(header); + entitiesSet.put(header.getGuid(), header); } }); + + ret.setEntities(new ArrayList<>(entitiesSet.values())); + if (!searchParams.getEnableFullRestriction()) { + scrubSearchResults(ret, searchParams.getSuppressLogs()); + } } // Non-recursive collapse processing From 52e645dbfbde073e0c6c1532b992203eb915c56f Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 18:27:06 +0530 Subject: [PATCH 041/241] update libraries --- .../apache/atlas/model/discovery/AtlasSearchResult.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java index 96bc0dc587..963e850636 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java @@ -31,13 +31,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.LinkedHashMap; +import java.util.*; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; From c323ad700f4b258015b07867c9ad43999fe575cc Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 18 Nov 2024 18:41:52 +0530 Subject: [PATCH 042/241] remove scrub results --- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 1622cd3ea4..7e9cbb9890 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1149,11 +1149,6 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i entitiesSet.put(header.getGuid(), header); } }); - - ret.setEntities(new ArrayList<>(entitiesSet.values())); - if (!searchParams.getEnableFullRestriction()) { - scrubSearchResults(ret, searchParams.getSuppressLogs()); - } } // Non-recursive collapse processing From 9741f59812ef42cb5ee7fc0928e5f52a765b1229 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 09:14:25 +0530 Subject: [PATCH 043/241] fix results --- .../java/org/apache/atlas/discovery/EntityDiscoveryService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 7e9cbb9890..9f2fd920ef 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1149,6 +1149,8 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i entitiesSet.put(header.getGuid(), header); } }); + + ret.setEntities(new ArrayList<>(entitiesSet.values())); } // Non-recursive collapse processing From 62e362b6fcc628153ddcc0a7c6bdd0b49462e998 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 09:19:43 +0530 Subject: [PATCH 044/241] branch based deployment --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index afec7cf4e5..fee12b8437 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - dg1908 + - janus-optimisation jobs: build: From 15451640f30b32b5cdb4ce0dd991ae44d2ca1ec6 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 10:40:41 +0530 Subject: [PATCH 045/241] add dummy commit --- .../java/org/apache/atlas/discovery/EntityDiscoveryService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 9f2fd920ef..d69c0cae4c 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1149,7 +1149,6 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i entitiesSet.put(header.getGuid(), header); } }); - ret.setEntities(new ArrayList<>(entitiesSet.values())); } From c034fcbc581dfced0e98eb42cedb5cc21720eb82 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 19 Nov 2024 10:51:07 +0530 Subject: [PATCH 046/241] feat: remove redundunt code --- .../src/main/java/org/apache/atlas/web/filters/AuditFilter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java index 53a8213be9..d92bb6b794 100755 --- a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java @@ -90,7 +90,6 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha final Set userGroups = AtlasAuthorizationUtils.getCurrentUserGroups(); final String deleteType = httpRequest.getParameter("deleteType"); final boolean skipFailedEntities = Boolean.parseBoolean(httpRequest.getParameter("skipFailedEntities")); - final String clientOrigin = httpRequest.getHeader("X-Atlan-Client-Origin"); try { currentThread.setName(formatName(oldName, internalRequestId)); From 953bce5bfce5ff0ddb385cabfe2d353ccdc1d9e9 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 11:01:08 +0530 Subject: [PATCH 047/241] add branch --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..bfc6835693 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - janus-optimisation jobs: build: From c6c284bcf7a85f6b28d9140c5ec6588b9cf9dcef Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 11:09:02 +0530 Subject: [PATCH 048/241] add janus branch --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bfc6835693..6227a0cf64 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,7 @@ on: - development - master - lineageondemand - - janus-optimisation + - janusoptimisation jobs: build: From 40f98972d10ce42f04d6ed5e5188bc8416e5a83e Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:23:44 +0530 Subject: [PATCH 049/241] nit: made the hardcoded string o var --- .../main/java/org/apache/atlas/web/filters/AuditFilter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java index d92bb6b794..1f6776592e 100755 --- a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java @@ -60,6 +60,7 @@ public class AuditFilter implements Filter { private static final Logger AUDIT_LOG = LoggerFactory.getLogger("AUDIT"); public static final String TRACE_ID = "trace_id"; public static final String X_ATLAN_REQUEST_ID = "X-Atlan-Request-Id"; + public static final String X_ATLAN_CLIENT_ORIGIN = "X-Atlan-Client-Origin"; private boolean deleteTypeOverrideEnabled = false; private boolean createShellEntityForNonExistingReference = false; @@ -103,7 +104,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha requestContext.setCreateShellEntityForNonExistingReference(createShellEntityForNonExistingReference); requestContext.setForwardedAddresses(AtlasAuthorizationUtils.getForwardedAddressesFromRequest(httpRequest)); requestContext.setSkipFailedEntities(skipFailedEntities); - requestContext.setClientOrigin(httpRequest.getHeader("X-Atlan-Client-Origin")); + requestContext.setClientOrigin(httpRequest.getHeader(X_ATLAN_CLIENT_ORIGIN)); requestContext.setMetricRegistry(metricsRegistry); MDC.put(TRACE_ID, internalRequestId); MDC.put(X_ATLAN_REQUEST_ID, ofNullable(httpRequest.getHeader(X_ATLAN_REQUEST_ID)).orElse(EMPTY)); From b3f24c4cabd1004ed37e144329c1e0d7d0001349 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:47:43 +0530 Subject: [PATCH 050/241] feat: add default tag --- server-api/src/main/java/org/apache/atlas/RequestContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index a2dad1b68d..45e63641c5 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -750,7 +750,7 @@ public String getClientOrigin() { } public void setClientOrigin(String clientOrigin) { - this.clientOrigin = clientOrigin; + this.clientOrigin = StringUtils.isEmpty(this.clientOrigin) ? "other" :clientOrigin; } public class EntityGuidPair { From d27ea4dcb9a5f4a22875175ec9497565437e65a5 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 17:52:14 +0530 Subject: [PATCH 051/241] scrub results --- .../java/org/apache/atlas/discovery/EntityDiscoveryService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index d69c0cae4c..cfdac220a3 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1150,6 +1150,7 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i } }); ret.setEntities(new ArrayList<>(entitiesSet.values())); + scrubSearchResults(ret, searchParams.getSuppressLogs()); } // Non-recursive collapse processing From 0015ef7762eb9ea519b921584b144839d306ad42 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 18:13:18 +0530 Subject: [PATCH 052/241] add flag for janusgraph optimisation --- .../main/java/org/apache/atlas/AtlasConfiguration.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index df2bca7860..388914946c 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -113,7 +113,16 @@ public enum AtlasConfiguration { HERACLES_API_SERVER_URL("atlas.heracles.api.service.url", "http://heracles-service.heracles.svc.cluster.local"), INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), + + /** + * hits elastic search async APU + */ ENABLE_ASYNC_INDEXSEARCH("atlas.indexsearch.async.enable", false), + + /*** + * enables parallel processing of janus graph vertices from cassandra + */ + ENABLE_JANUS_GRAPH_OPTIMISATION("atlas.janus.graph.optimisation.enable", false), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), From 54103c8a1341ae8699a6b84c2460352b08a84857 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 18:14:05 +0530 Subject: [PATCH 053/241] add flag while retrieving from cassandra --- .../discovery/EntityDiscoveryService.java | 83 ++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index cfdac220a3..bba024bcb4 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1077,7 +1077,7 @@ public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) thro } @SuppressWarnings("rawtypes") - private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { + private void prepareSearchResultAsync(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { SearchParams searchParams = ret.getSearchParameters(); boolean showSearchScore = searchParams.getShowSearchScore(); List results = new ArrayList<>(); @@ -1153,6 +1153,87 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i scrubSearchResults(ret, searchParams.getSuppressLogs()); } + private void prepareSearchResultSync(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { + SearchParams searchParams = ret.getSearchParameters(); + try { + if(LOG.isDebugEnabled()){ + LOG.debug("Preparing search results for ({})", ret.getSearchParameters()); + } + Iterator iterator = indexQueryResult.getIterator(); + boolean showSearchScore = searchParams.getShowSearchScore(); + if (iterator == null) { + return; + } + + while (iterator.hasNext()) { + Result result = iterator.next(); + AtlasVertex vertex = result.getVertex(); + + if (vertex == null) { + LOG.warn("vertex in null"); + continue; + } + + AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); + if(RequestContext.get().includeClassifications()){ + header.setClassifications(entityRetriever.getAllClassifications(vertex)); + } + if (showSearchScore) { + ret.addEntityScore(header.getGuid(), result.getScore()); + } + if (fetchCollapsedResults) { + Map collapse = new HashMap<>(); + + Set collapseKeys = result.getCollapseKeys(); + for (String collapseKey : collapseKeys) { + AtlasSearchResult collapseRet = new AtlasSearchResult(); + collapseRet.setSearchParameters(ret.getSearchParameters()); + + Set collapseResultAttributes = new HashSet<>(); + if (searchParams.getCollapseAttributes() != null) { + collapseResultAttributes.addAll(searchParams.getCollapseAttributes()); + } else { + collapseResultAttributes = resultAttributes; + } + + if (searchParams.getCollapseRelationAttributes() != null) { + RequestContext.get().getRelationAttrsForSearch().clear(); + RequestContext.get().setRelationAttrsForSearch(searchParams.getCollapseRelationAttributes()); + } + + DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); + collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); + prepareSearchResultSync(collapseRet, indexQueryCollapsedResult, collapseResultAttributes, false); + + collapseRet.setSearchParameters(null); + collapse.put(collapseKey, collapseRet); + } + if (!collapse.isEmpty()) { + header.setCollapse(collapse); + } + } + if (searchParams.getShowSearchMetadata()) { + ret.addHighlights(header.getGuid(), result.getHighLights()); + ret.addSort(header.getGuid(), result.getSort()); + } else if (searchParams.getShowHighlights()) { + ret.addHighlights(header.getGuid(), result.getHighLights()); + } + + ret.addEntity(header); + } + } catch (Exception e) { + throw e; + } + scrubSearchResults(ret, searchParams.getSuppressLogs()); + } + + private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { + if (AtlasConfiguration.ENABLE_JANUS_GRAPH_OPTIMISATION.getBoolean()) { + LOG.debug("enabled janusGraphOptimisation"); + prepareSearchResultAsync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); + } + prepareSearchResultSync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); + } // Non-recursive collapse processing private Map processCollapseResults(Result result, SearchParams searchParams, Set resultAttributes) throws AtlasBaseException { Map collapse = new HashMap<>(); From b903c7b602178b8cc37d84a01a878b330b78b233 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 19 Nov 2024 19:40:29 +0530 Subject: [PATCH 054/241] log ES query error --- .../atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index dc46861b12..fb6fe85271 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -431,7 +431,7 @@ private String performDirectIndexQuery(String query, boolean source) throws Atla LOG.warn(String.format("ES index with name %s not found", index)); throw new AtlasBaseException(INDEX_NOT_FOUND, index); } else { - throw new AtlasBaseException(rex); + throw new AtlasBaseException(String.format("Error in executing elastic query: %s", EntityUtils.toString(entity)), rex); } } From 72f46d087a541882c329352b987028878006ad3f Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Tue, 19 Nov 2024 23:01:49 +0530 Subject: [PATCH 055/241] add elastic query timeout metric only on timeouts --- .../janus/AtlasElasticsearchQuery.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index dc46861b12..dd182e3040 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -207,6 +207,10 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP if (response == null) { // Rather than null (if the response is null wil help returning @204 HTTP_NO_CONTENT to the user) // return timeout exception to user + throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED_DUE_TO_TIMEOUT, KeepAliveTime); + } + + if(response.isTimedOut()) { LOG.error("timeout exceeded for query {}:", searchParams.getQuery()); RequestContext.get().endMetricRecord(RequestContext.get().startMetricRecord("elasticQueryTimeout")); throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED_DUE_TO_TIMEOUT, KeepAliveTime); @@ -294,6 +298,7 @@ public void onSuccess(Response response) { Map responseMap = AtlasType.fromJson(respString, Map.class); Boolean isInComplete = AtlasType.fromJson(AtlasType.toJson(responseMap.get("is_partial")), Boolean.class); String id = AtlasType.fromJson(AtlasType.toJson(responseMap.get("id")), String.class); + boolean isTimedOut = AtlasType.fromJson(AtlasType.toJson(responseMap.get("response").get("timed_out")), Boolean.class); if (isInComplete != null && isInComplete) { /* @@ -304,7 +309,7 @@ public void onSuccess(Response response) { deleteAsyncSearchResponse(id); future.complete(null); } - AsyncQueryResult result = new AsyncQueryResult(respString, false); + AsyncQueryResult result = new AsyncQueryResult(respString, false, isTimedOut); future.complete(result); } catch (IOException e) { future.completeExceptionally(e); @@ -382,7 +387,8 @@ public void onSuccess(Response response) { Map responseMap = AtlasType.fromJson(respString, Map.class); boolean isRunning = AtlasType.fromJson(AtlasType.toJson(responseMap.get("is_running")), Boolean.class); String id = AtlasType.fromJson(AtlasType.toJson(responseMap.get("id")), String.class); - AsyncQueryResult result = new AsyncQueryResult(respString, isRunning); + boolean isTimedOut = AtlasType.fromJson(AtlasType.toJson(responseMap.get("response").get("timed_out")), Boolean.class); + AsyncQueryResult result = new AsyncQueryResult(respString, isRunning, isTimedOut); /* * If the response is running, then we need to complete the future with the ID to retrieve this later * Else we will complete the future with the response, if it completes within default timeout of 100ms @@ -431,7 +437,7 @@ private String performDirectIndexQuery(String query, boolean source) throws Atla LOG.warn(String.format("ES index with name %s not found", index)); throw new AtlasBaseException(INDEX_NOT_FOUND, index); } else { - throw new AtlasBaseException(rex); + throw new AtlasBaseException(String.format("Error in executing elastic query: %s", EntityUtils.toString(entity)), rex); } } @@ -633,6 +639,7 @@ public class AsyncQueryResult { private boolean isRunning; private String id; private String fullResponse; + private boolean timedOut; private boolean success; // Constructor for a running process @@ -643,10 +650,11 @@ public AsyncQueryResult(String id) { } // Constructor for a completed process - public AsyncQueryResult(String fullResponse, boolean isRunning) { + public AsyncQueryResult(String fullResponse, boolean isRunning, boolean timedOut) { this.isRunning = isRunning; this.id = null; this.fullResponse = fullResponse; + this.timedOut = timedOut; } public void setRunning(boolean running) { @@ -658,6 +666,15 @@ public boolean isRunning() { return isRunning; } + public void setTimedOut(boolean timedOut) { + this.timedOut = timedOut; + } + + // Getters + public boolean isTimedOut() { + return timedOut; + } + void setId(String id) { this.id = id; } From 6fac8e4028022a659becb2661130644fe7d080dd Mon Sep 17 00:00:00 2001 From: Abhijeet Kumar Date: Wed, 20 Nov 2024 11:53:04 +0530 Subject: [PATCH 056/241] added params for phase1 tag propogation scaling --- .../apache/atlas/repository/Constants.java | 2 ++ .../apache/atlas/model/tasks/AtlasTask.java | 27 +++++++++++++++++++ .../graph/GraphBackedSearchIndexer.java | 2 ++ .../store/graph/v2/EntityGraphMapper.java | 5 +++- 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 4fc62e0e6e..11e08bf78b 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -357,6 +357,8 @@ public final class Constants { public static final String TASK_CLASSIFICATION_ID = encodePropertyKey(TASK_PREFIX + "classificationId"); public static final String TASK_ENTITY_GUID = encodePropertyKey(TASK_PREFIX + "entityGuid"); public static final String TASK_CLASSIFICATION_TYPENAME = encodePropertyKey(TASK_PREFIX + "classificationTypeName"); + public static final String TASK_ASSET_COUNT_TO_PROPAGATE = encodePropertyKey(TASK_PREFIX + "assetsCountToPropagate"); + public static final String TASK_ASSET_COUNT_PROPAGATED = encodePropertyKey(TASK_PREFIX + "assetsCountPropagated"); public static final String ACTIVE_STATE_VALUE = "ACTIVE"; public static final String TASK_HEADER_ATLAN_AGENT = "x-atlan-agent"; public static final String TASK_HEADER_ATLAN_AGENT_ID = "x-atlan-agent-id"; diff --git a/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java b/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java index e4b87620d9..463da44df9 100644 --- a/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java +++ b/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java @@ -95,6 +95,8 @@ public static Status from(String s) { private String classificationId; private String entityGuid; private String classificationTypeName; + private Long assetsCountToPropagate; + private Long assetsCountPropagated; public AtlasTask() { } @@ -113,6 +115,22 @@ public AtlasTask(String type, String createdBy, Map parameters, this.entityGuid = entityGuid; } + public AtlasTask(String type, String createdBy, Map parameters, String classificationId, + String entityGuid, Long assetsCountToPropagate, Long assetsCountPropagated) { + this.guid = UUID.randomUUID().toString(); + this.type = type; + this.createdBy = createdBy; + this.createdTime = new Date(); + this.updatedTime = this.createdTime; + this.parameters = parameters; + this.status = Status.PENDING; + this.attemptCount = 0; + this.classificationId = classificationId; + this.entityGuid = entityGuid; + this.assetsCountToPropagate = assetsCountToPropagate; + this.assetsCountPropagated = assetsCountPropagated; + } + public String getGuid() { return guid; } @@ -239,6 +257,13 @@ public String getEntityGuid() { return entityGuid; } + public void setAssetsCountToPropagate(Long assetsCount) { + this.assetsCountToPropagate = assetsCount ; + } + public Long getAssetsCountToPropagate() { + return assetsCountToPropagate ; + } + @JsonIgnore public void start() { this.setStatus(Status.IN_PROGRESS); @@ -270,6 +295,8 @@ public String toString() { ", attemptCount=" + attemptCount + ", errorMessage='" + errorMessage + '\'' + ", status=" + status + + ", assetsCountToPropagate=" + assetsCountToPropagate + + ", assetsCountPropagated=" + assetsCountPropagated + '}'; } } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index a8bacbbb5f..7b2c3da7ac 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -407,6 +407,8 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep createCommonVertexIndex(management, TASK_ENTITY_GUID, UniqueKind.NONE, String.class, SINGLE, false, false, true); createCommonVertexIndex(management, TASK_ERROR_MESSAGE, UniqueKind.NONE, String.class, SINGLE, false, false); createCommonVertexIndex(management, TASK_ATTEMPT_COUNT, UniqueKind.NONE, Integer.class, SINGLE, false, false); + createCommonVertexIndex(management, TASK_ASSET_COUNT_TO_PROPAGATE, UniqueKind.NONE, Long.class, SINGLE, false, false); + createCommonVertexIndex(management, TASK_ASSET_COUNT_PROPAGATED, UniqueKind.NONE, Long.class, SINGLE, false, false); createCommonVertexIndex(management, TASK_UPDATED_TIME, UniqueKind.NONE, Long.class, SINGLE, false, false); createCommonVertexIndex(management, TASK_TIME_TAKEN_IN_SECONDS, UniqueKind.NONE, Long.class, SINGLE, false, false); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 76f0be44cb..3c7b6dbf89 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3459,7 +3459,6 @@ public List propagateClassification(String entityGuid, String classifica List edgeLabelsToCheck = CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode); Boolean toExclude = propagationMode == CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ? true:false; List impactedVertices = entityRetriever.getIncludedImpactedVerticesV2(entityVertex, relationshipGuid, classificationVertexId, edgeLabelsToCheck,toExclude); - if (CollectionUtils.isEmpty(impactedVertices)) { LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", entityGuid, classificationVertexId); @@ -3919,6 +3918,10 @@ public void updateClassifications(EntityMutationContext context, String guid, Li } Boolean toExclude = propagationMode == CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE ? true : false; entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex, null, classificationVertex.getIdForDisplay(), CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode),toExclude); + + //update the 'assetsCountToPropagate' in the current task vertex. + AtlasTask currentTask = RequestContext.get().getCurrentTask(); + currentTask.setAssetsCountToPropagate((long) entitiesToPropagateTo.size()); } if (CollectionUtils.isNotEmpty(entitiesToPropagateTo)) { From 1f0c6612d92c1517fd7f3f1cd4d06d4ad3f2679a Mon Sep 17 00:00:00 2001 From: Abhijeet Kumar Date: Wed, 20 Nov 2024 12:30:48 +0530 Subject: [PATCH 057/241] added code to update 'AssetsCountToPropagate' in task vertex. --- .../repository/store/graph/v2/EntityGraphMapper.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 3c7b6dbf89..6ff0127bc0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3459,6 +3459,14 @@ public List propagateClassification(String entityGuid, String classifica List edgeLabelsToCheck = CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode); Boolean toExclude = propagationMode == CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ? true:false; List impactedVertices = entityRetriever.getIncludedImpactedVerticesV2(entityVertex, relationshipGuid, classificationVertexId, edgeLabelsToCheck,toExclude); + + // update the 'assetsCountToPropagate' on in memory java object. + AtlasTask currentTask = RequestContext.get().getCurrentTask(); + currentTask.setAssetsCountToPropagate((long) impactedVertices.size()); + + //update the 'assetsCountToPropagate' in the current task vertex. + AtlasVertex currentTaskVertex = graphHelper.getVertexForGUID(currentTask.getGuid()); + currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, impactedVertices.size()); if (CollectionUtils.isEmpty(impactedVertices)) { LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", entityGuid, classificationVertexId); @@ -3918,10 +3926,6 @@ public void updateClassifications(EntityMutationContext context, String guid, Li } Boolean toExclude = propagationMode == CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE ? true : false; entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex, null, classificationVertex.getIdForDisplay(), CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode),toExclude); - - //update the 'assetsCountToPropagate' in the current task vertex. - AtlasTask currentTask = RequestContext.get().getCurrentTask(); - currentTask.setAssetsCountToPropagate((long) entitiesToPropagateTo.size()); } if (CollectionUtils.isNotEmpty(entitiesToPropagateTo)) { From 68c1b4c3da4b4973f29e66475ebc9394359255bf Mon Sep 17 00:00:00 2001 From: Abhijeet Kumar Date: Wed, 20 Nov 2024 13:03:20 +0530 Subject: [PATCH 058/241] changed code to fetch current task using guid and updating 'assetsCountToPropagate' --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 6ff0127bc0..52b36582f9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3465,7 +3465,7 @@ public List propagateClassification(String entityGuid, String classifica currentTask.setAssetsCountToPropagate((long) impactedVertices.size()); //update the 'assetsCountToPropagate' in the current task vertex. - AtlasVertex currentTaskVertex = graphHelper.getVertexForGUID(currentTask.getGuid()); + AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, impactedVertices.size()); if (CollectionUtils.isEmpty(impactedVertices)) { LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", entityGuid, classificationVertexId); From 3fd83b23614e7f9760185f2555862081191b7d40 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Tue, 19 Nov 2024 23:01:49 +0530 Subject: [PATCH 059/241] add elastic query timeout metric only on timeouts --- .../janus/AtlasElasticsearchQuery.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index dc46861b12..dd182e3040 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -207,6 +207,10 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP if (response == null) { // Rather than null (if the response is null wil help returning @204 HTTP_NO_CONTENT to the user) // return timeout exception to user + throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED_DUE_TO_TIMEOUT, KeepAliveTime); + } + + if(response.isTimedOut()) { LOG.error("timeout exceeded for query {}:", searchParams.getQuery()); RequestContext.get().endMetricRecord(RequestContext.get().startMetricRecord("elasticQueryTimeout")); throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED_DUE_TO_TIMEOUT, KeepAliveTime); @@ -294,6 +298,7 @@ public void onSuccess(Response response) { Map responseMap = AtlasType.fromJson(respString, Map.class); Boolean isInComplete = AtlasType.fromJson(AtlasType.toJson(responseMap.get("is_partial")), Boolean.class); String id = AtlasType.fromJson(AtlasType.toJson(responseMap.get("id")), String.class); + boolean isTimedOut = AtlasType.fromJson(AtlasType.toJson(responseMap.get("response").get("timed_out")), Boolean.class); if (isInComplete != null && isInComplete) { /* @@ -304,7 +309,7 @@ public void onSuccess(Response response) { deleteAsyncSearchResponse(id); future.complete(null); } - AsyncQueryResult result = new AsyncQueryResult(respString, false); + AsyncQueryResult result = new AsyncQueryResult(respString, false, isTimedOut); future.complete(result); } catch (IOException e) { future.completeExceptionally(e); @@ -382,7 +387,8 @@ public void onSuccess(Response response) { Map responseMap = AtlasType.fromJson(respString, Map.class); boolean isRunning = AtlasType.fromJson(AtlasType.toJson(responseMap.get("is_running")), Boolean.class); String id = AtlasType.fromJson(AtlasType.toJson(responseMap.get("id")), String.class); - AsyncQueryResult result = new AsyncQueryResult(respString, isRunning); + boolean isTimedOut = AtlasType.fromJson(AtlasType.toJson(responseMap.get("response").get("timed_out")), Boolean.class); + AsyncQueryResult result = new AsyncQueryResult(respString, isRunning, isTimedOut); /* * If the response is running, then we need to complete the future with the ID to retrieve this later * Else we will complete the future with the response, if it completes within default timeout of 100ms @@ -431,7 +437,7 @@ private String performDirectIndexQuery(String query, boolean source) throws Atla LOG.warn(String.format("ES index with name %s not found", index)); throw new AtlasBaseException(INDEX_NOT_FOUND, index); } else { - throw new AtlasBaseException(rex); + throw new AtlasBaseException(String.format("Error in executing elastic query: %s", EntityUtils.toString(entity)), rex); } } @@ -633,6 +639,7 @@ public class AsyncQueryResult { private boolean isRunning; private String id; private String fullResponse; + private boolean timedOut; private boolean success; // Constructor for a running process @@ -643,10 +650,11 @@ public AsyncQueryResult(String id) { } // Constructor for a completed process - public AsyncQueryResult(String fullResponse, boolean isRunning) { + public AsyncQueryResult(String fullResponse, boolean isRunning, boolean timedOut) { this.isRunning = isRunning; this.id = null; this.fullResponse = fullResponse; + this.timedOut = timedOut; } public void setRunning(boolean running) { @@ -658,6 +666,15 @@ public boolean isRunning() { return isRunning; } + public void setTimedOut(boolean timedOut) { + this.timedOut = timedOut; + } + + // Getters + public boolean isTimedOut() { + return timedOut; + } + void setId(String id) { this.id = id; } From dc7765771f714fff69430a6946cad921a250f672 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 20 Nov 2024 15:42:00 +0530 Subject: [PATCH 060/241] fix map vertex method --- .../store/graph/v2/EntityGraphRetriever.java | 212 ++++++++++++++---- 1 file changed, 169 insertions(+), 43 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 34d3036042..cb4f515b8b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1009,90 +1009,216 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } +// private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { +// AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); +// AtlasEntityHeader ret = new AtlasEntityHeader(); +// try { +// String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); +// String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); +// Boolean isIncomplete = isEntityIncomplete(entityVertex); +// +// ret.setTypeName(typeName); +// ret.setGuid(guid); +// ret.setStatus(GraphHelper.getStatus(entityVertex)); +// RequestContext context = RequestContext.get(); +// boolean includeClassifications = context.includeClassifications(); +// boolean includeClassificationNames = context.isIncludeClassificationNames(); +// if(includeClassifications){ +// ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); +// } else if (!includeClassifications && includeClassificationNames) { +// ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); +// } +// ret.setIsIncomplete(isIncomplete); +// ret.setLabels(getLabels(entityVertex)); +// +// ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); +// ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); +// ret.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); +// ret.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); +// +// if(RequestContext.get().includeMeanings()) { +// List termAssignmentHeaders = mapAssignedTerms(entityVertex); +// ret.setMeanings(termAssignmentHeaders); +// ret.setMeaningNames( +// termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) +// .collect(Collectors.toList())); +// } +// AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); +// +// if (entityType != null) { +// for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { +// Object attrValue = getVertexAttribute(entityVertex, headerAttribute); +// +// if (attrValue != null) { +// ret.setAttribute(headerAttribute.getName(), attrValue); +// } +// } +// +// Object displayText = getDisplayText(entityVertex, entityType); +// +// if (displayText != null) { +// ret.setDisplayText(displayText.toString()); +// } +// +// if (CollectionUtils.isNotEmpty(attributes)) { +// for (String attrName : attributes) { +// AtlasAttribute attribute = entityType.getAttribute(attrName); +// +// if (attribute == null) { +// attrName = toNonQualifiedName(attrName); +// +// if (ret.hasAttribute(attrName)) { +// continue; +// } +// +// attribute = entityType.getAttribute(attrName); +// +// if (attribute == null) { +// attribute = entityType.getRelationshipAttribute(attrName, null); +// } +// } +// +// Object attrValue = getVertexAttribute(entityVertex, attribute); +// +// if (attrValue != null) { +// ret.setAttribute(attrName, attrValue); +// } +// } +// } +// } +// } +// finally { +// RequestContext.get().endMetricRecord(metricRecorder); +// } +// return ret; +// } + + private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); + AtlasEntityHeader ret = new AtlasEntityHeader(); + try { - String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); - String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); + // Batch fetch all properties + Map vertexProperties = batchGetProperties(entityVertex); + + // Common properties + String typeName = (String) getCommonProperty(vertexProperties, Constants.TYPE_NAME_PROPERTY_KEY); + String guid = (String) getCommonProperty(vertexProperties, GUID_PROPERTY_KEY); + String createdBy = (String) getCommonProperty(vertexProperties, CREATED_BY_KEY); + String updatedBy = (String) getCommonProperty(vertexProperties, MODIFIED_BY_KEY); + Long createTime = (Long) getCommonProperty(vertexProperties, TIMESTAMP_PROPERTY_KEY); + Long updateTime = (Long) getCommonProperty(vertexProperties, MODIFICATION_TIMESTAMP_PROPERTY_KEY); Boolean isIncomplete = isEntityIncomplete(entityVertex); + // Set properties in header ret.setTypeName(typeName); ret.setGuid(guid); ret.setStatus(GraphHelper.getStatus(entityVertex)); - RequestContext context = RequestContext.get(); - boolean includeClassifications = context.includeClassifications(); - boolean includeClassificationNames = context.isIncludeClassificationNames(); - if(includeClassifications){ - ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); - } else if (!includeClassifications && includeClassificationNames) { - ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); - } ret.setIsIncomplete(isIncomplete); + ret.setCreatedBy(createdBy); + ret.setUpdatedBy(updatedBy); + ret.setCreateTime(createTime != null ? new Date(createTime) : null); + ret.setUpdateTime(updateTime != null ? new Date(updateTime) : null); ret.setLabels(getLabels(entityVertex)); - ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); - ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); - ret.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); - ret.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); + // Classifications + RequestContext context = RequestContext.get(); + if (context.includeClassifications() || context.isIncludeClassificationNames()) { + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } - if(RequestContext.get().includeMeanings()) { + // Meanings + if (context.includeMeanings()) { List termAssignmentHeaders = mapAssignedTerms(entityVertex); ret.setMeanings(termAssignmentHeaders); - ret.setMeaningNames( - termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) - .collect(Collectors.toList())); + ret.setMeaningNames(termAssignmentHeaders.stream() + .map(AtlasTermAssignmentHeader::getDisplayText) + .collect(Collectors.toList())); } - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + // Process entity type and attributes + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); if (entityType != null) { + // Header attributes for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { - Object attrValue = getVertexAttribute(entityVertex, headerAttribute); - + Object attrValue = getVertexAttributeFromBatch(vertexProperties, headerAttribute); if (attrValue != null) { ret.setAttribute(headerAttribute.getName(), attrValue); } } + // Display text Object displayText = getDisplayText(entityVertex, entityType); - if (displayText != null) { ret.setDisplayText(displayText.toString()); } + // Additional attributes if (CollectionUtils.isNotEmpty(attributes)) { for (String attrName : attributes) { - AtlasAttribute attribute = entityType.getAttribute(attrName); - - if (attribute == null) { - attrName = toNonQualifiedName(attrName); - - if (ret.hasAttribute(attrName)) { - continue; + AtlasAttribute attribute = getEntityOrRelationshipAttribute(entityType, attrName); + if (attribute != null) { + Object attrValue = getVertexAttributeFromBatch(vertexProperties, attribute); + if (attrValue != null) { + ret.setAttribute(attrName, attrValue); } - - attribute = entityType.getAttribute(attrName); - - if (attribute == null) { - attribute = entityType.getRelationshipAttribute(attrName, null); - } - } - - Object attrValue = getVertexAttribute(entityVertex, attribute); - - if (attrValue != null) { - ret.setAttribute(attrName, attrValue); } } } } - } - finally { + } finally { RequestContext.get().endMetricRecord(metricRecorder); } + return ret; } + + private Object getCommonProperty(Map vertexProperties, String propertyName) { + if (vertexProperties.get(propertyName) instanceof List) { + return ((List) vertexProperties.get(propertyName)).get(0); + } + return ""; + } + /** + * Fetches all properties of a vertex in one call and returns them as a map. + */ + private Map batchGetProperties(AtlasVertex vertex) { + // Use JanusGraph's Gremlin API for efficient property retrieval + return (Map) graph.V(vertex.getId()).valueMap().next(); + } + + /** + * Retrieves a vertex attribute from the pre-fetched batch of properties. + */ + private Object getVertexAttributeFromBatch(Map properties, AtlasAttribute attribute) { + if (properties == null || attribute == null) { + return null; + } + + String propertyKey = attribute.getVertexPropertyName(); + return properties.get(propertyKey); + } + + /** + * Retrieves an entity or relationship attribute from the entity type. + */ + private AtlasAttribute getEntityOrRelationshipAttribute(AtlasEntityType entityType, String attrName) { + AtlasAttribute attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attrName = toNonQualifiedName(attrName); + + attribute = entityType.getAttribute(attrName); + if (attribute == null) { + attribute = entityType.getRelationshipAttribute(attrName, null); + } + } + + return attribute; + } private String toNonQualifiedName(String attrName) { String ret; if (attrName.contains(".")) { From ec4a32d1c7b780b91da9f83a7a3cbae0fca8c427 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 20 Nov 2024 15:52:06 +0530 Subject: [PATCH 061/241] add configuration in flag --- .../src/main/java/org/apache/atlas/AtlasConfiguration.java | 7 ++++++- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 3 +-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 388914946c..3e931b4d67 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -115,7 +115,7 @@ public enum AtlasConfiguration { INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), /** - * hits elastic search async APU + * hits elastic search async API */ ENABLE_ASYNC_INDEXSEARCH("atlas.indexsearch.async.enable", false), @@ -123,6 +123,11 @@ public enum AtlasConfiguration { * enables parallel processing of janus graph vertices from cassandra */ ENABLE_JANUS_GRAPH_OPTIMISATION("atlas.janus.graph.optimisation.enable", false), + + /** + * No. of threads to be spawned for parallel processing + */ + THREADS_TO_BE_SPAWNED("atlas.janus.graph.optimisation.thread_count", (Runtime.getRuntime().availableProcessors())/2), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index bba024bcb4..efa124a3fa 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -87,8 +87,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class); private static final String DEFAULT_SORT_ATTRIBUTE_NAME = "name"; - private static final int AVAILABLEPROCESSORS = Runtime.getRuntime().availableProcessors(); - private static final ForkJoinPool CUSTOMTHREADPOOL = new ForkJoinPool(AVAILABLEPROCESSORS/2); // Use half of available cores + private static final ForkJoinPool CUSTOMTHREADPOOL = new ForkJoinPool(AtlasConfiguration.THREADS_TO_BE_SPAWNED.getInt()); // Use half of available cores private final AtlasGraph graph; private final EntityGraphRetriever entityRetriever; From f51f8f678ef7fb30bfdf6cc10ecd08b8eba942e0 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 20 Nov 2024 15:56:30 +0530 Subject: [PATCH 062/241] fix return type --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index cb4f515b8b..0f11a100fc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1180,7 +1180,7 @@ private Object getCommonProperty(Map vertexProperties, String pr if (vertexProperties.get(propertyName) instanceof List) { return ((List) vertexProperties.get(propertyName)).get(0); } - return ""; + return new Object(); } /** * Fetches all properties of a vertex in one call and returns them as a map. From ef5ed89d60a3a529826bacebf51615aee37a28d3 Mon Sep 17 00:00:00 2001 From: Abhijeet Kumar Date: Wed, 20 Nov 2024 16:49:40 +0530 Subject: [PATCH 063/241] added code to update 'assetCountToPropagate' for 'updateClassificationTextPropagation' 'Delete', 'RefreshPropagation' --- .../store/graph/v2/EntityGraphMapper.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 52b36582f9..cdb4514859 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3467,6 +3467,7 @@ public List propagateClassification(String entityGuid, String classifica //update the 'assetsCountToPropagate' in the current task vertex. AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, impactedVertices.size()); + if (CollectionUtils.isEmpty(impactedVertices)) { LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", entityGuid, classificationVertexId); @@ -4103,6 +4104,14 @@ public List deleteClassificationPropagation(String entityGuid, String cl int propagatedEdgesSize = propagatedEdges.size(); + // update the 'assetsCountToPropagate' on in memory java object. + AtlasTask currentTask = RequestContext.get().getCurrentTask(); + currentTask.setAssetsCountToPropagate((long) propagatedEdgesSize); + + //update the 'assetsCountToPropagate' in the current task vertex. + AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); + currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, propagatedEdgesSize); + LOG.info(String.format("Number of edges to be deleted : %s for classification vertex with id : %s", propagatedEdgesSize, classificationVertexId)); List deletedPropagationsGuid = processClassificationEdgeDeletionInChunk(classification, propagatedEdges); @@ -4262,6 +4271,16 @@ public void classificationRefreshPropagation(String classificationId) throws Atl .filter(vertex -> vertex != null) .collect(Collectors.toList()); + Integer taskCount = verticesToRemove.size() + verticesToAddClassification.size(); + + // update the 'assetsCountToPropagate' on in memory java object. + AtlasTask currentTask = RequestContext.get().getCurrentTask(); + currentTask.setAssetsCountToPropagate((long) taskCount); + + //update the 'assetsCountToPropagate' in the current task vertex. + AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); + currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, taskCount); + //Remove classifications from unreachable vertices processPropagatedClassificationDeletionFromVertices(verticesToRemove, currentClassificationVertex, classification); From 234bcce1787396de54c0a32d0949e57a1180c3fd Mon Sep 17 00:00:00 2001 From: Abhijeet Kumar Date: Wed, 20 Nov 2024 17:20:59 +0530 Subject: [PATCH 064/241] added AssetsCountToPropagate in taskregistry --- .../src/main/java/org/apache/atlas/tasks/TaskRegistry.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index 8c08a26121..df257560d9 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -558,6 +558,10 @@ public static AtlasTask toAtlasTask(AtlasVertex v) { ret.setErrorMessage(errorMessage); } + long countToPropagate = v.getProperty(Constants.TASK_ASSET_COUNT_TO_PROPAGATE, Long.class); + if (errorMessage != null) { + ret.setAssetsCountToPropagate(countToPropagate); + } return ret; } From 391e510ea0bf632ce0c3b0a91177a7b8059cc872 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 20 Nov 2024 17:50:15 +0530 Subject: [PATCH 065/241] add flag for collapsed result --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 1 + .../java/org/apache/atlas/discovery/EntityDiscoveryService.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 3e931b4d67..9390f3959a 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -128,6 +128,7 @@ public enum AtlasConfiguration { * No. of threads to be spawned for parallel processing */ THREADS_TO_BE_SPAWNED("atlas.janus.graph.optimisation.thread_count", (Runtime.getRuntime().availableProcessors())/2), + FETCH_COLLAPSED_RESULT("atlas.indexsearch.fetch.collapsed.result", true), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index efa124a3fa..3f8155d9c1 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1006,7 +1006,7 @@ public AtlasSearchResult directIndexSearch(SearchParams searchParams) throws Atl return null; } RequestContext.get().endMetricRecord(elasticSearchQueryMetric); - prepareSearchResult(ret, indexQueryResult, resultAttributes, true); + prepareSearchResult(ret, indexQueryResult, resultAttributes, AtlasConfiguration.FETCH_COLLAPSED_RESULT.getBoolean()); ret.setAggregations(indexQueryResult.getAggregationMap()); ret.setApproximateCount(indexQuery.vertexTotals()); From 09adb515307ad9d4d2b40f27dcd8012fee60e40c Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 20 Nov 2024 17:53:39 +0530 Subject: [PATCH 066/241] mesh-283: unlink domain from asset to delete --- .../graph/v2/preprocessor/AssetPreProcessor.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 791b19fca8..b500ba3847 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -61,6 +61,9 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co case UPDATE: processUpdateAsset(entity, vertex); break; + case DELETE: + processDelete(vertex); + break; } } @@ -116,6 +119,14 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept } } + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + //remove the domain link + if (vertex != null) { + vertex.removeProperty(DOMAIN_GUIDS); + } + } + private void isAuthorized(AtlasVertex vertex) throws AtlasBaseException { AtlasEntityHeader sourceEntity = retrieverNoRelation.toAtlasEntityHeaderWithClassifications(vertex); From 875138795c9189fccb53b25b6f0e56e227cbb03e Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 20 Nov 2024 18:10:21 +0530 Subject: [PATCH 067/241] fix updated by and createdby field --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 0f11a100fc..bb2c3bf071 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1106,8 +1106,6 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, // Common properties String typeName = (String) getCommonProperty(vertexProperties, Constants.TYPE_NAME_PROPERTY_KEY); String guid = (String) getCommonProperty(vertexProperties, GUID_PROPERTY_KEY); - String createdBy = (String) getCommonProperty(vertexProperties, CREATED_BY_KEY); - String updatedBy = (String) getCommonProperty(vertexProperties, MODIFIED_BY_KEY); Long createTime = (Long) getCommonProperty(vertexProperties, TIMESTAMP_PROPERTY_KEY); Long updateTime = (Long) getCommonProperty(vertexProperties, MODIFICATION_TIMESTAMP_PROPERTY_KEY); Boolean isIncomplete = isEntityIncomplete(entityVertex); @@ -1117,8 +1115,8 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, ret.setGuid(guid); ret.setStatus(GraphHelper.getStatus(entityVertex)); ret.setIsIncomplete(isIncomplete); - ret.setCreatedBy(createdBy); - ret.setUpdatedBy(updatedBy); + ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); + ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); ret.setCreateTime(createTime != null ? new Date(createTime) : null); ret.setUpdateTime(updateTime != null ? new Date(updateTime) : null); ret.setLabels(getLabels(entityVertex)); From 6f1549dc44946043e3e459eb4802de03d317d6b9 Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 21 Nov 2024 10:49:53 +0530 Subject: [PATCH 068/241] fix if condition --- .../org/apache/atlas/discovery/EntityDiscoveryService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 3f8155d9c1..561cc456eb 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1230,8 +1230,9 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i if (AtlasConfiguration.ENABLE_JANUS_GRAPH_OPTIMISATION.getBoolean()) { LOG.debug("enabled janusGraphOptimisation"); prepareSearchResultAsync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); + }else { + prepareSearchResultSync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); } - prepareSearchResultSync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); } // Non-recursive collapse processing private Map processCollapseResults(Result result, SearchParams searchParams, Set resultAttributes) throws AtlasBaseException { From 7feef651db9a8be898645da266bb4dd8ca45222a Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 21 Nov 2024 12:17:45 +0530 Subject: [PATCH 069/241] remove branch deployment --- .github/workflows/maven.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 6227a0cf64..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - janusoptimisation jobs: build: From d4c2f423d460647e90941fe8f806374b5b587a6f Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 21 Nov 2024 12:24:46 +0530 Subject: [PATCH 070/241] mesh-280: resolved error --- .../graph/v2/preprocessor/datamesh/DataProductPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 458a5c695f..6d2a122217 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -83,7 +83,7 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws entity.removeAttribute(OUTPUT_PORT_GUIDS_ATTR); entity.removeAttribute(INPUT_PORT_GUIDS_ATTR); - if(!entity.hasAttribute(DAAP_ASSET_DSL_ATTR) && entity.getAttribute(DAAP_ASSET_DSL_ATTR) == null){ + if(!entity.hasAttribute(DAAP_ASSET_DSL_ATTR) || entity.getAttribute(DAAP_ASSET_DSL_ATTR) == null){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "DataProductAssetDSL attribute is mandatory for DataProduct creation"); } From bd1ddb085f738478d168d39ce18f19fe3d7777bf Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 21 Nov 2024 13:04:31 +0530 Subject: [PATCH 071/241] use multi query to improve response time of post elastic query lookups --- .../graphdb/janus/AtlasJanusGraph.java | 11 +- .../discovery/EntityDiscoveryService.java | 24 +-- .../audit/EntityAuditListenerV2.java | 2 +- .../store/graph/v2/EntityGraphRetriever.java | 176 ++++++++++++++++++ 4 files changed, 195 insertions(+), 18 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java index 6c5e0c9886..9bd2fbcdcd 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java @@ -69,12 +69,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.janusgraph.core.Cardinality; -import org.janusgraph.core.JanusGraph; -import org.janusgraph.core.JanusGraphFactory; -import org.janusgraph.core.JanusGraphIndexQuery; -import org.janusgraph.core.PropertyKey; -import org.janusgraph.core.SchemaViolationException; +import org.janusgraph.core.*; import org.janusgraph.core.schema.JanusGraphIndex; import org.janusgraph.core.schema.JanusGraphManagement; import org.janusgraph.core.schema.Parameter; @@ -696,4 +691,8 @@ public void setEnableCache(boolean enableCache) { public Boolean isCacheEnabled() { return this.janusGraph.isCacheEnabled(); } + + public JanusGraphTransaction getTransaction() { + return this.janusGraph.newThreadBoundTransaction(); + } } diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index efa124a3fa..8694623f0a 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -70,9 +70,7 @@ import javax.script.ScriptEngine; import javax.script.ScriptException; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; +import java.util.function.Function; import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.*; @@ -87,7 +85,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class); private static final String DEFAULT_SORT_ATTRIBUTE_NAME = "name"; - private static final ForkJoinPool CUSTOMTHREADPOOL = new ForkJoinPool(AtlasConfiguration.THREADS_TO_BE_SPAWNED.getInt()); // Use half of available cores private final AtlasGraph graph; private final EntityGraphRetriever entityRetriever; @@ -1094,15 +1091,19 @@ private void prepareSearchResultAsync(AtlasSearchResult ret, DirectIndexQueryRes .collect(Collectors.toList()); // Use ConcurrentHashMap for thread-safe access - ConcurrentHashMap headers = new ConcurrentHashMap<>(); - ConcurrentHashMap entitiesSet = new ConcurrentHashMap<>(); + //ConcurrentHashMap headers = new ConcurrentHashMap<>(); + //ConcurrentHashMap entitiesSet = new ConcurrentHashMap<>(); + + List headers = entityRetriever.mapVerticesToAtlasEntityHeader(vertices, resultAttributes); + // Create a Set entitiesSet = headers.stream().collect(Collectors.toMap(AtlasEntityHeader::getGuid, Function.identity())); // Run vertex processing in limited parallel threads - CompletableFuture.runAsync(() -> vertices.parallelStream().forEach(vertex -> { + /**CompletableFuture.runAsync(() -> vertices.parallelStream().forEach(vertex -> { String guid = vertex.getProperty("__guid", String.class); headers.computeIfAbsent(guid, k -> { try { - AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); + //AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); if (RequestContext.get().includeClassifications()) { header.setClassifications(entityRetriever.getAllClassifications(vertex)); } @@ -1111,7 +1112,7 @@ private void prepareSearchResultAsync(AtlasSearchResult ret, DirectIndexQueryRes throw new RuntimeException("Failed to process vertex with GUID: " + guid, e); } }); - }), CUSTOMTHREADPOOL).join(); + }), CUSTOMTHREADPOOL).join();*/ // Process results and handle collapse in parallel results.parallelStream().forEach(result -> { @@ -1119,7 +1120,7 @@ private void prepareSearchResultAsync(AtlasSearchResult ret, DirectIndexQueryRes if (vertex == null) return; String guid = vertex.getProperty("__guid", String.class); - AtlasEntityHeader header = headers.get(guid); + AtlasEntityHeader header = entitiesSet.get(guid); if (showSearchScore) { ret.addEntityScore(header.getGuid(), result.getScore()); @@ -1230,8 +1231,9 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i if (AtlasConfiguration.ENABLE_JANUS_GRAPH_OPTIMISATION.getBoolean()) { LOG.debug("enabled janusGraphOptimisation"); prepareSearchResultAsync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); + } else { + prepareSearchResultSync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); } - prepareSearchResultSync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); } // Non-recursive collapse processing private Map processCollapseResults(Result result, SearchParams searchParams, Set resultAttributes) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java index d0dbecad38..a0e9d6eed6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java @@ -493,7 +493,7 @@ private EntityAuditEventV2 createEvent(EntityAuditEventV2 entityAuditEventV2, At String qualifiedName = (String) originalEntity.getAttribute(QUALIFIED_NAME); entityAuditEventV2.setEntityQualifiedName(AtlasType.toJson(qualifiedName)); } else { - String qualifiedName = (String) entity.getAttribute(QUALIFIED_NAME); + String qualifiedName = ((List)entity.getAttribute(QUALIFIED_NAME)).get(0).toString(); entityAuditEventV2.setEntityQualifiedName(AtlasType.toJson(qualifiedName)); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 0f11a100fc..16053afab8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.store.graph.v2; import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; @@ -49,6 +50,8 @@ import org.apache.atlas.repository.graphdb.AtlasElement; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; +import org.apache.atlas.repository.graphdb.janus.AtlasJanusVertex; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; import org.apache.atlas.type.AtlasEntityType; @@ -67,6 +70,9 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.janusgraph.core.JanusGraphMultiVertexQuery; +import org.janusgraph.core.JanusGraphTransaction; +import org.janusgraph.core.JanusGraphVertex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -1176,6 +1182,176 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, } + public List mapVerticesToAtlasEntityHeader(List entityVertices, Set attributes) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVerticesToAtlasEntityHeader"); + + List results = new ArrayList<>(); + + try { + // Convert AtlasVertex to JanusGraphVertex + + // Use multiQuery for optimized property fetching + Map> multiQueryResults = getBatchPropertiesWithMultiQuery(entityVertices, attributes); + Map map = getJanusGraphVerticesMap(entityVertices); + multiQueryResults.forEach((janusGraphVertex, vertexProperties) -> { + AtlasEntityHeader ret = new AtlasEntityHeader(); + + // Populate AtlasEntityHeader with fetched properties + try { + populateEntityHeader(ret, map.get(janusGraphVertex), vertexProperties, attributes); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); + } + results.add(ret); + }); + + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + return results; + } + + private Map getJanusGraphVerticesMap(List vertices) { + Map resultMap = new HashMap<>(); + + for (AtlasVertex vertex : vertices) { + if (vertex instanceof AtlasJanusVertex) { + Object wrappedElement = ((AtlasJanusVertex) vertex).getWrappedElement(); + + if (wrappedElement instanceof JanusGraphVertex) { + resultMap.put((JanusGraphVertex) wrappedElement, vertex); + } else { + throw new IllegalArgumentException("Wrapped element is not an instance of JanusGraphVertex"); + } + } else { + throw new IllegalArgumentException("Provided vertex is not an instance of AtlasJanusVertex"); + } + } + + return resultMap; + } + + // Helper to convert AtlasVertex to JanusGraphVertex + private List getJanusGraphVertices(List vertices) { + List results = new ArrayList<>(); + for(AtlasVertex vertex : vertices) { + if (((AtlasJanusVertex) vertex).getWrappedElement() instanceof JanusGraphVertex) { + results.add(vertex.getWrappedElement()); + } else { + throw new IllegalArgumentException("Provided vertex is not an instance of JanusGraphVertex"); + } + } + return results; + } + + // Use multiQuery to batch-fetch properties + private Map> getBatchPropertiesWithMultiQuery(List entityVertices, Set attributes) { + Iterable vertices = getJanusGraphVertices(entityVertices); + List target = new ArrayList<>(); + vertices.forEach(target::add); + JanusGraphTransaction transaction = ((AtlasJanusGraph)graph).getTransaction(); + try { + JanusGraphMultiVertexQuery multiQuery = transaction.multiQuery(target); + + Set keys = Sets.newHashSet(Constants.TYPE_NAME_PROPERTY_KEY, Constants.GUID_PROPERTY_KEY, + Constants.CREATED_BY_KEY, Constants.MODIFIED_BY_KEY, + Constants.TIMESTAMP_PROPERTY_KEY, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + + keys.addAll(attributes); + + multiQuery.keys(Constants.TYPE_NAME_PROPERTY_KEY, Constants.GUID_PROPERTY_KEY, + Constants.CREATED_BY_KEY, Constants.MODIFIED_BY_KEY, + Constants.TIMESTAMP_PROPERTY_KEY, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, STATE_PROPERTY_KEY); + + Map> vertexPropertiesMap = new HashMap<>(); + + for (JanusGraphVertex vertex : vertices) { + Map properties = new HashMap<>(); + for (String key : keys) { + properties.put(key, vertex.property(key).orElse(null)); + } + vertexPropertiesMap.put(vertex, properties); + } + return vertexPropertiesMap; + } finally { + /*if(transaction != null) { + transaction.commit(); + transaction.close(); + }*/ + } + + } + + // Populate AtlasEntityHeader + private void populateEntityHeader(AtlasEntityHeader ret, AtlasVertex entityVertex, Map vertexProperties, Set attributes) throws AtlasBaseException { + String typeName = (String) vertexProperties.get(Constants.TYPE_NAME_PROPERTY_KEY); + String guid = (String) vertexProperties.get(Constants.GUID_PROPERTY_KEY); + String createdBy = (String) vertexProperties.get(Constants.CREATED_BY_KEY); + String updatedBy = (String) vertexProperties.get(Constants.MODIFIED_BY_KEY); + Long createTime = (Long) vertexProperties.get(Constants.TIMESTAMP_PROPERTY_KEY); + Long updateTime = (Long) vertexProperties.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); + Boolean isIncomplete = isEntityIncomplete(entityVertex); + + ret.setTypeName(typeName); + ret.setGuid(guid); + ret.setStatus(GraphHelper.getStatus(entityVertex)); + ret.setIsIncomplete(isIncomplete); + ret.setCreatedBy(createdBy); + ret.setUpdatedBy(updatedBy); + ret.setCreateTime(createTime != null ? new Date(createTime) : null); + ret.setUpdateTime(updateTime != null ? new Date(updateTime) : null); + ret.setLabels(getLabels(entityVertex)); + + // Classifications + RequestContext context = RequestContext.get(); + if (context.includeClassifications() || context.isIncludeClassificationNames()) { + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } + + // Meanings + if (context.includeMeanings()) { + List termAssignmentHeaders = mapAssignedTerms(entityVertex); + ret.setMeanings(termAssignmentHeaders); + ret.setMeaningNames(termAssignmentHeaders.stream() + .map(AtlasTermAssignmentHeader::getDisplayText) + .collect(Collectors.toList())); + } + + // Process entity type and attributes + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + if (entityType != null) { + // Header attributes + for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { + Object attrValue = getVertexAttributeFromBatch(vertexProperties, headerAttribute); + if (attrValue != null) { + ret.setAttribute(headerAttribute.getName(), attrValue); + } + } + + // Display text + Object displayText = getDisplayText(entityVertex, entityType); + if (displayText != null) { + ret.setDisplayText(displayText.toString()); + } + + // Additional attributes + if (CollectionUtils.isNotEmpty(attributes)) { + for (String attrName : attributes) { + AtlasAttribute attribute = getEntityOrRelationshipAttribute(entityType, attrName); + if (attribute != null) { + Object attrValue = getVertexAttributeFromBatch(vertexProperties, attribute); + if (attrValue != null) { + ret.setAttribute(attrName, attrValue); + } + } + } + } + } + + // Additional properties like classifications, meanings, and attributes... + } + private Object getCommonProperty(Map vertexProperties, String propertyName) { if (vertexProperties.get(propertyName) instanceof List) { return ((List) vertexProperties.get(propertyName)).get(0); From cc0be01e6ca841cbb66c6d3c4b5956f9944b179b Mon Sep 17 00:00:00 2001 From: Abhijeet Kumar Date: Thu, 21 Nov 2024 14:26:36 +0530 Subject: [PATCH 072/241] some cosmetic changes --- .../store/graph/v2/EntityGraphMapper.java | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index cdb4514859..dc7122a00a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3227,6 +3227,123 @@ public void cleanUpClassificationPropagation(String classificationName, int batc LOG.info("Completed cleaning up classification {}", classificationName); } + + +// Huge change to get affected edge count, to be reviewed hence keeping in comments for future review. +// public void cleanUpClassificationPropagation(String classificationName, int batchLimit) { +// int CLEANUP_MAX = batchLimit <= 0 ? CLEANUP_BATCH_SIZE : batchLimit * CLEANUP_BATCH_SIZE; +// int cleanedUpCount = 0; +// long classificationEdgeCount = 0; // To hold the total count +// long classificationEdgeInMemoryCount = 0; +// +// // Cache vertices and their classification edges during the initial iteration +// List allTagVertices = new ArrayList<>(); +// Map vertexEdgeCounts = new HashMap<>(); +// +// Iterator tagVerticesIterator = GraphHelper.getClassificationVertices(graph, classificationName, CLEANUP_BATCH_SIZE); +// +// while (tagVerticesIterator != null && tagVerticesIterator.hasNext()) { +// AtlasVertex tagVertex = tagVerticesIterator.next(); +// long edgeCount = GraphHelper.getClassificationEdges(tagVertex, null, classificationName).size(); +// classificationEdgeCount += edgeCount; +// +// // Cache the vertex and its edge count +// allTagVertices.add(tagVertex); +// vertexEdgeCounts.put(tagVertex, edgeCount); +// } +// +// LOG.info("Total classification edges to be cleaned for {}: {}", classificationName, classificationEdgeCount); +// +// // Now use the cached vertices for cleanup +// List tagVerticesProcessed = new ArrayList<>(); +// List currentAssetVerticesBatch = new ArrayList<>(); +// +// Iterator cachedVerticesIterator = allTagVertices.iterator(); +// while (cachedVerticesIterator.hasNext()) { +// if (cleanedUpCount >= CLEANUP_MAX) { +// return; +// } +// +// while (cachedVerticesIterator.hasNext() && currentAssetVerticesBatch.size() < CLEANUP_BATCH_SIZE) { +// AtlasVertex tagVertex = cachedVerticesIterator.next(); +// +// int availableSlots = CLEANUP_BATCH_SIZE - currentAssetVerticesBatch.size(); +// long assetCountForCurrentTagVertex = GraphHelper.getAssetsCountOfClassificationVertex(tagVertex); +// currentAssetVerticesBatch.addAll(GraphHelper.getAllAssetsWithClassificationVertex(tagVertex, availableSlots)); +// LOG.info("Available slots : {}, assetCountForCurrentTagVertex : {}, queueSize : {}", availableSlots, assetCountForCurrentTagVertex, currentAssetVerticesBatch.size()); +// if (assetCountForCurrentTagVertex <= availableSlots) { +// tagVerticesProcessed.add(tagVertex); +// } +// } +// +// int currentAssetsBatchSize = currentAssetVerticesBatch.size(); +// if (currentAssetsBatchSize > 0) { +// LOG.info("To clean up tag {} from {} entities", classificationName, currentAssetsBatchSize); +// int offset = 0; +// do { +// try { +// int toIndex = Math.min((offset + CHUNK_SIZE), currentAssetsBatchSize); +// List entityVertices = currentAssetVerticesBatch.subList(offset, toIndex); +// for (AtlasVertex vertex : entityVertices) { +// List deletedClassifications = new ArrayList<>(); +// GraphTransactionInterceptor.lockObjectAndReleasePostCommit(graphHelper.getGuid(vertex)); +// List classificationEdges = GraphHelper.getClassificationEdges(vertex, null, classificationName); +// int batchSize = CHUNK_SIZE; +// for (int i = 0; i < classificationEdges.size(); i += batchSize) { +// int end = Math.min(i + batchSize, classificationEdges.size()); +// List batch = classificationEdges.subList(i, end); +// for (AtlasEdge edge : batch) { +// try { +// AtlasClassification classification = entityRetriever.toAtlasClassification(edge.getInVertex()); +// deletedClassifications.add(classification); +// deleteDelegate.getHandler().deleteEdgeReference(edge, TypeCategory.CLASSIFICATION, false, true, null, vertex); +// classificationEdgeInMemoryCount++; +// } catch (IllegalStateException | AtlasBaseException e) { +// e.printStackTrace(); +// } +// } +// if (classificationEdgeInMemoryCount >= CHUNK_SIZE) { +// transactionInterceptHelper.intercept(); +// classificationEdgeInMemoryCount = 0; +// } +// } +// try { +// AtlasEntity entity = repairClassificationMappings(vertex); +// entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassifications); +// } catch (IllegalStateException | AtlasBaseException e) { +// e.printStackTrace(); +// } +// } +// +// transactionInterceptHelper.intercept(); +// +// offset += CHUNK_SIZE; +// } finally { +// LOG.info("For offset {} , cleaned up classification edges.", offset); +// LOG.info("Cleaned up {} entities for classification {}", offset, classificationName); +// } +// +// } while (offset < currentAssetsBatchSize); +// +// for (AtlasVertex classificationVertex : tagVerticesProcessed) { +// try { +// deleteDelegate.getHandler().deleteClassificationVertex(classificationVertex, true); +// } catch (IllegalStateException e) { +// e.printStackTrace(); +// } +// } +// transactionInterceptHelper.intercept(); +// +// cleanedUpCount += currentAssetsBatchSize; +// currentAssetVerticesBatch.clear(); +// tagVerticesProcessed.clear(); +// } +// } + +// LOG.info("Completed cleaning up classification {}", classificationName); +// } + + public AtlasEntity repairClassificationMappings(AtlasVertex entityVertex) throws AtlasBaseException { String guid = GraphHelper.getGuid(entityVertex); AtlasEntity entity = instanceConverter.getEntity(guid, ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES); @@ -4059,6 +4176,15 @@ public void updateClassificationTextPropagation(String classificationVertexId) t AtlasClassification classification = entityRetriever.toAtlasClassification(classificationVertex); LOG.info("Fetched classification : {} ", classification.toString()); List impactedVertices = graphHelper.getAllPropagatedEntityVertices(classificationVertex); + + // update the 'assetsCountToPropagate' on in memory java object. + AtlasTask currentTask = RequestContext.get().getCurrentTask(); + currentTask.setAssetsCountToPropagate((long) impactedVertices.size()); + + //update the 'assetsCountToPropagate' in the current task vertex. + AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); + currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, impactedVertices.size()); + LOG.info("impactedVertices : {}", impactedVertices.size()); int batchSize = 100; for (int i = 0; i < impactedVertices.size(); i += batchSize) { From 650cfd16565ea87adbc5a87cf3c58cecef17214b Mon Sep 17 00:00:00 2001 From: aarshi Date: Thu, 21 Nov 2024 14:29:28 +0530 Subject: [PATCH 073/241] import function --- .../java/org/apache/atlas/discovery/EntityDiscoveryService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index fb932df7aa..29a33d676b 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -73,6 +73,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ForkJoinPool; +import java.util.function.Function; import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.*; From 08c0df0f7adfbada8a9a99f54e599974fa869780 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 21 Nov 2024 18:48:05 +0530 Subject: [PATCH 074/241] revert to previous version of mapVertexToAtlasEntityHeader --- .../discovery/EntityDiscoveryService.java | 118 ------ .../store/graph/v2/EntityGraphRetriever.java | 357 ++---------------- 2 files changed, 37 insertions(+), 438 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 29a33d676b..905256da93 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -70,10 +70,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptException; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; -import java.util.function.Function; import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.*; @@ -1075,87 +1071,6 @@ public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) thro } } - @SuppressWarnings("rawtypes") - private void prepareSearchResultAsync(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { - SearchParams searchParams = ret.getSearchParameters(); - boolean showSearchScore = searchParams.getShowSearchScore(); - List results = new ArrayList<>(); - - // Collect results for batch processing - Iterator iterator = indexQueryResult.getIterator(); - while (iterator != null && iterator.hasNext()) { - results.add(iterator.next()); - } - - // Batch fetch vertices - List vertices = results.stream() - .map(Result::getVertex) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - // Use ConcurrentHashMap for thread-safe access - //ConcurrentHashMap headers = new ConcurrentHashMap<>(); - //ConcurrentHashMap entitiesSet = new ConcurrentHashMap<>(); - - List headers = entityRetriever.mapVerticesToAtlasEntityHeader(vertices, resultAttributes); - // Create a Set entitiesSet = headers.stream().collect(Collectors.toMap(AtlasEntityHeader::getGuid, Function.identity())); - - // Run vertex processing in limited parallel threads - /**CompletableFuture.runAsync(() -> vertices.parallelStream().forEach(vertex -> { - String guid = vertex.getProperty("__guid", String.class); - headers.computeIfAbsent(guid, k -> { - try { - //AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); - if (RequestContext.get().includeClassifications()) { - header.setClassifications(entityRetriever.getAllClassifications(vertex)); - } - return header; - } catch (AtlasBaseException e) { - throw new RuntimeException("Failed to process vertex with GUID: " + guid, e); - } - }); - }), CUSTOMTHREADPOOL).join();*/ - - // Process results and handle collapse in parallel - results.parallelStream().forEach(result -> { - AtlasVertex vertex = result.getVertex(); - if (vertex == null) return; - - String guid = vertex.getProperty("__guid", String.class); - AtlasEntityHeader header = entitiesSet.get(guid); - - if (showSearchScore) { - ret.addEntityScore(header.getGuid(), result.getScore()); - } - - if (fetchCollapsedResults) { - Map collapse; - try { - collapse = processCollapseResults(result, searchParams, resultAttributes); - } catch (AtlasBaseException e) { - throw new RuntimeException(e); - } - if (!collapse.isEmpty()) { - header.setCollapse(collapse); - } - } - - if (searchParams.getShowSearchMetadata()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); - ret.addSort(header.getGuid(), result.getSort()); - } else if (searchParams.getShowHighlights()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); - } - - if (header != null) { - entitiesSet.put(header.getGuid(), header); - } - }); - ret.setEntities(new ArrayList<>(entitiesSet.values())); - scrubSearchResults(ret, searchParams.getSuppressLogs()); - } - private void prepareSearchResultSync(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { SearchParams searchParams = ret.getSearchParameters(); try { @@ -1231,40 +1146,7 @@ private void prepareSearchResultSync(AtlasSearchResult ret, DirectIndexQueryResu } private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { - if (AtlasConfiguration.ENABLE_JANUS_GRAPH_OPTIMISATION.getBoolean()) { - LOG.debug("enabled janusGraphOptimisation"); - prepareSearchResultAsync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); - } else { prepareSearchResultSync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); - } - } - // Non-recursive collapse processing - private Map processCollapseResults(Result result, SearchParams searchParams, Set resultAttributes) throws AtlasBaseException { - Map collapse = new HashMap<>(); - Set collapseKeys = result.getCollapseKeys(); - - for (String collapseKey : collapseKeys) { - AtlasSearchResult collapseRet = new AtlasSearchResult(); - collapseRet.setSearchParameters(searchParams); - Set collapseResultAttributes = new HashSet<>(Optional.ofNullable(searchParams.getCollapseAttributes()).orElse(resultAttributes)); - DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); - collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); - - // Directly iterate over collapse vertices - Iterator iterator = indexQueryCollapsedResult.getIterator(); - while (iterator != null && iterator.hasNext()) { - Result collapseResult = iterator.next(); - AtlasVertex collapseVertex = collapseResult.getVertex(); - if (collapseVertex == null) continue; - - AtlasEntityHeader collapseHeader = entityRetriever.toAtlasEntityHeader(collapseVertex, collapseResultAttributes); - collapseRet.addEntity(collapseHeader); - } - - collapse.put(collapseKey, collapseRet); - } - - return collapse; } private Map getMap(String key, Object value) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 3782982bc9..3b8144fed3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -18,7 +18,6 @@ package org.apache.atlas.repository.store.graph.v2; import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; @@ -50,8 +49,6 @@ import org.apache.atlas.repository.graphdb.AtlasElement; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; -import org.apache.atlas.repository.graphdb.janus.AtlasJanusVertex; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; import org.apache.atlas.type.AtlasEntityType; @@ -70,9 +67,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; -import org.janusgraph.core.JanusGraphMultiVertexQuery; -import org.janusgraph.core.JanusGraphTransaction; -import org.janusgraph.core.JanusGraphVertex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -1015,330 +1009,77 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } -// private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { -// AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); -// AtlasEntityHeader ret = new AtlasEntityHeader(); -// try { -// String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); -// String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); -// Boolean isIncomplete = isEntityIncomplete(entityVertex); -// -// ret.setTypeName(typeName); -// ret.setGuid(guid); -// ret.setStatus(GraphHelper.getStatus(entityVertex)); -// RequestContext context = RequestContext.get(); -// boolean includeClassifications = context.includeClassifications(); -// boolean includeClassificationNames = context.isIncludeClassificationNames(); -// if(includeClassifications){ -// ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); -// } else if (!includeClassifications && includeClassificationNames) { -// ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); -// } -// ret.setIsIncomplete(isIncomplete); -// ret.setLabels(getLabels(entityVertex)); -// -// ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); -// ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); -// ret.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); -// ret.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); -// -// if(RequestContext.get().includeMeanings()) { -// List termAssignmentHeaders = mapAssignedTerms(entityVertex); -// ret.setMeanings(termAssignmentHeaders); -// ret.setMeaningNames( -// termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) -// .collect(Collectors.toList())); -// } -// AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); -// -// if (entityType != null) { -// for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { -// Object attrValue = getVertexAttribute(entityVertex, headerAttribute); -// -// if (attrValue != null) { -// ret.setAttribute(headerAttribute.getName(), attrValue); -// } -// } -// -// Object displayText = getDisplayText(entityVertex, entityType); -// -// if (displayText != null) { -// ret.setDisplayText(displayText.toString()); -// } -// -// if (CollectionUtils.isNotEmpty(attributes)) { -// for (String attrName : attributes) { -// AtlasAttribute attribute = entityType.getAttribute(attrName); -// -// if (attribute == null) { -// attrName = toNonQualifiedName(attrName); -// -// if (ret.hasAttribute(attrName)) { -// continue; -// } -// -// attribute = entityType.getAttribute(attrName); -// -// if (attribute == null) { -// attribute = entityType.getRelationshipAttribute(attrName, null); -// } -// } -// -// Object attrValue = getVertexAttribute(entityVertex, attribute); -// -// if (attrValue != null) { -// ret.setAttribute(attrName, attrValue); -// } -// } -// } -// } -// } -// finally { -// RequestContext.get().endMetricRecord(metricRecorder); -// } -// return ret; -// } - - private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); - AtlasEntityHeader ret = new AtlasEntityHeader(); - try { - // Batch fetch all properties - Map vertexProperties = batchGetProperties(entityVertex); - - // Common properties - String typeName = (String) getCommonProperty(vertexProperties, Constants.TYPE_NAME_PROPERTY_KEY); - String guid = (String) getCommonProperty(vertexProperties, GUID_PROPERTY_KEY); - Long createTime = (Long) getCommonProperty(vertexProperties, TIMESTAMP_PROPERTY_KEY); - Long updateTime = (Long) getCommonProperty(vertexProperties, MODIFICATION_TIMESTAMP_PROPERTY_KEY); + String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); + String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); Boolean isIncomplete = isEntityIncomplete(entityVertex); - // Set properties in header ret.setTypeName(typeName); ret.setGuid(guid); ret.setStatus(GraphHelper.getStatus(entityVertex)); - ret.setIsIncomplete(isIncomplete); - ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); - ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); - ret.setCreateTime(createTime != null ? new Date(createTime) : null); - ret.setUpdateTime(updateTime != null ? new Date(updateTime) : null); - ret.setLabels(getLabels(entityVertex)); - - // Classifications RequestContext context = RequestContext.get(); - if (context.includeClassifications() || context.isIncludeClassificationNames()) { + boolean includeClassifications = context.includeClassifications(); + boolean includeClassificationNames = context.isIncludeClassificationNames(); + if(includeClassifications){ + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } else if (!includeClassifications && includeClassificationNames) { ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); } + ret.setIsIncomplete(isIncomplete); + ret.setLabels(getLabels(entityVertex)); - // Meanings - if (context.includeMeanings()) { + ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); + ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); + ret.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); + ret.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); + + if(RequestContext.get().includeMeanings()) { List termAssignmentHeaders = mapAssignedTerms(entityVertex); ret.setMeanings(termAssignmentHeaders); - ret.setMeaningNames(termAssignmentHeaders.stream() - .map(AtlasTermAssignmentHeader::getDisplayText) - .collect(Collectors.toList())); + ret.setMeaningNames( + termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) + .collect(Collectors.toList())); } - - // Process entity type and attributes AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + if (entityType != null) { - // Header attributes for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { - Object attrValue = getVertexAttributeFromBatch(vertexProperties, headerAttribute); + Object attrValue = getVertexAttribute(entityVertex, headerAttribute); + if (attrValue != null) { ret.setAttribute(headerAttribute.getName(), attrValue); } } - // Display text Object displayText = getDisplayText(entityVertex, entityType); + if (displayText != null) { ret.setDisplayText(displayText.toString()); } - // Additional attributes if (CollectionUtils.isNotEmpty(attributes)) { for (String attrName : attributes) { - AtlasAttribute attribute = getEntityOrRelationshipAttribute(entityType, attrName); - if (attribute != null) { - Object attrValue = getVertexAttributeFromBatch(vertexProperties, attribute); - if (attrValue != null) { - ret.setAttribute(attrName, attrValue); - } - } - } - } - } - } finally { - RequestContext.get().endMetricRecord(metricRecorder); - } - - return ret; - } - - - public List mapVerticesToAtlasEntityHeader(List entityVertices, Set attributes) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVerticesToAtlasEntityHeader"); - - List results = new ArrayList<>(); - - try { - // Convert AtlasVertex to JanusGraphVertex - - // Use multiQuery for optimized property fetching - Map> multiQueryResults = getBatchPropertiesWithMultiQuery(entityVertices, attributes); - Map map = getJanusGraphVerticesMap(entityVertices); - multiQueryResults.forEach((janusGraphVertex, vertexProperties) -> { - AtlasEntityHeader ret = new AtlasEntityHeader(); - - // Populate AtlasEntityHeader with fetched properties - try { - populateEntityHeader(ret, map.get(janusGraphVertex), vertexProperties, attributes); - } catch (AtlasBaseException e) { - throw new RuntimeException(e); - } - results.add(ret); - }); - - } finally { - RequestContext.get().endMetricRecord(metricRecorder); - } - - return results; - } - - private Map getJanusGraphVerticesMap(List vertices) { - Map resultMap = new HashMap<>(); + AtlasAttribute attribute = entityType.getAttribute(attrName); - for (AtlasVertex vertex : vertices) { - if (vertex instanceof AtlasJanusVertex) { - Object wrappedElement = ((AtlasJanusVertex) vertex).getWrappedElement(); + if (attribute == null) { + attrName = toNonQualifiedName(attrName); - if (wrappedElement instanceof JanusGraphVertex) { - resultMap.put((JanusGraphVertex) wrappedElement, vertex); - } else { - throw new IllegalArgumentException("Wrapped element is not an instance of JanusGraphVertex"); - } - } else { - throw new IllegalArgumentException("Provided vertex is not an instance of AtlasJanusVertex"); - } - } - - return resultMap; - } - - // Helper to convert AtlasVertex to JanusGraphVertex - private List getJanusGraphVertices(List vertices) { - List results = new ArrayList<>(); - for(AtlasVertex vertex : vertices) { - if (((AtlasJanusVertex) vertex).getWrappedElement() instanceof JanusGraphVertex) { - results.add(vertex.getWrappedElement()); - } else { - throw new IllegalArgumentException("Provided vertex is not an instance of JanusGraphVertex"); - } - } - return results; - } - - // Use multiQuery to batch-fetch properties - private Map> getBatchPropertiesWithMultiQuery(List entityVertices, Set attributes) { - Iterable vertices = getJanusGraphVertices(entityVertices); - List target = new ArrayList<>(); - vertices.forEach(target::add); - JanusGraphTransaction transaction = ((AtlasJanusGraph)graph).getTransaction(); - try { - JanusGraphMultiVertexQuery multiQuery = transaction.multiQuery(target); - - Set keys = Sets.newHashSet(Constants.TYPE_NAME_PROPERTY_KEY, Constants.GUID_PROPERTY_KEY, - Constants.CREATED_BY_KEY, Constants.MODIFIED_BY_KEY, - Constants.TIMESTAMP_PROPERTY_KEY, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); - - keys.addAll(attributes); - - multiQuery.keys(Constants.TYPE_NAME_PROPERTY_KEY, Constants.GUID_PROPERTY_KEY, - Constants.CREATED_BY_KEY, Constants.MODIFIED_BY_KEY, - Constants.TIMESTAMP_PROPERTY_KEY, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, STATE_PROPERTY_KEY); + if (ret.hasAttribute(attrName)) { + continue; + } - Map> vertexPropertiesMap = new HashMap<>(); + attribute = entityType.getAttribute(attrName); - for (JanusGraphVertex vertex : vertices) { - Map properties = new HashMap<>(); - for (String key : keys) { - properties.put(key, vertex.property(key).orElse(null)); - } - vertexPropertiesMap.put(vertex, properties); - } - return vertexPropertiesMap; - } finally { - /*if(transaction != null) { - transaction.commit(); - transaction.close(); - }*/ - } - - } - - // Populate AtlasEntityHeader - private void populateEntityHeader(AtlasEntityHeader ret, AtlasVertex entityVertex, Map vertexProperties, Set attributes) throws AtlasBaseException { - String typeName = (String) vertexProperties.get(Constants.TYPE_NAME_PROPERTY_KEY); - String guid = (String) vertexProperties.get(Constants.GUID_PROPERTY_KEY); - String createdBy = (String) vertexProperties.get(Constants.CREATED_BY_KEY); - String updatedBy = (String) vertexProperties.get(Constants.MODIFIED_BY_KEY); - Long createTime = (Long) vertexProperties.get(Constants.TIMESTAMP_PROPERTY_KEY); - Long updateTime = (Long) vertexProperties.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); - Boolean isIncomplete = isEntityIncomplete(entityVertex); - - ret.setTypeName(typeName); - ret.setGuid(guid); - ret.setStatus(GraphHelper.getStatus(entityVertex)); - ret.setIsIncomplete(isIncomplete); - ret.setCreatedBy(createdBy); - ret.setUpdatedBy(updatedBy); - ret.setCreateTime(createTime != null ? new Date(createTime) : null); - ret.setUpdateTime(updateTime != null ? new Date(updateTime) : null); - ret.setLabels(getLabels(entityVertex)); - - // Classifications - RequestContext context = RequestContext.get(); - if (context.includeClassifications() || context.isIncludeClassificationNames()) { - ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); - } - - // Meanings - if (context.includeMeanings()) { - List termAssignmentHeaders = mapAssignedTerms(entityVertex); - ret.setMeanings(termAssignmentHeaders); - ret.setMeaningNames(termAssignmentHeaders.stream() - .map(AtlasTermAssignmentHeader::getDisplayText) - .collect(Collectors.toList())); - } - - // Process entity type and attributes - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); - if (entityType != null) { - // Header attributes - for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { - Object attrValue = getVertexAttributeFromBatch(vertexProperties, headerAttribute); - if (attrValue != null) { - ret.setAttribute(headerAttribute.getName(), attrValue); - } - } + if (attribute == null) { + attribute = entityType.getRelationshipAttribute(attrName, null); + } + } - // Display text - Object displayText = getDisplayText(entityVertex, entityType); - if (displayText != null) { - ret.setDisplayText(displayText.toString()); - } + Object attrValue = getVertexAttribute(entityVertex, attribute); - // Additional attributes - if (CollectionUtils.isNotEmpty(attributes)) { - for (String attrName : attributes) { - AtlasAttribute attribute = getEntityOrRelationshipAttribute(entityType, attrName); - if (attribute != null) { - Object attrValue = getVertexAttributeFromBatch(vertexProperties, attribute); if (attrValue != null) { ret.setAttribute(attrName, attrValue); } @@ -1346,34 +1087,10 @@ private void populateEntityHeader(AtlasEntityHeader ret, AtlasVertex entityVerte } } } - - // Additional properties like classifications, meanings, and attributes... - } - - private Object getCommonProperty(Map vertexProperties, String propertyName) { - if (vertexProperties.get(propertyName) instanceof List) { - return ((List) vertexProperties.get(propertyName)).get(0); - } - return new Object(); - } - /** - * Fetches all properties of a vertex in one call and returns them as a map. - */ - private Map batchGetProperties(AtlasVertex vertex) { - // Use JanusGraph's Gremlin API for efficient property retrieval - return (Map) graph.V(vertex.getId()).valueMap().next(); - } - - /** - * Retrieves a vertex attribute from the pre-fetched batch of properties. - */ - private Object getVertexAttributeFromBatch(Map properties, AtlasAttribute attribute) { - if (properties == null || attribute == null) { - return null; + finally { + RequestContext.get().endMetricRecord(metricRecorder); } - - String propertyKey = attribute.getVertexPropertyName(); - return properties.get(propertyKey); + return ret; } /** From f20e85958e567760f30f4134b0bccdb7b072df18 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 21 Nov 2024 19:47:55 +0530 Subject: [PATCH 075/241] remove the mapVertex method that uses valueMap --- .../store/graph/v2/EntityGraphRetriever.java | 167 +++++------------- 1 file changed, 43 insertions(+), 124 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 3782982bc9..be52870d4f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1015,171 +1015,90 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } -// private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { -// AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); -// AtlasEntityHeader ret = new AtlasEntityHeader(); -// try { -// String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); -// String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); -// Boolean isIncomplete = isEntityIncomplete(entityVertex); -// -// ret.setTypeName(typeName); -// ret.setGuid(guid); -// ret.setStatus(GraphHelper.getStatus(entityVertex)); -// RequestContext context = RequestContext.get(); -// boolean includeClassifications = context.includeClassifications(); -// boolean includeClassificationNames = context.isIncludeClassificationNames(); -// if(includeClassifications){ -// ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); -// } else if (!includeClassifications && includeClassificationNames) { -// ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); -// } -// ret.setIsIncomplete(isIncomplete); -// ret.setLabels(getLabels(entityVertex)); -// -// ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); -// ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); -// ret.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); -// ret.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); -// -// if(RequestContext.get().includeMeanings()) { -// List termAssignmentHeaders = mapAssignedTerms(entityVertex); -// ret.setMeanings(termAssignmentHeaders); -// ret.setMeaningNames( -// termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) -// .collect(Collectors.toList())); -// } -// AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); -// -// if (entityType != null) { -// for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { -// Object attrValue = getVertexAttribute(entityVertex, headerAttribute); -// -// if (attrValue != null) { -// ret.setAttribute(headerAttribute.getName(), attrValue); -// } -// } -// -// Object displayText = getDisplayText(entityVertex, entityType); -// -// if (displayText != null) { -// ret.setDisplayText(displayText.toString()); -// } -// -// if (CollectionUtils.isNotEmpty(attributes)) { -// for (String attrName : attributes) { -// AtlasAttribute attribute = entityType.getAttribute(attrName); -// -// if (attribute == null) { -// attrName = toNonQualifiedName(attrName); -// -// if (ret.hasAttribute(attrName)) { -// continue; -// } -// -// attribute = entityType.getAttribute(attrName); -// -// if (attribute == null) { -// attribute = entityType.getRelationshipAttribute(attrName, null); -// } -// } -// -// Object attrValue = getVertexAttribute(entityVertex, attribute); -// -// if (attrValue != null) { -// ret.setAttribute(attrName, attrValue); -// } -// } -// } -// } -// } -// finally { -// RequestContext.get().endMetricRecord(metricRecorder); -// } -// return ret; -// } - - private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); - AtlasEntityHeader ret = new AtlasEntityHeader(); - try { - // Batch fetch all properties - Map vertexProperties = batchGetProperties(entityVertex); - - // Common properties - String typeName = (String) getCommonProperty(vertexProperties, Constants.TYPE_NAME_PROPERTY_KEY); - String guid = (String) getCommonProperty(vertexProperties, GUID_PROPERTY_KEY); - Long createTime = (Long) getCommonProperty(vertexProperties, TIMESTAMP_PROPERTY_KEY); - Long updateTime = (Long) getCommonProperty(vertexProperties, MODIFICATION_TIMESTAMP_PROPERTY_KEY); + String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); + String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); Boolean isIncomplete = isEntityIncomplete(entityVertex); - // Set properties in header ret.setTypeName(typeName); ret.setGuid(guid); ret.setStatus(GraphHelper.getStatus(entityVertex)); - ret.setIsIncomplete(isIncomplete); - ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); - ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); - ret.setCreateTime(createTime != null ? new Date(createTime) : null); - ret.setUpdateTime(updateTime != null ? new Date(updateTime) : null); - ret.setLabels(getLabels(entityVertex)); - - // Classifications RequestContext context = RequestContext.get(); - if (context.includeClassifications() || context.isIncludeClassificationNames()) { + boolean includeClassifications = context.includeClassifications(); + boolean includeClassificationNames = context.isIncludeClassificationNames(); + if(includeClassifications){ + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } else if (!includeClassifications && includeClassificationNames) { ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); } + ret.setIsIncomplete(isIncomplete); + ret.setLabels(getLabels(entityVertex)); + + ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); + ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); + ret.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); + ret.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); - // Meanings - if (context.includeMeanings()) { + if(RequestContext.get().includeMeanings()) { List termAssignmentHeaders = mapAssignedTerms(entityVertex); ret.setMeanings(termAssignmentHeaders); - ret.setMeaningNames(termAssignmentHeaders.stream() - .map(AtlasTermAssignmentHeader::getDisplayText) - .collect(Collectors.toList())); + ret.setMeaningNames( + termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) + .collect(Collectors.toList())); } - - // Process entity type and attributes AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + if (entityType != null) { - // Header attributes for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { - Object attrValue = getVertexAttributeFromBatch(vertexProperties, headerAttribute); + Object attrValue = getVertexAttribute(entityVertex, headerAttribute); + if (attrValue != null) { ret.setAttribute(headerAttribute.getName(), attrValue); } } - // Display text Object displayText = getDisplayText(entityVertex, entityType); + if (displayText != null) { ret.setDisplayText(displayText.toString()); } - // Additional attributes if (CollectionUtils.isNotEmpty(attributes)) { for (String attrName : attributes) { - AtlasAttribute attribute = getEntityOrRelationshipAttribute(entityType, attrName); - if (attribute != null) { - Object attrValue = getVertexAttributeFromBatch(vertexProperties, attribute); - if (attrValue != null) { - ret.setAttribute(attrName, attrValue); + AtlasAttribute attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attrName = toNonQualifiedName(attrName); + + if (ret.hasAttribute(attrName)) { + continue; + } + + attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attribute = entityType.getRelationshipAttribute(attrName, null); } } + + Object attrValue = getVertexAttribute(entityVertex, attribute); + + if (attrValue != null) { + ret.setAttribute(attrName, attrValue); + } } } } - } finally { + } + finally { RequestContext.get().endMetricRecord(metricRecorder); } - return ret; } - public List mapVerticesToAtlasEntityHeader(List entityVertices, Set attributes) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVerticesToAtlasEntityHeader"); From 0022dae4a3d58d369211951955ab0170b4e11ac4 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 21 Nov 2024 19:48:43 +0530 Subject: [PATCH 076/241] build this branch as docker image --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..6227a0cf64 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - janusoptimisation jobs: build: From 1be36584941c5f6b0c1e62c8f2a8430d32543cad Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 21 Nov 2024 22:53:58 +0530 Subject: [PATCH 077/241] close thread bound janusgraph transaction --- .../store/graph/v2/EntityGraphRetriever.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index be52870d4f..36592b6655 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1192,10 +1192,10 @@ private Map> getBatchPropertiesWithMultiQu } return vertexPropertiesMap; } finally { - /*if(transaction != null) { + if(transaction != null) { transaction.commit(); transaction.close(); - }*/ + } } } @@ -1269,20 +1269,6 @@ private void populateEntityHeader(AtlasEntityHeader ret, AtlasVertex entityVerte // Additional properties like classifications, meanings, and attributes... } - private Object getCommonProperty(Map vertexProperties, String propertyName) { - if (vertexProperties.get(propertyName) instanceof List) { - return ((List) vertexProperties.get(propertyName)).get(0); - } - return new Object(); - } - /** - * Fetches all properties of a vertex in one call and returns them as a map. - */ - private Map batchGetProperties(AtlasVertex vertex) { - // Use JanusGraph's Gremlin API for efficient property retrieval - return (Map) graph.V(vertex.getId()).valueMap().next(); - } - /** * Retrieves a vertex attribute from the pre-fetched batch of properties. */ From d73d858505668b7c8b221a8907e054f6bca1e3d1 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 21 Nov 2024 23:57:23 +0530 Subject: [PATCH 078/241] revert entity audit listener --- .../apache/atlas/repository/audit/EntityAuditListenerV2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java index a0e9d6eed6..b6b3fd4d44 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java @@ -493,7 +493,7 @@ private EntityAuditEventV2 createEvent(EntityAuditEventV2 entityAuditEventV2, At String qualifiedName = (String) originalEntity.getAttribute(QUALIFIED_NAME); entityAuditEventV2.setEntityQualifiedName(AtlasType.toJson(qualifiedName)); } else { - String qualifiedName = ((List)entity.getAttribute(QUALIFIED_NAME)).get(0).toString(); + String qualifiedName = (String)entity.getAttribute(QUALIFIED_NAME); entityAuditEventV2.setEntityQualifiedName(AtlasType.toJson(qualifiedName)); } From be1a41b357d24bd5cbdd38c73b296b57f50e4d8d Mon Sep 17 00:00:00 2001 From: Arnab Saha Date: Fri, 22 Nov 2024 11:56:52 +0530 Subject: [PATCH 079/241] Dummy commit Dummy commit --- README.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.txt b/README.txt index 0c4dad9681..4e6db235a3 100755 --- a/README.txt +++ b/README.txt @@ -72,3 +72,5 @@ Build Process distro/target/apache-atlas--falcon-hook.tar.gz 4. For more details on installing and running Apache Atlas, please refer to https://atlas.apache.org/#/Installation. + + From da886c37197acc42074dee3f11fea2ea2c45ef61 Mon Sep 17 00:00:00 2001 From: Hitesh Khandelwal Date: Fri, 22 Nov 2024 12:51:07 +0530 Subject: [PATCH 080/241] debug --- .github/workflows/maven.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 6227a0cf64..9a67d5806d 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -49,8 +49,9 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: get_branch + run: | + echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + echo BRANCH_NAME=$BRANCH_NAME - name: Create Maven Settings uses: s4u/maven-settings-action@v2.8.0 @@ -78,8 +79,12 @@ jobs: shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" - id: get_version + # run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: | + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7)abcd" + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }})" + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" >> $GITHUB_ENV - name: Set up Buildx id: buildx From c628a605666a714b54e5afed34daf5f44721f575 Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 22 Nov 2024 13:13:35 +0530 Subject: [PATCH 081/241] fix --- .github/workflows/maven.yml | 2 +- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index ed4b5eaff7..afec7cf4e5 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - lineageondemand + - dg1908 jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index c85b48d034..e6d89ae702 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -17,7 +17,7 @@ */ package org.apache.atlas.repository.store.graph.v2; -import com.datastax.oss.driver.shaded.guava.common.collect.Sets; +import com.google.common.collect.Sets; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.atlas.AtlasConfiguration; From 0af18e1f1cec5705ec9948a06bd9e1291dd4ac12 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 22 Nov 2024 23:56:25 +0530 Subject: [PATCH 082/241] multi query with janusgraph 1.0.0 --- .../atlas/repository/graphdb/AtlasGraph.java | 4 ---- graphdb/janus/pom.xml | 4 ++++ .../graphdb/janus/AtlasJanusGraph.java | 8 -------- .../graphdb/janus/AtlasJanusGraphDatabase.java | 2 +- .../graphdb/janus/AtlasJanusIndexQuery.java | 6 +++--- .../diskstorage/solr/Solr6Index.java | 13 ++++++++++++- .../org/apache/atlas/AtlasConfiguration.java | 2 +- pom.xml | 18 ++++-------------- repository/pom.xml | 14 +++++++++++++- .../repository/graph/IndexRecoveryService.java | 1 - .../store/graph/v2/AtlasEntityStoreV2.java | 1 - .../apache/atlas/services/MetricsService.java | 1 - .../apache/atlas/util/AtlasMetricsUtil.java | 5 ----- .../web/service/AtlasDebugMetricsSink.java | 10 ++++++---- 14 files changed, 44 insertions(+), 45 deletions(-) diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java index 666ab376a7..8b7a18695d 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java @@ -384,8 +384,4 @@ public interface AtlasGraph { */ AtlasGraphIndexClient getGraphIndexClient()throws AtlasException; - - void setEnableCache(boolean enableCache); - - Boolean isCacheEnabled(); } diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 75c9079eee..6497397fb9 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -95,6 +95,10 @@ org.apache.tinkerpop gremlin-driver + + cassandra-hadoop-util + * + diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java index 9bd2fbcdcd..7aca9ebe82 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java @@ -684,14 +684,6 @@ private Edge getFirstActiveEdge(GraphTraversal gt) { return null; } - public void setEnableCache(boolean enableCache) { - this.janusGraph.setEnableCache(enableCache); - } - - public Boolean isCacheEnabled() { - return this.janusGraph.isCacheEnabled(); - } - public JanusGraphTransaction getTransaction() { return this.janusGraph.newThreadBoundTransaction(); } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java index db69d77ee3..88a989241c 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java @@ -90,7 +90,7 @@ public class AtlasJanusGraphDatabase implements GraphDatabase directIndexQuery(String query) throws AtlasBaseExcept @Override public Iterator> vertices() { - Iterator> results = query.vertices().iterator(); + Iterator> results = query.vertexStream().iterator(); Function, Result> function = new Function, Result>() { @@ -77,7 +77,7 @@ public Iterator> vertices(int offset, i Iterator> results = query .offset(offset) .limit(limit) - .vertices().iterator(); + .vertexStream().iterator(); Function, Result> function = new Function, Result>() { @@ -100,7 +100,7 @@ public Iterator> vertices(int offset, i .orderBy(sortBy, sortOrder) .offset(offset) .limit(limit) - .vertices().iterator(); + .vertexStream().iterator(); Function, Result> function = new Function, Result>() { diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java index 23c11de301..3f5b3688ba 100644 --- a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java +++ b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java @@ -121,6 +121,7 @@ import org.janusgraph.graphdb.query.condition.Not; import org.janusgraph.graphdb.query.condition.Or; import org.janusgraph.graphdb.query.condition.PredicateCondition; +import org.janusgraph.graphdb.tinkerpop.optimize.step.Aggregation; import org.janusgraph.graphdb.types.ParameterType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -672,7 +673,7 @@ public Stream query(IndexQuery query, KeyInformation.IndexRetriever info doc -> doc.getFieldValue(keyIdField).toString()); } - @Override + //@Override public Long queryCount(IndexQuery query, KeyInformation.IndexRetriever information, BaseTransaction tx) throws BackendException { try { String collection = query.getStore(); @@ -1178,6 +1179,16 @@ public boolean exists() throws BackendException { } } + @Override + public Number queryAggregation(IndexQuery indexQuery, KeyInformation.IndexRetriever indexRetriever, BaseTransaction baseTransaction, Aggregation aggregation) throws BackendException { + return null; + } + + @Override + public void clearStore(String s) throws BackendException { + + } + /* ################# UTILITY METHODS ####################### */ diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 9390f3959a..6735cb4c8a 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -122,7 +122,7 @@ public enum AtlasConfiguration { /*** * enables parallel processing of janus graph vertices from cassandra */ - ENABLE_JANUS_GRAPH_OPTIMISATION("atlas.janus.graph.optimisation.enable", false), + ENABLE_JANUS_GRAPH_OPTIMISATION("atlas.janus.graph.optimisation.enable", true), /** * No. of threads to be spawned for parallel processing diff --git a/pom.xml b/pom.xml index 45bbee7382..bee12727c5 100644 --- a/pom.xml +++ b/pom.xml @@ -697,7 +697,7 @@ 4.3.0 1.8 3.2.2 - 7.16.2 + 7.17.4 org.apache.atlas.repository.audit.InMemoryEntityAuditRepository 2.13.2 2.18.1 @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.03 + 1.0.0 0.5.3 1 3.1.0 @@ -773,7 +773,7 @@ 2C 3.0.0-M5 6.9.4 - 3.5.1 + 3.7.0 5.0.3 3.5.5 3.20.1 @@ -836,16 +836,6 @@ false - - github - https://maven.pkg.github.com/atlanhq/atlan-janusgraph - - true - - - true - - hortonworks.repo https://repo.hortonworks.com/content/repositories/releases @@ -1806,7 +1796,7 @@ com.github.eirslett frontend-maven-plugin - 1.4 + 1.11.0 diff --git a/repository/pom.xml b/repository/pom.xml index bbe15338f8..0c8a7d9c92 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -59,6 +59,18 @@ atlas-graphdb-api + + org.apache.tinkerpop + gremlin-util + ${tinkerpop.version} + + + + org.jctools + jctools-core + 4.0.1 + + org.antlr antlr4-runtime @@ -111,7 +123,7 @@ io.netty - * + netty-handler org.slf4j diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java b/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java index c39876d19c..4600699149 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/IndexRecoveryService.java @@ -228,7 +228,6 @@ private void startMonitoring() { try { startTime = recoveryInfoManagement.getStartTime(); Instant newStartTime = Instant.now(); - this.graph.setEnableCache(false); txRecoveryObject = this.graph.getManagementSystem().startIndexRecovery(startTime); recoveryInfoManagement.updateStartTime(newStartTime.toEpochMilli()); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index f48d206653..c2129216c7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1589,7 +1589,6 @@ private void executePreProcessor(EntityMutationContext context) throws AtlasBase private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, EntityGraphMapper entityGraphMapper, boolean isPartialUpdate) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("preCreateOrUpdate"); - this.graph.setEnableCache(RequestContext.get().isCacheEnabled()); EntityGraphDiscovery graphDiscoverer = new AtlasEntityGraphDiscoveryV2(graph, typeRegistry, entityStream, entityGraphMapper); EntityGraphDiscoveryContext discoveryContext = graphDiscoverer.discoverEntities(); EntityMutationContext context = new EntityMutationContext(discoveryContext); diff --git a/repository/src/main/java/org/apache/atlas/services/MetricsService.java b/repository/src/main/java/org/apache/atlas/services/MetricsService.java index 5a8f445348..f8001720dd 100644 --- a/repository/src/main/java/org/apache/atlas/services/MetricsService.java +++ b/repository/src/main/java/org/apache/atlas/services/MetricsService.java @@ -98,7 +98,6 @@ public MetricsService(final AtlasGraph graph, final AtlasTypeRegistry typeRegist @SuppressWarnings("unchecked") @GraphTransaction public AtlasMetrics getMetrics() { - this.atlasGraph.setEnableCache(false); final AtlasTypesDef typesDef = getTypesDef(); Collection entityDefs = typesDef.getEntityDefs(); diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java b/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java index 971c4d6404..224a8a5120 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java @@ -194,13 +194,10 @@ public Map getStats() { private boolean getBackendStoreStatus(){ try { - boolean isCacheEnabled = this.graph.isCacheEnabled(); runWithTimeout(new Runnable() { @Override public void run() { - graph.setEnableCache(isCacheEnabled); graph.query().has(TYPE_NAME_PROPERTY_KEY, TYPE_NAME_INTERNAL).vertices(1); - graphCommit(); } }, 10, TimeUnit.SECONDS); @@ -219,11 +216,9 @@ private boolean getIndexStoreStatus(){ final String query = AtlasGraphUtilsV2.getIndexSearchPrefix() + "\"" + Constants.TYPE_NAME_PROPERTY_KEY + "\":(" + TYPE_NAME_INTERNAL + ")"; try { - boolean isCacheEnabled = this.graph.isCacheEnabled(); runWithTimeout(new Runnable() { @Override public void run() { - graph.setEnableCache(isCacheEnabled); graph.indexQuery(Constants.VERTEX_INDEX, query).vertices(0, 1); graphCommit(); diff --git a/webapp/src/main/java/org/apache/atlas/web/service/AtlasDebugMetricsSink.java b/webapp/src/main/java/org/apache/atlas/web/service/AtlasDebugMetricsSink.java index ef24b861ea..39ee4b7f36 100644 --- a/webapp/src/main/java/org/apache/atlas/web/service/AtlasDebugMetricsSink.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/AtlasDebugMetricsSink.java @@ -19,6 +19,7 @@ import org.apache.atlas.web.model.DebugMetrics; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hbase.shaded.org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsSink; @@ -58,10 +59,6 @@ public HashMap getMetrics() { return metricStructuredSnapshot; } - @Override - public void init(org.apache.commons.configuration2.SubsetConfiguration subsetConfiguration) { - } - @Override public void flush() { } @@ -109,6 +106,11 @@ private void updateMetricType(DebugMetrics debugMetrics, String metricType, Abst } } + @Override + public void init(SubsetConfiguration subsetConfiguration) { + + } + private static String inferMeasureType(String fullName, String nameWithoutMetricType) { return fullName.replaceFirst(nameWithoutMetricType, ""); } From 4e39d437f3b6ea44e4665bdf8f848ecb0059bf17 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sat, 23 Nov 2024 22:46:53 +0530 Subject: [PATCH 083/241] add attributes part of multi query --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 36592b6655..1224675693 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1177,9 +1177,7 @@ private Map> getBatchPropertiesWithMultiQu keys.addAll(attributes); - multiQuery.keys(Constants.TYPE_NAME_PROPERTY_KEY, Constants.GUID_PROPERTY_KEY, - Constants.CREATED_BY_KEY, Constants.MODIFIED_BY_KEY, - Constants.TIMESTAMP_PROPERTY_KEY, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, STATE_PROPERTY_KEY); + multiQuery.keys(keys.toArray(new String[0])); Map> vertexPropertiesMap = new HashMap<>(); From 9c5a6435c35de7719f044044728460fae36ed8ef Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sat, 23 Nov 2024 23:47:42 +0530 Subject: [PATCH 084/241] include janusoptimisation-multiQuery-1.0.0 in ecr build push --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 6227a0cf64..2d031f41ee 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,6 +27,7 @@ on: - master - lineageondemand - janusoptimisation + - janusoptimisation-multiQuery-1.0.0 jobs: build: From 01e5b3b0be72e99f31a5f112f2659e0b5c58e57b Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sun, 24 Nov 2024 00:34:57 +0530 Subject: [PATCH 085/241] include janusoptimisation-1.0.0 in ecr build push --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 2d031f41ee..c6574d259d 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,7 +27,7 @@ on: - master - lineageondemand - janusoptimisation - - janusoptimisation-multiQuery-1.0.0 + - janusoptimisation-1.0.0 jobs: build: From ee86e6e8a09bc171ef19cf858373dacde3cb98d6 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sun, 24 Nov 2024 00:47:30 +0530 Subject: [PATCH 086/241] include janusupgrade in ecr build push --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index c6574d259d..d5fb6ea459 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,7 +27,7 @@ on: - master - lineageondemand - janusoptimisation - - janusoptimisation-1.0.0 + - janusupgrade jobs: build: From b873d3538123ad0415f2d7d6e11740193f0b5f63 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 25 Nov 2024 13:28:15 +0530 Subject: [PATCH 087/241] Added secure in ATLASSESSIONID Cookie. --- webapp/src/main/webapp/WEB-INF/web.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/src/main/webapp/WEB-INF/web.xml b/webapp/src/main/webapp/WEB-INF/web.xml index efb26298e2..291c871a8c 100755 --- a/webapp/src/main/webapp/WEB-INF/web.xml +++ b/webapp/src/main/webapp/WEB-INF/web.xml @@ -139,6 +139,7 @@ ATLASSESSIONID true + true From 6a3ddd6068b7d07ac0dc51b4429ed0d22eaad526 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 25 Nov 2024 15:57:55 +0530 Subject: [PATCH 088/241] Cleanup ranger id of transformed policy to get atlas policy guid --- .../java/org/apache/atlas/plugin/model/RangerPolicy.java | 7 +++++++ .../apache/atlas/plugin/util/RangerPolicyDeltaUtil.java | 4 ++-- .../policytransformer/CachePolicyTransformerImpl.java | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java index f81f8e2854..e1a6bda610 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -529,6 +529,13 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) { this.isDenyAllElse = isDenyAllElse == null ? Boolean.FALSE : isDenyAllElse; } + public String getAtlasGuid() { + if (getGuid().length() > 36) { + return getGuid().substring(0, 36); + } + return getGuid(); + } + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index b92ae004d4..e63345c10a 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -222,11 +222,11 @@ public static Boolean hasPolicyDeltas(final ServicePolicies servicePolicies) { if (isPoliciesExist && isPolicyDeltasExist) { LOG.warn("ServicePolicies contain both policies and policy-deltas!! Cannot build policy-engine from these servicePolicies. Please check server-side code!"); - LOG.warn("Downloaded ServicePolicies are [" + servicePolicies + "]"); + LOG.warn("Downloaded ServicePolicies for [" + servicePolicies.getServiceName() + "]"); ret = null; } else if (!isPoliciesExist && !isPolicyDeltasExist) { LOG.warn("ServicePolicies do not contain any policies or policy-deltas!! There are no material changes in the policies."); - LOG.warn("Downloaded ServicePolicies are [" + servicePolicies + "]"); + LOG.warn("Downloaded ServicePolicies for [" + servicePolicies.getServiceName() + "]"); ret = null; } else { ret = isPolicyDeltasExist; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 3a97d0144e..ebe8812db1 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -182,9 +182,9 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); + tagPolicies.setPolicies(tagRangerPolicies); servicePolicies.setTagPolicies(tagPolicies); - servicePolicies.getTagPolicies().setPolicies(tagRangerPolicies); LOG.info("PolicyDelta: {}: Found tag policies - {}", serviceName, tagRangerPolicies.size()); } } @@ -308,6 +308,7 @@ private List getServicePoliciesDelta(AtlasEntityHeader servic } ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); + // this will have less policies as deleted won't be found List atlasPolicies = getAtlasPolicies(serviceName, batchSize, policyGuids); List rangerPolicies = new ArrayList<>(); @@ -316,7 +317,7 @@ private List getServicePoliciesDelta(AtlasEntityHeader servic } for (RangerPolicy policy : rangerPolicies) { - Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getGuid())); + Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getAtlasGuid())); RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); policyDeltas.add(delta); } From a6b60410ba2a800396cd714f92846ea0390c4741 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 25 Nov 2024 16:50:02 +0530 Subject: [PATCH 089/241] Use App Configuration instead of reading the config again --- .../java/org/apache/atlas/plugin/util/PolicyRefresher.java | 4 ++-- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 617dc10e7f..4a5dc552bd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -22,6 +22,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; @@ -114,10 +115,9 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); try { - this.atlasConfig = ApplicationProperties.get(); + this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); this.auditRepository = new ESBasedAuditRepository(atlasConfig); this.auditRepository.start(); - this.enableDeltaBasedRefresh = this.atlasConfig.getBoolean(DELTA_BASED_REFRESH, false); } catch (AtlasException e) { LOG.error("PolicyDelta: Error while reading atlas configuration", e); this.enableDeltaBasedRefresh = false; diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 2819aa9ead..b85a48a645 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -121,6 +121,8 @@ public enum AtlasConfiguration { ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), + DELTA_BASED_REFRESH_ENABLED("atlas.authorizer.enable.delta_based_refresh", false), + ATLAS_UD_RELATIONSHIPS_MAX_COUNT("atlas.ud.relationship.max.count", 100); From e042b4fe34b3c020ebbffd557fdd6277d21053a8 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Mon, 25 Nov 2024 16:36:58 +0530 Subject: [PATCH 090/241] bulk fetch with auto batching --- .../store/graph/v2/EntityGraphRetriever.java | 78 ++++++++++++------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 1224675693..d23401d570 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -18,7 +18,6 @@ package org.apache.atlas.repository.store.graph.v2; import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; @@ -70,9 +69,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; -import org.janusgraph.core.JanusGraphMultiVertexQuery; -import org.janusgraph.core.JanusGraphTransaction; -import org.janusgraph.core.JanusGraphVertex; +import org.janusgraph.core.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -94,11 +91,10 @@ import java.util.Objects; import java.util.Queue; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CREATED_BY; @@ -137,6 +133,7 @@ @Component public class EntityGraphRetriever { private static final Logger LOG = LoggerFactory.getLogger(EntityGraphRetriever.class); + private static final int BATCH_SIZE = 300; private static final String GLOSSARY_TERM_DISPLAY_NAME_ATTR = "name"; public static final String TERM_RELATION_NAME = "AtlasGlossarySemanticAssignment"; @@ -1103,12 +1100,12 @@ public List mapVerticesToAtlasEntityHeader(List AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVerticesToAtlasEntityHeader"); List results = new ArrayList<>(); - try { // Convert AtlasVertex to JanusGraphVertex // Use multiQuery for optimized property fetching - Map> multiQueryResults = getBatchPropertiesWithMultiQuery(entityVertices, attributes); + AtlasJanusGraph janusGraph = ((AtlasJanusGraph)graph); + Map> multiQueryResults = getBatchPropertiesWithMultiQuery(entityVertices, attributes, janusGraph); Map map = getJanusGraphVerticesMap(entityVertices); multiQueryResults.forEach((janusGraphVertex, vertexProperties) -> { AtlasEntityHeader ret = new AtlasEntityHeader(); @@ -1162,40 +1159,67 @@ private List getJanusGraphVertices(List vertices) return results; } - // Use multiQuery to batch-fetch properties - private Map> getBatchPropertiesWithMultiQuery(List entityVertices, Set attributes) { - Iterable vertices = getJanusGraphVertices(entityVertices); - List target = new ArrayList<>(); - vertices.forEach(target::add); - JanusGraphTransaction transaction = ((AtlasJanusGraph)graph).getTransaction(); + public Map> getBatchPropertiesWithMultiQuery( + List entityVertices, Set attributes, AtlasJanusGraph graph) { + + Map> result = new ConcurrentHashMap<>(); + List vertices = getJanusGraphVertices(entityVertices); + + // Split vertices into batches and process them in parallel + AtomicInteger counter = new AtomicInteger(0); + StreamSupport.stream(vertices.spliterator(), false) + .collect(Collectors.groupingBy(v -> counter.getAndIncrement() / BATCH_SIZE)) + .values() + .parallelStream() + .forEach(batch -> { + Map> batchResult = processBatch(batch, attributes, graph); + result.putAll(batchResult); + }); + + return result; + } + + private Map> processBatch( + List batch, Set attributes, AtlasJanusGraph graph) { + + JanusGraphTransaction transaction = graph.getTransaction(); try { - JanusGraphMultiVertexQuery multiQuery = transaction.multiQuery(target); + JanusGraphMultiVertexQuery multiQuery = transaction.multiQuery(batch); - Set keys = Sets.newHashSet(Constants.TYPE_NAME_PROPERTY_KEY, Constants.GUID_PROPERTY_KEY, + // Add all required keys to fetch + Set keys = new HashSet<>(Arrays.asList( + Constants.TYPE_NAME_PROPERTY_KEY, Constants.GUID_PROPERTY_KEY, Constants.CREATED_BY_KEY, Constants.MODIFIED_BY_KEY, - Constants.TIMESTAMP_PROPERTY_KEY, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); - + Constants.TIMESTAMP_PROPERTY_KEY, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY + )); keys.addAll(attributes); + // Set the keys in the multiQuery multiQuery.keys(keys.toArray(new String[0])); + // Fetch properties in bulk + Map> results = multiQuery.properties(); + + // Convert results to the desired map format Map> vertexPropertiesMap = new HashMap<>(); + for (Map.Entry> entry : results.entrySet()) { + JanusGraphVertex vertex = entry.getKey(); + Iterable properties = entry.getValue(); - for (JanusGraphVertex vertex : vertices) { - Map properties = new HashMap<>(); - for (String key : keys) { - properties.put(key, vertex.property(key).orElse(null)); + Map propertiesMap = new HashMap<>(); + for (JanusGraphProperty property : properties) { + propertiesMap.put(property.key(), property.value()); } - vertexPropertiesMap.put(vertex, properties); + vertexPropertiesMap.put(vertex, propertiesMap); } + return vertexPropertiesMap; } finally { - if(transaction != null) { + if (transaction != null) { transaction.commit(); transaction.close(); } } - } // Populate AtlasEntityHeader From b3ea0b952dea0452199032ae4d29d19e9c04c496 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 26 Nov 2024 13:13:30 +0530 Subject: [PATCH 091/241] Enable build on branch to test before merging --- .github/workflows/maven.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index afec7cf4e5..c8d3c2f911 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - dg1908 + - ns/fix/delta-refresh jobs: build: @@ -48,7 +49,9 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + run: | + echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + echo "##[set-output name=branch_lower;]$(echo ${GITHUB_REF#refs/heads/} | awk '{gsub("/", "-"); print tolower($0)}')" id: get_branch - name: Create Maven Settings @@ -63,7 +66,7 @@ jobs: - name: Build with Maven run: | - branch_name=${{ steps.get_branch.outputs.branch }} + branch_name='${{ steps.get_branch.outputs.branch }}' if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'dg1908' ]] then echo "build without dashboard" @@ -102,8 +105,8 @@ jobs: provenance: true push: true tags: | - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:latest + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:${{ steps.get_version.outputs.version }} - name: Scan Image uses: aquasecurity/trivy-action@master From fbede428bc8a5bff6ee31afb4e162138b2e78f35 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 26 Nov 2024 18:33:46 +0530 Subject: [PATCH 092/241] Init commit. Updated POGO and logic to skip computeChanges if override is false. --- .../model/instance/AtlasEntityHeader.java | 28 +++++++++++++++++++ .../graph/v2/ClassificationAssociator.java | 12 ++++++-- .../org/apache/atlas/web/rest/EntityREST.java | 4 +-- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index 13820f5255..c628451c2c 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -61,6 +61,11 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private String displayText = null; private List classificationNames = null; private List classifications = null; + private List appendClassifications = null; + + + private List updateClassifications = null; + private List removeClassifications = null; private List meaningNames = null; private List meanings = null; private Boolean isIncomplete = Boolean.FALSE; @@ -101,6 +106,29 @@ public AtlasEntityHeader(String typeName, Map attributes) { setLabels(null); } + public List getAppendClassifications() { + return appendClassifications; + } + + public void setAppendClassifications(List appendClassifications) { + this.appendClassifications = appendClassifications; + } + + public List getUpdateClassifications() { + return updateClassifications; + } + + public void setUpdateClassifications(List updateClassifications) { + this.updateClassifications = updateClassifications; + } + + public List getRemoveClassifications() { + return removeClassifications; + } + + public void setRemoveClassifications(List removeClassifications) { + this.removeClassifications = removeClassifications; + } public AtlasEntityHeader(String typeName, String guid, Map attributes) { super(typeName, attributes); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java index 6cd9ac4e4b..5469fe70c6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java @@ -154,7 +154,7 @@ public Updater(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore, this(AtlasGraphProvider.getGraphInstance(), typeRegistry, entitiesStore, entityGraphMapper, entityChangeNotifier, instanceConverter); } - public void setClassifications(Map map) throws AtlasBaseException { + public void setClassifications(Map map, boolean overrideClassifications) throws AtlasBaseException { RequestContext.get().setDelayTagNotifications(true); for (String guid : map.keySet()) { @@ -178,8 +178,14 @@ public void setClassifications(Map map) throws AtlasB if (entityToBeChanged == null) { throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); } - - Map> operationListMap = computeChanges(incomingEntityHeader, entityToBeChanged); + Map> operationListMap = new HashMap<>(); + if(overrideClassifications) { + operationListMap = computeChanges(incomingEntityHeader, entityToBeChanged); + } else { + bucket(PROCESS_DELETE, operationListMap, incomingEntityHeader.getRemoveClassifications()); + bucket(PROCESS_UPDATE, operationListMap, incomingEntityHeader.getUpdateClassifications()); + bucket(PROCESS_ADD, operationListMap, incomingEntityHeader.getAppendClassifications()); + } try { commitChanges(guid, typeName, operationListMap); } catch (AtlasBaseException e) { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 5333d63e7b..fc214fea3c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1265,7 +1265,7 @@ public AtlasEntityHeaders getEntityHeaders(@QueryParam("tagUpdateStartTime") lon @Produces(Servlets.JSON_MEDIA_TYPE) @Consumes(Servlets.JSON_MEDIA_TYPE) @Timed - public void setClassifications(AtlasEntityHeaders entityHeaders) throws AtlasBaseException { + public void setClassifications(AtlasEntityHeaders entityHeaders, @QueryParam("overrideClassifications") @DefaultValue("true") boolean overrideClassifications) throws AtlasBaseException { AtlasPerfTracer perf = null; try { @@ -1274,7 +1274,7 @@ public void setClassifications(AtlasEntityHeaders entityHeaders) throws AtlasBas } ClassificationAssociator.Updater associator = new ClassificationAssociator.Updater(typeRegistry, entitiesStore, entityGraphMapper, entityChangeNotifier, instanceConverter); - associator.setClassifications(entityHeaders.getGuidHeaderMap()); + associator.setClassifications(entityHeaders.getGuidHeaderMap(), overrideClassifications); } finally { AtlasPerfTracer.log(perf); } From 924b588a4c362c3ad29b1b40894114bedb19dfd8 Mon Sep 17 00:00:00 2001 From: Rahul Madan Date: Tue, 26 Nov 2024 20:45:17 +0530 Subject: [PATCH 093/241] initial commit --- .../apache/atlas/repository/Constants.java | 6 + .../atlas/discovery/EntityLineageService.java | 87 +++- .../store/graph/v2/AtlasEntityStoreV2.java | 8 + .../lineage/LineagePreProcessor.java | 491 ++++++++++++++++++ 4 files changed, 568 insertions(+), 24 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 11e08bf78b..917bc71936 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -207,6 +207,11 @@ public final class Constants { public static String[] PROCESS_EDGE_LABELS = {PROCESS_OUTPUTS, PROCESS_INPUTS}; + public static final String PROCESS_ENTITY_TYPE = "Process"; + + public static final String CONNECTION_PROCESS_ENTITY_TYPE = "ConnectionProcess"; + public static final String PARENT_CONNECTION_PROCESS_QUALIFIED_NAME = "parentConnectionProcessQualifiedName"; + /** * The homeId field is used when saving into Atlas a copy of an object that is being imported from another * repository. The homeId will be set to a String that identifies the other repository. The specific format @@ -269,6 +274,7 @@ public final class Constants { public static final String NAME = "name"; public static final String QUALIFIED_NAME = "qualifiedName"; + public static final String CONNECTION_QUALIFIED_NAME = "connectionQualifiedName"; public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName"; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length"; diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 9dd862b333..be882a7f28 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -84,6 +84,8 @@ public class EntityLineageService implements AtlasLineageService { private static final String PROCESS_INPUTS_EDGE = "__Process.inputs"; private static final String PROCESS_OUTPUTS_EDGE = "__Process.outputs"; + private static final String CONNECTION_PROCESS_INPUTS_EDGE = "__ConnectionProcess.inputs"; + private static final String CONNECTION_PROCESS_OUTPUTS_EDGE = "__ConnectionProcess.outputs"; private static final String COLUMNS = "columns"; private static final boolean LINEAGE_USING_GREMLIN = AtlasConfiguration.LINEAGE_USING_GREMLIN.getBoolean(); private static final Integer DEFAULT_LINEAGE_MAX_NODE_COUNT = 9000; @@ -178,8 +180,8 @@ public AtlasLineageOnDemandInfo getAtlasLineageInfo(String guid, LineageOnDemand RequestContext.get().setRelationAttrsForSearch(lineageOnDemandRequest.getRelationAttributes()); AtlasLineageOnDemandContext atlasLineageOnDemandContext = new AtlasLineageOnDemandContext(lineageOnDemandRequest, atlasTypeRegistry); - boolean isDataSet = validateEntityTypeAndCheckIfDataSet(guid); - AtlasLineageOnDemandInfo ret = getLineageInfoOnDemand(guid, atlasLineageOnDemandContext, isDataSet); + EntityValidationResult entityValidationResult = validateEntityTypeAndCheckIfDataSet(guid); + AtlasLineageOnDemandInfo ret = getLineageInfoOnDemand(guid, atlasLineageOnDemandContext, entityValidationResult); appendLineageOnDemandPayload(ret, lineageOnDemandRequest); // filtering out on-demand relations which has input & output nodes within the limit cleanupRelationsOnDemand(ret); @@ -204,20 +206,44 @@ public AtlasLineageListInfo getLineageListInfoOnDemand(String guid, LineageListR return ret; } - private boolean validateEntityTypeAndCheckIfDataSet(String guid) throws AtlasBaseException { + public class EntityValidationResult { + public final boolean isProcess; + public final boolean isDataSet; + public final boolean isConnection; + public final boolean isConnectionProcess; + + public EntityValidationResult(boolean isProcess, boolean isDataSet, boolean isConnection, boolean isConnectionProcess) { + this.isProcess = isProcess; + this.isDataSet = isDataSet; + this.isConnection = isConnection; + this.isConnectionProcess = isConnectionProcess; + } + } + + + private EntityValidationResult validateEntityTypeAndCheckIfDataSet(String guid) throws AtlasBaseException { String typeName = entityRetriever.getEntityVertex(guid).getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName(typeName); if (entityType == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, typeName); } boolean isProcess = entityType.getTypeAndAllSuperTypes().contains(PROCESS_SUPER_TYPE); + boolean isConnectionProcess = false; + boolean isDataSet = false; + boolean isConnection = false; if (!isProcess) { - boolean isDataSet = entityType.getTypeAndAllSuperTypes().contains(DATA_SET_SUPER_TYPE); - if (!isDataSet) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_LINEAGE_ENTITY_TYPE, guid, typeName); + isConnectionProcess = entityType.getTypeAndAllSuperTypes().contains(CONNECTION_PROCESS_ENTITY_TYPE); + if(!isConnectionProcess){ + isDataSet = entityType.getTypeAndAllSuperTypes().contains(DATA_SET_SUPER_TYPE); + if (!isDataSet) { + isConnection = entityType.getTypeAndAllSuperTypes().contains(CONNECTION_ENTITY_TYPE); + if(!isConnection){ + throw new AtlasBaseException(AtlasErrorCode.INVALID_LINEAGE_ENTITY_TYPE, guid, typeName); + } + } } } - return !isProcess; + return new EntityValidationResult(isProcess, isDataSet, isConnection, isConnectionProcess); } private LineageOnDemandConstraints getLineageConstraints(String guid, LineageOnDemandBaseParams defaultParams) { @@ -282,7 +308,7 @@ private void cleanupRelationsOnDemand(AtlasLineageOnDemandInfo lineageInfo) { } } - private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineageOnDemandContext atlasLineageOnDemandContext, boolean isDataSet) throws AtlasBaseException { + private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineageOnDemandContext atlasLineageOnDemandContext, EntityValidationResult entityValidationResult) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getLineageInfoOnDemand"); LineageOnDemandConstraints lineageConstraintsByGuid = getAndValidateLineageConstraintsByGuid(guid, atlasLineageOnDemandContext); @@ -301,12 +327,12 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag AtomicInteger traversalOrder = new AtomicInteger(1); TimeoutChecker timeoutChecker = new TimeoutChecker(LINEAGE_TIMEOUT_MS); atlasLineageOnDemandContext.setTimeoutChecker(timeoutChecker); - if (isDataSet) { + if (entityValidationResult.isConnection || entityValidationResult.isDataSet) { AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, true, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed, traversalOrder); + traverseEdgesOnDemand(datasetVertex, true, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed, traversalOrder, entityValidationResult); if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed, traversalOrder); + traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed, traversalOrder, entityValidationResult); AtlasEntityHeader baseEntityHeader = entityRetriever.toAtlasEntityHeader(datasetVertex, atlasLineageOnDemandContext.getAttributes()); setGraphTraversalMetadata(level, traversalOrder, baseEntityHeader); ret.getGuidEntityMap().put(guid, baseEntityHeader); @@ -314,12 +340,12 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); // make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1' if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { - Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_INPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, true, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed, traversalOrder); + Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, entityValidationResult.isProcess? PROCESS_INPUTS_EDGE:CONNECTION_PROCESS_INPUTS_EDGE).iterator(); + traverseEdgesOnDemand(processEdges, true, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed, traversalOrder, entityValidationResult); } if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { - Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_OUTPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, false, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed, traversalOrder); + Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, entityValidationResult.isProcess? PROCESS_OUTPUTS_EDGE:CONNECTION_PROCESS_OUTPUTS_EDGE).iterator(); + traverseEdgesOnDemand(processEdges, false, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed, traversalOrder, entityValidationResult); } } RequestContext.get().endMetricRecord(metricRecorder); @@ -332,7 +358,7 @@ private static void setGraphTraversalMetadata(int level, AtomicInteger traversal baseEntityHeader.setFinishTime(traversalOrder.get()); } - private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder) throws AtlasBaseException { + private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder, EntityValidationResult entityValidationResult) throws AtlasBaseException { AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; int nextLevel = isInput ? level - 1: level + 1; @@ -363,11 +389,12 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI ret.getRelationsOnDemand().put(inGuid, new LineageInfoOnDemand(inGuidLineageConstrains)); } - traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, nextLevel, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); + EntityValidationResult entityValidationResult1 = validateEntityTypeAndCheckIfDataSet(inGuid); + traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, nextLevel, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder, entityValidationResult1); } } - private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder) throws AtlasBaseException { + private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder, EntityValidationResult entityValidationResult) throws AtlasBaseException { // Get timeout checker from context or create new one TimeoutChecker timeoutChecker = atlasLineageOnDemandContext.getTimeoutChecker(); // Check timeout before starting traversal @@ -385,8 +412,13 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i // keep track of visited vertices to avoid circular loop visitedVertices.add(getId(datasetVertex)); + Iterator incomingEdges = null; AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesIn = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesIn"); - Iterator incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); + if(entityValidationResult.isDataSet){ + incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); + } else if (entityValidationResult.isConnection) { + incomingEdges = datasetVertex.getEdges(IN, isInput ? CONNECTION_PROCESS_OUTPUTS_EDGE : CONNECTION_PROCESS_INPUTS_EDGE).iterator(); + } RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesIn); while (incomingEdges.hasNext()) { @@ -420,7 +452,13 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i } AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); - Iterator outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); + Iterator outgoingEdges = null; + if(entityValidationResult.isDataSet){ + outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); + } else if (entityValidationResult.isConnection) { + outgoingEdges = processVertex.getEdges(OUT, isInput ? CONNECTION_PROCESS_INPUTS_EDGE : CONNECTION_PROCESS_OUTPUTS_EDGE).iterator(); + } + RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesOut); while (outgoingEdges.hasNext()) { @@ -457,7 +495,8 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i setEntityLimitReachedFlag(isInput, ret); } if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { - traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); // execute inner depth + EntityValidationResult entityValidationResult1 = validateEntityTypeAndCheckIfDataSet(getGuid(entityVertex)); + traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder, entityValidationResult1); // execute inner depth AtlasEntityHeader traversedEntity = ret.getGuidEntityMap().get(AtlasGraphUtilsV2.getIdFromVertex(entityVertex)); if (traversedEntity != null) traversedEntity.setFinishTime(traversalOrder.get()); @@ -494,7 +533,7 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line AtlasVertex baseVertex = AtlasGraphUtilsV2.findByGuid(this.graph, baseGuid); - boolean isBaseNodeDataset = validateEntityTypeAndCheckIfDataSet(baseGuid); + boolean isBaseNodeDataset = validateEntityTypeAndCheckIfDataSet(baseGuid).isDataSet; // Get the neighbors for the current node enqueueNeighbours(baseVertex, isBaseNodeDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices, lineageParentsForEntityMap, lineageChildrenForEntityMap); int currentDepth = 0; @@ -522,7 +561,7 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line if (Objects.isNull(currentVertex)) throw new AtlasBaseException("Found null vertex during lineage graph traversal for guid: " + currentGUID); - boolean isDataset = validateEntityTypeAndCheckIfDataSet(currentGUID); + boolean isDataset = validateEntityTypeAndCheckIfDataSet(currentGUID).isDataSet; if (!lineageListContext.evaluateVertexFilter(currentVertex)) { enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices, lineageParentsForEntityMap, lineageChildrenForEntityMap); continue; @@ -1539,7 +1578,7 @@ private void processEdge(final AtlasEdge edge, AtlasLineageInfo lineageInfo, String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex); String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(edge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); - boolean isInputEdge = edge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); + boolean isInputEdge = edge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE) || edge.getLabel().equalsIgnoreCase(CONNECTION_PROCESS_INPUTS_EDGE); if (!entities.containsKey(inGuid)) { AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(inVertex, lineageContext.getAttributes()); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index f48d206653..75e55cd8a3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -56,6 +56,11 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.AssetPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.StakeholderPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.contract.ContractPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.lineage.LineagePreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PersonaPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PurposePreProcessor; @@ -1925,6 +1930,9 @@ public List getPreProcessor(String typeName) { case STAKEHOLDER_TITLE_ENTITY_TYPE: preProcessors.add(new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever)); break; + + case PROCESS_ENTITY_TYPE: + preProcessors.add(new LineagePreProcessor(typeRegistry, entityRetriever, graph, this)); } // The default global pre-processor for all AssetTypes diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java new file mode 100644 index 0000000000..c99fa59a34 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java @@ -0,0 +1,491 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor.lineage; + + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.AtlasException; +import org.apache.atlas.DeleteType; +import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.*; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.EntityStream; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.util.AtlasEntityUtils; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE; +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.graph.GraphHelper.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; + +public class LineagePreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(LineagePreProcessor.class); + private static final List FETCH_ENTITY_ATTRIBUTES = Arrays.asList(CONNECTION_QUALIFIED_NAME); + private final AtlasTypeRegistry typeRegistry; + private final EntityGraphRetriever entityRetriever; + private AtlasEntityStore entityStore; + protected EntityDiscoveryService discovery; + private static final String HAS_LINEAGE = "__hasLineage"; + + public LineagePreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph, AtlasEntityStore entityStore) { + this.entityRetriever = entityRetriever; + this.typeRegistry = typeRegistry; + this.entityStore = entityStore; + try { + this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); + } catch (AtlasException e) { + e.printStackTrace(); + } + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processAttributesForLineagePreprocessor"); + + try { + if (LOG.isDebugEnabled()) { + LOG.debug("LineageProcessPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + AtlasVertex vertex = context.getVertex(entity.getGuid()); + ArrayList connectionProcessQNs = getConnectionProcessQNsForTheGivenInputOutputs(entity); + + switch (operation) { + case CREATE: + processCreateLineageProcess(entity, connectionProcessQNs); + break; + case UPDATE: + processUpdateLineageProcess(entity, vertex, context, connectionProcessQNs); + break; + } + }catch(Exception exp){ + if (LOG.isDebugEnabled()) { + LOG.debug("Lineage preprocessor: " + exp); + } + }finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + } + + private void processCreateLineageProcess(AtlasEntity entity, ArrayList connectionProcessList) { + // if not exist create lineage process + // add owner connection process + if(!connectionProcessList.isEmpty()){ + entity.setAttribute(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, connectionProcessList); + } + } + + private void processUpdateLineageProcess(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context, ArrayList connectionProcessList) { + // check if connection lineage exists + // add owner connection process + if(!connectionProcessList.isEmpty()){ + entity.setAttribute(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, connectionProcessList); + } + } + + private AtlasEntity createConnectionProcessEntity(Map connectionProcessInfo) throws AtlasBaseException { + AtlasEntity processEntity = new AtlasEntity(); + processEntity.setTypeName(CONNECTION_PROCESS_ENTITY_TYPE); + processEntity.setAttribute(NAME, connectionProcessInfo.get("connectionProcessName")); + processEntity.setAttribute(QUALIFIED_NAME, connectionProcessInfo.get("connectionProcessQualifiedName")); + + // Set up relationship attributes for input and output connections + AtlasObjectId inputConnection = new AtlasObjectId(); + inputConnection.setTypeName(CONNECTION_ENTITY_TYPE); + inputConnection.setUniqueAttributes(mapOf(QUALIFIED_NAME, connectionProcessInfo.get("input"))); + + AtlasObjectId outputConnection = new AtlasObjectId(); + outputConnection.setTypeName(CONNECTION_ENTITY_TYPE); + outputConnection.setUniqueAttributes(mapOf(QUALIFIED_NAME, connectionProcessInfo.get("output"))); + + Map relationshipAttributes = new HashMap<>(); + relationshipAttributes.put("inputs", Collections.singletonList(inputConnection)); + relationshipAttributes.put("outputs", Collections.singletonList(outputConnection)); + processEntity.setRelationshipAttributes(relationshipAttributes); + + try { + RequestContext.get().setSkipAuthorizationCheck(true); + AtlasEntity.AtlasEntitiesWithExtInfo processExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); + processExtInfo.addEntity(processEntity); + EntityStream entityStream = new AtlasEntityStream(processExtInfo); + entityStore.createOrUpdate(entityStream, false); + + // Update hasLineage for both connections + updateConnectionLineageFlag((String) connectionProcessInfo.get("input"), true); + updateConnectionLineageFlag((String) connectionProcessInfo.get("output"), true); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + + return processEntity; + } + + private void updateConnectionLineageFlag(String connectionQualifiedName, boolean hasLineage) throws AtlasBaseException { + AtlasObjectId connectionId = new AtlasObjectId(); + connectionId.setTypeName(CONNECTION_ENTITY_TYPE); + connectionId.setUniqueAttributes(mapOf(QUALIFIED_NAME, connectionQualifiedName)); + + try { + AtlasVertex connectionVertex = entityRetriever.getEntityVertex(connectionId); + AtlasEntity connection = entityRetriever.toAtlasEntity(connectionVertex); + connection.setAttribute(HAS_LINEAGE, hasLineage); + + AtlasEntity.AtlasEntitiesWithExtInfo connectionExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); + connectionExtInfo.addEntity(connection); + EntityStream entityStream = new AtlasEntityStream(connectionExtInfo); + + RequestContext.get().setSkipAuthorizationCheck(true); + try { + entityStore.createOrUpdate(entityStream, false); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + } catch (AtlasBaseException e) { + if (!e.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)) { + throw e; + } + } + } + + private void checkAndUpdateConnectionLineage(String connectionQualifiedName) throws AtlasBaseException { + AtlasObjectId connectionId = new AtlasObjectId(); + connectionId.setTypeName(CONNECTION_ENTITY_TYPE); + connectionId.setUniqueAttributes(mapOf(QUALIFIED_NAME, connectionQualifiedName)); + + try { + AtlasVertex connectionVertex = entityRetriever.getEntityVertex(connectionId); + + // Check both input and output edges + boolean hasActiveConnectionProcess = hasActiveConnectionProcesses(connectionVertex); + + // Only update if the hasLineage status needs to change + boolean currentHasLineage = getEntityHasLineage(connectionVertex); + if (currentHasLineage != hasActiveConnectionProcess) { + if (LOG.isDebugEnabled()) { + LOG.debug("Updating hasLineage for connection {} from {} to {}", + connectionQualifiedName, currentHasLineage, hasActiveConnectionProcess); + } + + AtlasEntity connection = entityRetriever.toAtlasEntity(connectionVertex); + connection.setAttribute(HAS_LINEAGE, hasActiveConnectionProcess); + + AtlasEntity.AtlasEntitiesWithExtInfo connectionExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); + connectionExtInfo.addEntity(connection); + EntityStream entityStream = new AtlasEntityStream(connectionExtInfo); + + RequestContext.get().setSkipAuthorizationCheck(true); + try { + entityStore.createOrUpdate(entityStream, false); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + } + } catch (AtlasBaseException e) { + if (!e.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)) { + throw e; + } + } + } + + private boolean hasActiveConnectionProcesses(AtlasVertex connectionVertex) { + // Check both input and output edges + Iterator edges = connectionVertex.getEdges(AtlasEdgeDirection.BOTH, + new String[]{"__ConnectionProcess.inputs", "__ConnectionProcess.outputs"}).iterator(); + + while (edges.hasNext()) { + AtlasEdge edge = edges.next(); + + if (getStatus(edge) == ACTIVE) { + AtlasVertex processVertex = edge.getLabel().equals("__ConnectionProcess.inputs") ? + edge.getOutVertex() : edge.getInVertex(); + + // If this is an active connection process + if (getStatus(processVertex) == ACTIVE && + getTypeName(processVertex).equals(CONNECTION_PROCESS_ENTITY_TYPE)) { + + // Get the other connection in this process + AtlasVertex otherConnectionVertex = null; + Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.BOTH, + new String[]{"__ConnectionProcess.inputs", "__ConnectionProcess.outputs"}).iterator(); + + while (processEdges.hasNext()) { + AtlasEdge processEdge = processEdges.next(); + if (getStatus(processEdge) == ACTIVE) { + AtlasVertex connVertex = processEdge.getInVertex(); + if (!connVertex.getId().equals(connectionVertex.getId())) { + otherConnectionVertex = connVertex; + break; + } + } + } + + // If the other connection is active, this connection process is valid + if (otherConnectionVertex != null && getStatus(otherConnectionVertex) == ACTIVE) { + return true; + } + } + } + } + + return false; + } + + private ArrayList getConnectionProcessQNsForTheGivenInputOutputs(AtlasEntity processEntity) throws AtlasBaseException{ + + // check connection lineage exists or not + // check if connection lineage exists + Map entityAttrValues = processEntity.getRelationshipAttributes(); + + ArrayList inputsAssets = (ArrayList) entityAttrValues.get("inputs"); + ArrayList outputsAssets = (ArrayList) entityAttrValues.get("outputs"); + + // get connection process + Set> uniquesSetOfConnectionProcess = new HashSet<>(); + + for (AtlasObjectId input : inputsAssets){ + AtlasVertex inputVertex = entityRetriever.getEntityVertex(input); + Map inputVertexConnectionQualifiedName = fetchAttributes(inputVertex, FETCH_ENTITY_ATTRIBUTES); + for (AtlasObjectId output : outputsAssets){ + AtlasVertex outputVertex = entityRetriever.getEntityVertex(output); + Map outputVertexConnectionQualifiedName = fetchAttributes(outputVertex, FETCH_ENTITY_ATTRIBUTES); + + if(inputVertexConnectionQualifiedName.get(CONNECTION_QUALIFIED_NAME) == outputVertexConnectionQualifiedName.get(CONNECTION_QUALIFIED_NAME)){ + continue; + } + + String connectionProcessName = "(" + inputVertexConnectionQualifiedName.get(CONNECTION_QUALIFIED_NAME) + ")->(" + outputVertexConnectionQualifiedName.get(CONNECTION_QUALIFIED_NAME) + ")"; + String connectionProcessQualifiedName = outputVertexConnectionQualifiedName.get(CONNECTION_QUALIFIED_NAME) + "/" + connectionProcessName; + // Create a map to store both connectionProcessName and connectionProcessQualifiedName + Map connectionProcessMap = new HashMap<>(); + connectionProcessMap.put("input", inputVertexConnectionQualifiedName.get(CONNECTION_QUALIFIED_NAME)); + connectionProcessMap.put("output", outputVertexConnectionQualifiedName.get(CONNECTION_QUALIFIED_NAME)); + connectionProcessMap.put("connectionProcessName", connectionProcessName); + connectionProcessMap.put("connectionProcessQualifiedName", connectionProcessQualifiedName); + + // Add the map to the set + uniquesSetOfConnectionProcess.add(connectionProcessMap); + } + } + + ArrayList connectionProcessList = new ArrayList<>(); + + // check if connection process exists + for (Map connectionProcessInfo : uniquesSetOfConnectionProcess){ + AtlasObjectId atlasObjectId = new AtlasObjectId(); + atlasObjectId.setTypeName(CONNECTION_PROCESS_ENTITY_TYPE); + atlasObjectId.setUniqueAttributes(mapOf(QUALIFIED_NAME, connectionProcessInfo.get("connectionProcessQualifiedName"))); + AtlasVertex connectionProcessVertex = null; + try { + // TODO add caching here + connectionProcessVertex = entityRetriever.getEntityVertex(atlasObjectId); + } + catch(AtlasBaseException exp){ + if(!exp.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)){ + throw exp; + } + } + + AtlasEntity connectionProcess; + if (connectionProcessVertex == null) { + connectionProcess = createConnectionProcessEntity(connectionProcessInfo); + } else { + // exist so retrieve and perform any update so below statement to retrieve + // TODO add caching here + connectionProcess = entityRetriever.toAtlasEntity(connectionProcessVertex); + } + // only add in list if created + connectionProcessList.add(connectionProcess.getAttribute(QUALIFIED_NAME)); + } + + return connectionProcessList; + } + + public boolean checkIfMoreChildProcessExistForConnectionProcess(String connectionProcessQn) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("checkIfMoreChileProcessExistForConnectionProcess"); + boolean ret = false; + + try { + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", PROCESS_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, connectionProcessQn))); + + Map dsl = mapOf("query", mapOf("bool", mapOf("must", mustClauseList))); + + List process = indexSearchPaginated(dsl, new HashSet<>(Arrays.asList(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME)) , this.discovery); + + if (CollectionUtils.isNotEmpty(process) && process.size()>1) { + ret = true; + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + return ret; + } + + // handle process delete logic + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processDeleteLineageProcess"); + + try { + // handle both soft and hard deletes + // Collect all connections involved in the process being deleted + AtlasEntity processEntity = entityRetriever.toAtlasEntity(vertex); + + Set involvedConnections = new HashSet<>(); + + // Retrieve inputs and outputs from the process + List inputs = (List) processEntity.getRelationshipAttribute("inputs"); + List outputs = (List) processEntity.getRelationshipAttribute("outputs"); + + if (inputs == null) inputs = Collections.emptyList(); + if (outputs == null) outputs = Collections.emptyList(); + + List allAssets = new ArrayList<>(); + allAssets.addAll(inputs); + allAssets.addAll(outputs); + + // For each asset, get its connection and add to involvedConnections + for (AtlasObjectId assetId : allAssets) { + try { + AtlasVertex assetVertex = entityRetriever.getEntityVertex(assetId); + Map assetConnectionAttributes = fetchAttributes(assetVertex, FETCH_ENTITY_ATTRIBUTES); + if (assetConnectionAttributes != null) { + String connectionQN = assetConnectionAttributes.get(CONNECTION_QUALIFIED_NAME); + if (StringUtils.isNotEmpty(connectionQN)) { + involvedConnections.add(connectionQN); + } + } + } catch (AtlasBaseException e) { + LOG.warn("Failed to retrieve connection for asset {}: {}", assetId.getGuid(), e.getMessage()); + } + } + + // Collect affected connections from connection processes to be deleted + Set connectionProcessQNs = new HashSet<>(); + + Object rawProperty = vertex.getProperty(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, Object.class); + + if (rawProperty instanceof List) { + // If the property is a List, cast and add all elements + List propertyList = (List) rawProperty; + connectionProcessQNs.addAll(propertyList); + } else if (rawProperty instanceof String) { + // If it's a single String, add it to the set + connectionProcessQNs.add((String) rawProperty); + } else if (rawProperty != null) { + // Handle other object types if necessary + connectionProcessQNs.add(rawProperty.toString()); + } + + if (connectionProcessQNs.isEmpty()) { + return; + } + + Set affectedConnections = new HashSet<>(); + + // Process each connection process + for (String connectionProcessQn : connectionProcessQNs) { + if (!checkIfMoreChildProcessExistForConnectionProcess(connectionProcessQn)) { + AtlasObjectId atlasObjectId = new AtlasObjectId(); + atlasObjectId.setTypeName(CONNECTION_PROCESS_ENTITY_TYPE); + atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME, connectionProcessQn)); + + try { + // Get connection process before deletion to track affected connections + AtlasVertex connectionProcessVertex = entityRetriever.getEntityVertex(atlasObjectId); + AtlasEntity connectionProcess = entityRetriever.toAtlasEntity(connectionProcessVertex); + + // Safely get connection qualified names + String inputConnQN = getConnectionQualifiedName(connectionProcess, "input"); + String outputConnQN = getConnectionQualifiedName(connectionProcess, "output"); + + // Add non-null qualified names to affected connections + if (StringUtils.isNotEmpty(inputConnQN)) { + affectedConnections.add(inputConnQN); + } + if (StringUtils.isNotEmpty(outputConnQN)) { + affectedConnections.add(outputConnQN); + } + + // Delete the connection process + entityStore.deleteById(connectionProcessVertex.getProperty("__guid", String.class)); + } catch (AtlasBaseException exp) { + if (!exp.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)) { + throw exp; + } + } + } + } + + // Combine involved and affected connections + Set connectionsToCheck = new HashSet<>(); + connectionsToCheck.addAll(involvedConnections); + connectionsToCheck.addAll(affectedConnections); + + // Check and update hasLineage for all connections involved + for (String connectionQN : connectionsToCheck) { + checkAndUpdateConnectionLineage(connectionQN); + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + // Helper method to safely get connection qualified name + private String getConnectionQualifiedName(AtlasEntity connectionProcess, String attributeName) { + try { + Object relationshipAttr = connectionProcess.getRelationshipAttribute(attributeName); + if (relationshipAttr instanceof AtlasObjectId) { + AtlasObjectId connObjectId = (AtlasObjectId) relationshipAttr; + Map uniqueAttributes = connObjectId.getUniqueAttributes(); + if (uniqueAttributes != null) { + return (String) uniqueAttributes.get(QUALIFIED_NAME); + } + } + } catch (Exception e) { + LOG.warn("Error getting {} qualified name for connection process {}: {}", + attributeName, connectionProcess.getGuid(), e.getMessage()); + } + return null; + } +} \ No newline at end of file From c1601620f94e1675ced335c7caabfb22a3db82e1 Mon Sep 17 00:00:00 2001 From: Rahul Madan Date: Tue, 26 Nov 2024 21:17:28 +0530 Subject: [PATCH 094/241] updated lineage preprocessor to check both input/output, not just 1 --- .../lineage/LineagePreProcessor.java | 37 +++++-------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java index c99fa59a34..cd24ac943d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java @@ -193,7 +193,7 @@ private void checkAndUpdateConnectionLineage(String connectionQualifiedName) thr try { AtlasVertex connectionVertex = entityRetriever.getEntityVertex(connectionId); - // Check both input and output edges + // Check if this connection has any active connection processes boolean hasActiveConnectionProcess = hasActiveConnectionProcesses(connectionVertex); // Only update if the hasLineage status needs to change @@ -226,45 +226,26 @@ private void checkAndUpdateConnectionLineage(String connectionQualifiedName) thr } private boolean hasActiveConnectionProcesses(AtlasVertex connectionVertex) { - // Check both input and output edges + // Iterate over both input and output edges connected to this connection Iterator edges = connectionVertex.getEdges(AtlasEdgeDirection.BOTH, new String[]{"__ConnectionProcess.inputs", "__ConnectionProcess.outputs"}).iterator(); while (edges.hasNext()) { AtlasEdge edge = edges.next(); + // Check if the edge is ACTIVE if (getStatus(edge) == ACTIVE) { - AtlasVertex processVertex = edge.getLabel().equals("__ConnectionProcess.inputs") ? - edge.getOutVertex() : edge.getInVertex(); + // Get the connected process vertex (the other vertex of the edge) + AtlasVertex processVertex = edge.getOutVertex().equals(connectionVertex) ? + edge.getInVertex() : edge.getOutVertex(); - // If this is an active connection process + // Check if the connected vertex is an ACTIVE ConnectionProcess if (getStatus(processVertex) == ACTIVE && getTypeName(processVertex).equals(CONNECTION_PROCESS_ENTITY_TYPE)) { - - // Get the other connection in this process - AtlasVertex otherConnectionVertex = null; - Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.BOTH, - new String[]{"__ConnectionProcess.inputs", "__ConnectionProcess.outputs"}).iterator(); - - while (processEdges.hasNext()) { - AtlasEdge processEdge = processEdges.next(); - if (getStatus(processEdge) == ACTIVE) { - AtlasVertex connVertex = processEdge.getInVertex(); - if (!connVertex.getId().equals(connectionVertex.getId())) { - otherConnectionVertex = connVertex; - break; - } - } - } - - // If the other connection is active, this connection process is valid - if (otherConnectionVertex != null && getStatus(otherConnectionVertex) == ACTIVE) { - return true; - } + return true; } } } - return false; } @@ -488,4 +469,4 @@ private String getConnectionQualifiedName(AtlasEntity connectionProcess, String } return null; } -} \ No newline at end of file +} From 25c5bcf61f5ab8838082053e9332c9aef4e106ba Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 27 Nov 2024 00:34:06 +0530 Subject: [PATCH 095/241] Added some validations to prevent non-existing classfications to be passed in updates or removes array. along with append will only happen for non-existing new classifications. --- .../graph/v2/ClassificationAssociator.java | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java index 5469fe70c6..732fabb120 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java @@ -53,6 +53,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.atlas.AtlasConfiguration.ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES; @@ -182,9 +183,7 @@ public void setClassifications(Map map, boolean overr if(overrideClassifications) { operationListMap = computeChanges(incomingEntityHeader, entityToBeChanged); } else { - bucket(PROCESS_DELETE, operationListMap, incomingEntityHeader.getRemoveClassifications()); - bucket(PROCESS_UPDATE, operationListMap, incomingEntityHeader.getUpdateClassifications()); - bucket(PROCESS_ADD, operationListMap, incomingEntityHeader.getAppendClassifications()); + operationListMap = validateAndTransfer(incomingEntityHeader, entityToBeChanged); } try { commitChanges(guid, typeName, operationListMap); @@ -242,6 +241,44 @@ public void setClassifications(Map map, boolean overr RequestContext.get().setDelayTagNotifications(false); } + private Map> validateAndTransfer(AtlasEntityHeader incomingEntityHeader, AtlasEntityHeader entityToBeChanged) throws AtlasBaseException { + Map> operationListMap = new HashMap<>(); + + Set requiredClassificationKeys = Stream.concat( + incomingEntityHeader.getRemoveClassifications().stream(), + incomingEntityHeader.getUpdateClassifications().stream() + ).map(this::generateClassificationComparisonKey) + .collect(Collectors.toSet()); + + Set preExistingClassificationKeys = entityToBeChanged.getClassifications() + .stream() + .map(this::generateClassificationComparisonKey) + .collect(Collectors.toSet()); + + Set diff = requiredClassificationKeys.stream() + .filter(key -> !preExistingClassificationKeys.contains(key)) + .collect(Collectors.toSet()); + + if (!diff.isEmpty()) { + String firstTypeName = diff.iterator().next().split("\\|")[1]; + throw new AtlasBaseException(AtlasErrorCode.CLASSIFICATION_NOT_ASSOCIATED_WITH_ENTITY, firstTypeName); + } + + List filteredClassifications = incomingEntityHeader.getAppendClassifications() + .stream() + .filter(appendClassification -> !preExistingClassificationKeys.contains(generateClassificationComparisonKey(appendClassification))) + .collect(Collectors.toList()); + + bucket(PROCESS_DELETE, operationListMap, incomingEntityHeader.getRemoveClassifications()); + bucket(PROCESS_UPDATE, operationListMap, incomingEntityHeader.getUpdateClassifications()); + bucket(PROCESS_ADD, operationListMap, filteredClassifications); + return operationListMap; + } + + private String generateClassificationComparisonKey(AtlasClassification classification) { + return classification.getEntityGuid() + "|" + classification.getTypeName(); + } + private void commitChanges(String entityGuid, String typeName, Map> operationListMap) throws AtlasBaseException { if (MapUtils.isEmpty(operationListMap)) { return; From 4dc6e579feff004d03582144f667ca7a581fafde Mon Sep 17 00:00:00 2001 From: Rahul Madan Date: Wed, 27 Nov 2024 17:38:14 +0530 Subject: [PATCH 096/241] update logic completed partially --- .../lineage/LineagePreProcessor.java | 160 ++++++++++++++---- 1 file changed, 124 insertions(+), 36 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java index cd24ac943d..f7e5b2edfb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java @@ -113,12 +113,47 @@ private void processCreateLineageProcess(AtlasEntity entity, ArrayList connectio } } - private void processUpdateLineageProcess(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context, ArrayList connectionProcessList) { - // check if connection lineage exists - // add owner connection process - if(!connectionProcessList.isEmpty()){ - entity.setAttribute(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, connectionProcessList); + private void processUpdateLineageProcess(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context, ArrayList newConnectionProcessList) throws AtlasBaseException { + // Get the old parentConnectionProcessQualifiedName from the existing vertex + List oldConnectionProcessList = null; + try { + Object propertyValue = vertex.getProperty(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, Object.class); + if (propertyValue instanceof String) { + oldConnectionProcessList = Arrays.asList((String) propertyValue); + } else if (propertyValue instanceof List) { + oldConnectionProcessList = (List) propertyValue; + } else if (propertyValue != null) { + oldConnectionProcessList = Collections.singletonList(propertyValue.toString()); + } else { + oldConnectionProcessList = Collections.emptyList(); + } + } catch (Exception e) { + oldConnectionProcessList = Collections.emptyList(); + } + + // Identify ConnectionProcesses to remove (present in old list but not in new list) + Set connectionProcessesToRemove = new HashSet<>(oldConnectionProcessList); + connectionProcessesToRemove.removeAll(newConnectionProcessList); + + // Identify ConnectionProcesses to add (present in new list but not in old list) + Set connectionProcessesToAdd = new HashSet<>(newConnectionProcessList); + connectionProcessesToAdd.removeAll(oldConnectionProcessList); + + // For each ConnectionProcess to remove + for (String connectionProcessQn : connectionProcessesToRemove) { + // Check if more child Processes exist for this ConnectionProcess + if (!checkIfMoreChildProcessExistForConnectionProcess(connectionProcessQn)) { + // Delete the ConnectionProcess + deleteConnectionProcess(connectionProcessQn); + } + // Update __hasLineage for involved Connections + updateConnectionsHasLineageForConnectionProcess(connectionProcessQn); } + + // For new ConnectionProcesses, we've already created or retrieved them in getConnectionProcessQNsForTheGivenInputOutputs + + // Update the Process entity's parentConnectionProcessQualifiedName attribute + entity.setAttribute(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, newConnectionProcessList); } private AtlasEntity createConnectionProcessEntity(Map connectionProcessInfo) throws AtlasBaseException { @@ -348,7 +383,6 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processDeleteLineageProcess"); try { - // handle both soft and hard deletes // Collect all connections involved in the process being deleted AtlasEntity processEntity = entityRetriever.toAtlasEntity(vertex); @@ -384,18 +418,27 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { // Collect affected connections from connection processes to be deleted Set connectionProcessQNs = new HashSet<>(); - Object rawProperty = vertex.getProperty(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, Object.class); - - if (rawProperty instanceof List) { - // If the property is a List, cast and add all elements - List propertyList = (List) rawProperty; - connectionProcessQNs.addAll(propertyList); - } else if (rawProperty instanceof String) { - // If it's a single String, add it to the set - connectionProcessQNs.add((String) rawProperty); - } else if (rawProperty != null) { - // Handle other object types if necessary - connectionProcessQNs.add(rawProperty.toString()); + // Handle both single value and multi-value cases + try { + // Try getting as a list first + Iterable propertyValues = vertex.getPropertyValues(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, Object.class); + if (propertyValues != null) { + for (Object value : propertyValues) { + if (value != null) { + connectionProcessQNs.add(value.toString()); + } + } + } + } catch (Exception e) { + // If getting as list fails, try getting as single value + try { + String singleValue = vertex.getProperty(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, String.class); + if (StringUtils.isNotEmpty(singleValue)) { + connectionProcessQNs.add(singleValue); + } + } catch (Exception ex) { + LOG.warn("Error getting parentConnectionProcessQualifiedName property: {}", ex.getMessage()); + } } if (connectionProcessQNs.isEmpty()) { @@ -416,17 +459,13 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { AtlasVertex connectionProcessVertex = entityRetriever.getEntityVertex(atlasObjectId); AtlasEntity connectionProcess = entityRetriever.toAtlasEntity(connectionProcessVertex); - // Safely get connection qualified names - String inputConnQN = getConnectionQualifiedName(connectionProcess, "input"); - String outputConnQN = getConnectionQualifiedName(connectionProcess, "output"); + // Get all input and output connections + List inputConnQNs = getConnectionQualifiedNames(connectionProcess, "inputs"); + List outputConnQNs = getConnectionQualifiedNames(connectionProcess, "outputs"); - // Add non-null qualified names to affected connections - if (StringUtils.isNotEmpty(inputConnQN)) { - affectedConnections.add(inputConnQN); - } - if (StringUtils.isNotEmpty(outputConnQN)) { - affectedConnections.add(outputConnQN); - } + // Add all connections to affected set + affectedConnections.addAll(inputConnQNs); + affectedConnections.addAll(outputConnQNs); // Delete the connection process entityStore.deleteById(connectionProcessVertex.getProperty("__guid", String.class)); @@ -452,21 +491,70 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { } } - // Helper method to safely get connection qualified name - private String getConnectionQualifiedName(AtlasEntity connectionProcess, String attributeName) { + private List getConnectionQualifiedNames(AtlasEntity connectionProcess, String attributeName) { + List connectionQualifiedNames = new ArrayList<>(); try { Object relationshipAttr = connectionProcess.getRelationshipAttribute(attributeName); - if (relationshipAttr instanceof AtlasObjectId) { - AtlasObjectId connObjectId = (AtlasObjectId) relationshipAttr; - Map uniqueAttributes = connObjectId.getUniqueAttributes(); - if (uniqueAttributes != null) { - return (String) uniqueAttributes.get(QUALIFIED_NAME); + if (relationshipAttr instanceof List) { + List connObjectIds = (List) relationshipAttr; + for (AtlasObjectId connObjectId : connObjectIds) { + Map uniqueAttributes = connObjectId.getUniqueAttributes(); + if (uniqueAttributes != null) { + String qualifiedName = (String) uniqueAttributes.get(QUALIFIED_NAME); + if (StringUtils.isNotEmpty(qualifiedName)) { + connectionQualifiedNames.add(qualifiedName); + } + } } } } catch (Exception e) { LOG.warn("Error getting {} qualified name for connection process {}: {}", attributeName, connectionProcess.getGuid(), e.getMessage()); } - return null; + return connectionQualifiedNames; + } + + private void deleteConnectionProcess(String connectionProcessQn) throws AtlasBaseException { + AtlasObjectId atlasObjectId = new AtlasObjectId(); + atlasObjectId.setTypeName(CONNECTION_PROCESS_ENTITY_TYPE); + atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME, connectionProcessQn)); + + try { + AtlasVertex connectionProcessVertex = entityRetriever.getEntityVertex(atlasObjectId); + entityStore.deleteById(connectionProcessVertex.getProperty("__guid", String.class)); + } catch (AtlasBaseException exp) { + if (!exp.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)) { + throw exp; + } + } + } + + private void updateConnectionsHasLineageForConnectionProcess(String connectionProcessQn) throws AtlasBaseException { + // Get the ConnectionProcess entity + AtlasObjectId atlasObjectId = new AtlasObjectId(); + atlasObjectId.setTypeName(CONNECTION_PROCESS_ENTITY_TYPE); + atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME, connectionProcessQn)); + + try { + AtlasVertex connectionProcessVertex = entityRetriever.getEntityVertex(atlasObjectId); + AtlasEntity connectionProcess = entityRetriever.toAtlasEntity(connectionProcessVertex); + + // Get input and output connections + List inputConnQNs = getConnectionQualifiedNames(connectionProcess, "inputs"); + List outputConnQNs = getConnectionQualifiedNames(connectionProcess, "outputs"); + + // For each connection, check and update __hasLineage + Set connectionsToUpdate = new HashSet<>(); + connectionsToUpdate.addAll(inputConnQNs); + connectionsToUpdate.addAll(outputConnQNs); + + for (String connectionQN : connectionsToUpdate) { + checkAndUpdateConnectionLineage(connectionQN); + } + } catch (AtlasBaseException exp) { + if (!exp.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)) { + throw exp; + } + } } } From b7aeeefa8feb9c6d7eca2dddfcc5d259643dadc1 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 28 Nov 2024 00:20:55 +0530 Subject: [PATCH 097/241] Move audit log checks to authrest itself --- .../admin/client/AtlasAuthAdminClient.java | 2 +- .../admin/client/AtlasAuthRESTClient.java | 12 ++-- .../atlas/plugin/util/PolicyRefresher.java | 20 ++---- .../CachePolicyTransformerImpl.java | 49 ++++++++------- .../org/apache/atlas/web/rest/AuthREST.java | 61 ++++++++++++++++--- 5 files changed, 92 insertions(+), 52 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java index a79407ba9c..7d2f675d9e 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java @@ -8,7 +8,7 @@ public interface AtlasAuthAdminClient { void init(RangerPluginConfig config); - ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception; + ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws Exception; RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java index 10c97008c3..c2a7dafd83 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java @@ -32,6 +32,7 @@ public class AtlasAuthRESTClient implements AtlasAuthAdminClient { private static final String PARAM_LAST_UPDATED_TIME = "lastUpdatedTime"; private static final String PARAM_PLUGIN_ID = "pluginId"; + private static final String PARAM_USE_POLICY_DELTA = "usePolicyDelta"; @Override public void init(RangerPluginConfig config) { @@ -66,20 +67,20 @@ public String getPluginId(String serviceName, String appId) { } @Override - public ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception { - URI uri = buildURI("/download/policies/" + serviceName, lastUpdatedTimeInMillis); + public ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws Exception { + URI uri = buildURI("/download/policies/" + serviceName, lastUpdatedTimeInMillis, usePolicyDelta); return sendRequestAndGetResponse(uri, ServicePolicies.class); } @Override public RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception { - URI uri = buildURI("/download/roles/" + serviceName, lastUpdatedTimeInMillis); + URI uri = buildURI("/download/roles/" + serviceName, lastUpdatedTimeInMillis, false); return sendRequestAndGetResponse(uri, RangerRoles.class); } @Override public RangerUserStore getUserStoreIfUpdated(long lastUpdatedTimeInMillis) throws Exception { - URI uri = buildURI("/download/users/" + serviceName, lastUpdatedTimeInMillis); + URI uri = buildURI("/download/users/" + serviceName, lastUpdatedTimeInMillis, false); return sendRequestAndGetResponse(uri, RangerUserStore.class); } @@ -114,13 +115,14 @@ private T sendRequestAndGetResponse(URI uri, Class responseClass) throws return null; } - private URI buildURI(String path, long lastUpdatedTimeInMillis) throws URISyntaxException { + private URI buildURI(String path, long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws URISyntaxException { return new URIBuilder() .setScheme(SCHEME) .setHost(adminUrl) .setPath(path) .setParameter(PARAM_LAST_UPDATED_TIME, String.valueOf(lastUpdatedTimeInMillis)) .setParameter(PARAM_PLUGIN_ID, pluginId) + .setParameter(PARAM_USE_POLICY_DELTA, String.valueOf(usePolicyDelta)) .build(); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 4a5dc552bd..5946dc7d79 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -21,7 +21,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; @@ -45,8 +44,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import static org.apache.atlas.ApplicationProperties.DELTA_BASED_REFRESH; - public class PolicyRefresher extends Thread { private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); @@ -335,20 +332,15 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { - if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null) { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { RangerRESTUtils restUtils = new RangerRESTUtils(); CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); - if (lastUpdatedTiemInMillis == -1) { - svcPolicies = transformer.getPoliciesAll(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); - } else if (this.enableDeltaBasedRefresh) { - svcPolicies = transformer.getPoliciesDelta(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); - } else { - svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); - } + + svcPolicies = transformer.getPoliciesAll(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis); } else { - svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh); } boolean isUpdated = svcPolicies != null; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index ebe8812db1..6715103ce0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -142,7 +142,7 @@ public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Long lastUpdatedTime) { + public ServicePolicies createPoliciesWithDelta(String serviceName, Map policyChanges, List atlasPolicies) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); ServicePolicies servicePolicies = new ServicePolicies(); @@ -161,7 +161,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon String serviceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, serviceName); servicePolicies.setServiceDef(getResourceAsObject(serviceDefName, RangerServiceDef.class)); - List policiesDelta = getServicePoliciesDelta(service, 250, lastUpdatedTime); + List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasPolicies); servicePolicies.setPolicyDeltas(policiesDelta); @@ -189,8 +189,6 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon } } - - LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); LOG.info("PolicyDelta: Found and set {} policies as delta and {} tag policies", servicePolicies.getPolicyDeltas().size(), servicePolicies.getTagPolicies().getPolicies().size()); } @@ -286,38 +284,23 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private List getServicePoliciesDelta(AtlasEntityHeader service, int batchSize, Long lastUpdatedTime) throws AtlasBaseException, IOException { - + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getServicePoliciesWithDelta." + serviceName); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getRangerPolicyDelta." + serviceName); List policyDeltas = new ArrayList<>(); - - // TODO: when getServicePolicies (without delta) is removed, merge the pagination for audit logs and policy fetch into one - List auditEvents = queryPoliciesAuditLogs(serviceName, lastUpdatedTime, batchSize); - Map policiesWithChangeType = new HashMap<>(); - for (EntityAuditEventV2 event : auditEvents) { - if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policiesWithChangeType.containsKey(event.getEntityId())) { - policiesWithChangeType.put(event.getEntityId(), event.getAction()); - } - } - LOG.info("PolicyDelta: {}: Total audit logs found = {}, events for {} ({}) = {}", serviceName, auditEvents.size(), POLICY_ENTITY_TYPE, policiesWithChangeType.size(), policiesWithChangeType); - if (policiesWithChangeType.isEmpty()) { + if (policyChanges.isEmpty()) { return policyDeltas; } - ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); - // this will have less policies as deleted won't be found - List atlasPolicies = getAtlasPolicies(serviceName, batchSize, policyGuids); - List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); } for (RangerPolicy policy : rangerPolicies) { - Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getAtlasGuid())); + Integer changeType = auditEventToDeltaChangeType.get(policyChanges.get(policy.getAtlasGuid())); RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); policyDeltas.add(delta); } @@ -364,6 +347,26 @@ private List queryPoliciesAuditLogs(String serviceName, Long return events; } + public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List events) { + Map policyChanges = new HashMap<>(); + for (EntityAuditEventV2 event : events) { + if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policyChanges.containsKey(event.getEntityId())) { + policyChanges.put(event.getEntityId(), event.getAction()); + } + } + + List atlasPolicies = new ArrayList<>(); + for (EntityAuditEventV2 event : events) { + AtlasEntityHeader policy = event.getEntityHeader(); + if (policy != null) { + atlasPolicies.add(policy); + } + } + LOG.info("PolicyDelta: {}: Found {} policy changes with {} policies", serviceName, policyChanges.size(), atlasPolicies.size()); + + return createPoliciesWithDelta(serviceName, policyChanges, atlasPolicies); + } + private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, String serviceType, String serviceName) throws IOException, AtlasBaseException { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index bb945ffa97..14ed3b1101 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -21,6 +21,7 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.plugin.util.KeycloakUserStore; @@ -51,10 +52,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; @@ -144,6 +142,7 @@ public RangerUserStore downloadUserStore(@PathParam("serviceName") final String @Timed public ServicePolicies downloadPolicies(@PathParam("serviceName") final String serviceName, @QueryParam("pluginId") String pluginId, + @DefaultValue("false") @QueryParam("usePolicyDelta") boolean usePolicyDelta, @DefaultValue("0") @QueryParam("lastUpdatedTime") Long lastUpdatedTime) throws AtlasBaseException { AtlasPerfTracer perf = null; @@ -152,12 +151,16 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); } - if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { - return null; + ServicePolicies ret; + if (usePolicyDelta) { + List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); + ret = policyTransformer.extractAndTransformPolicyDelta(serviceName, auditEvents); + } else { + if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { + return null; + } + ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); } - - ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); - updateLastSync(serviceName); return ret; @@ -166,6 +169,46 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s } } + private List getPolicyAuditLogs(String serviceName, long lastUpdatedTime) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthREST.getPolicyAuditLogs." + serviceName); + + List entityUpdateToWatch = new ArrayList<>(); + entityUpdateToWatch.add(POLICY_ENTITY_TYPE); + entityUpdateToWatch.add(PERSONA_ENTITY_TYPE); + entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); + + AuditSearchParams parameters = new AuditSearchParams(); + Map dsl = getMap("size", 100); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); + + lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; + mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + parameters.setDsl(dsl); + parameters.setAttributes(new HashSet<>(Arrays.asList( + "qualifiedName", + "name" + ))); + + List events = new ArrayList<>(); + try { + EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + } catch (AtlasBaseException e) { + LOG.error("ERROR in getPolicyAuditLogs while fetching entity audits {}: ", e.getMessage()); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + + return events; + } + private void updateLastSync(String serviceName) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.updateLastSync." + serviceName); From f88ecaeb6b14e1438b9fe976cbe809a6e1a5f15e Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 28 Nov 2024 12:01:55 +0530 Subject: [PATCH 098/241] Remove auditRepo from policy refresher and transformer --- .../atlas/plugin/util/PolicyRefresher.java | 14 ++----- .../CachePolicyTransformerImpl.java | 41 +------------------ 2 files changed, 4 insertions(+), 51 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 5946dc7d79..b9331a547c 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -68,9 +68,7 @@ public class PolicyRefresher extends Thread { private long lastActivationTimeInMillis; private boolean policiesSetInPlugin; private boolean serviceDefSetInPlugin; - private Configuration atlasConfig; private boolean enableDeltaBasedRefresh; - private ESBasedAuditRepository auditRepository; public PolicyRefresher(RangerBasePlugin plugIn) { @@ -111,14 +109,8 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.userStoreProvider = new RangerUserStoreProvider(getServiceType(), appId, getServiceName(), atlasAuthAdminClient, cacheDir, pluginConfig); this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); - try { - this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); - this.auditRepository = new ESBasedAuditRepository(atlasConfig); - this.auditRepository.start(); - } catch (AtlasException e) { - LOG.error("PolicyDelta: Error while reading atlas configuration", e); - this.enableDeltaBasedRefresh = false; - } + this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); + LOG.info("PolicyRefresher(serviceName=" + serviceName + ") - delta based policy refresh is enabled"); setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); @@ -334,7 +326,7 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { RangerRESTUtils restUtils = new RangerRESTUtils(); - CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); + CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); svcPolicies = transformer.getPoliciesAll(serviceName, restUtils.getPluginId(serviceName, plugIn.getAppId()), diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 6715103ce0..3edd214e3b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -112,15 +112,13 @@ public class CachePolicyTransformerImpl { private PurposeCachePolicyTransformer purposeTransformer; private AtlasEntityHeader service; - private final ESBasedAuditRepository auditRepository; private final Map auditEventToDeltaChangeType; @Inject - public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry, ESBasedAuditRepository auditRepository) throws AtlasBaseException { + public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { this.graph = new AtlasJanusGraph(); this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); - this.auditRepository = auditRepository; personaTransformer = new PersonaCachePolicyTransformer(entityRetriever); purposeTransformer = new PurposeCachePolicyTransformer(entityRetriever); @@ -310,43 +308,6 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } - private List queryPoliciesAuditLogs(String serviceName, Long afterTime, int batchSize) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.queryPoliciesAuditLogs." + serviceName); - - List entityUpdateToWatch = new ArrayList<>(); - entityUpdateToWatch.add(POLICY_ENTITY_TYPE); - entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); - - AuditSearchParams parameters = new AuditSearchParams(); - Map dsl = getMap("size", batchSize); - - List> mustClauseList = new ArrayList<>(); - mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); - afterTime = afterTime == -1 ? 0 : afterTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gte", afterTime)))); - - List> sortList = new ArrayList<>(0); - sortList.add(getMap("created", getMap("order", "desc"))); - dsl.put("sort", sortList); - - dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - - parameters.setDsl(dsl); - - List events = new ArrayList<>(); - try { - EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); - if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { - events = result.getEntityAudits(); - } - } catch (AtlasBaseException e) { - LOG.error("ERROR in queryPoliciesAuditLogs while fetching entity audits {}: ", e.getMessage(), e); - } finally { - RequestContext.get().endMetricRecord(recorder); - } - return events; - } - public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List events) { Map policyChanges = new HashMap<>(); for (EntityAuditEventV2 event : events) { From 0363a1c1b1db70badb59591a128742b49323e03f Mon Sep 17 00:00:00 2001 From: Rahul Madan Date: Thu, 28 Nov 2024 17:25:30 +0530 Subject: [PATCH 099/241] simplified delete --- .../lineage/LineagePreProcessor.java | 154 ++++++++---------- 1 file changed, 66 insertions(+), 88 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java index f7e5b2edfb..e36469d723 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/lineage/LineagePreProcessor.java @@ -286,7 +286,6 @@ private boolean hasActiveConnectionProcesses(AtlasVertex connectionVertex) { private ArrayList getConnectionProcessQNsForTheGivenInputOutputs(AtlasEntity processEntity) throws AtlasBaseException{ - // check connection lineage exists or not // check if connection lineage exists Map entityAttrValues = processEntity.getRelationshipAttributes(); @@ -383,111 +382,90 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processDeleteLineageProcess"); try { - // Collect all connections involved in the process being deleted AtlasEntity processEntity = entityRetriever.toAtlasEntity(vertex); + Set connectionsToCheck = new HashSet<>(); + + // Collect connections from inputs and outputs + collectConnectionsFromAssets(processEntity.getRelationshipAttribute("inputs"), connectionsToCheck); + collectConnectionsFromAssets(processEntity.getRelationshipAttribute("outputs"), connectionsToCheck); + + // Handle connection processes + Object rawConnectionProcessQNs = vertex.getProperty(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, Object.class); + if (rawConnectionProcessQNs == null) { + return; + } - Set involvedConnections = new HashSet<>(); - - // Retrieve inputs and outputs from the process - List inputs = (List) processEntity.getRelationshipAttribute("inputs"); - List outputs = (List) processEntity.getRelationshipAttribute("outputs"); + Set connectionProcessQNs = new HashSet<>(); + if (rawConnectionProcessQNs instanceof List) { + connectionProcessQNs.addAll((List) rawConnectionProcessQNs); + } else { + connectionProcessQNs.add(rawConnectionProcessQNs.toString()); + } - if (inputs == null) inputs = Collections.emptyList(); - if (outputs == null) outputs = Collections.emptyList(); + // Process each connection process + for (String connectionProcessQn : connectionProcessQNs) { + if (!checkIfMoreChildProcessExistForConnectionProcess(connectionProcessQn)) { + deleteConnectionProcess(connectionProcessQn, connectionsToCheck); + } + } - List allAssets = new ArrayList<>(); - allAssets.addAll(inputs); - allAssets.addAll(outputs); + // Update hasLineage flags + for (String connectionQN : connectionsToCheck) { + checkAndUpdateConnectionLineage(connectionQN); + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } - // For each asset, get its connection and add to involvedConnections - for (AtlasObjectId assetId : allAssets) { + private void collectConnectionsFromAssets(Object assets, Set connections) { + if (assets instanceof List) { + for (AtlasObjectId assetId : (List) assets) { try { AtlasVertex assetVertex = entityRetriever.getEntityVertex(assetId); - Map assetConnectionAttributes = fetchAttributes(assetVertex, FETCH_ENTITY_ATTRIBUTES); - if (assetConnectionAttributes != null) { - String connectionQN = assetConnectionAttributes.get(CONNECTION_QUALIFIED_NAME); - if (StringUtils.isNotEmpty(connectionQN)) { - involvedConnections.add(connectionQN); - } + Map connectionAttr = fetchAttributes(assetVertex, FETCH_ENTITY_ATTRIBUTES); + String connectionQN = connectionAttr.get(CONNECTION_QUALIFIED_NAME); + if (StringUtils.isNotEmpty(connectionQN)) { + connections.add(connectionQN); } } catch (AtlasBaseException e) { LOG.warn("Failed to retrieve connection for asset {}: {}", assetId.getGuid(), e.getMessage()); } } + } + } - // Collect affected connections from connection processes to be deleted - Set connectionProcessQNs = new HashSet<>(); - - // Handle both single value and multi-value cases - try { - // Try getting as a list first - Iterable propertyValues = vertex.getPropertyValues(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, Object.class); - if (propertyValues != null) { - for (Object value : propertyValues) { - if (value != null) { - connectionProcessQNs.add(value.toString()); - } - } - } - } catch (Exception e) { - // If getting as list fails, try getting as single value - try { - String singleValue = vertex.getProperty(PARENT_CONNECTION_PROCESS_QUALIFIED_NAME, String.class); - if (StringUtils.isNotEmpty(singleValue)) { - connectionProcessQNs.add(singleValue); - } - } catch (Exception ex) { - LOG.warn("Error getting parentConnectionProcessQualifiedName property: {}", ex.getMessage()); - } - } + private void deleteConnectionProcess(String connectionProcessQn, Set affectedConnections) throws AtlasBaseException { + AtlasObjectId atlasObjectId = new AtlasObjectId(); + atlasObjectId.setTypeName(CONNECTION_PROCESS_ENTITY_TYPE); + atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME, connectionProcessQn)); - if (connectionProcessQNs.isEmpty()) { - return; + try { + AtlasVertex connectionProcessVertex = entityRetriever.getEntityVertex(atlasObjectId); + AtlasEntity connectionProcess = entityRetriever.toAtlasEntity(connectionProcessVertex); + + // Add connection QNs to affected connections + addConnectionToSet(connectionProcess.getRelationshipAttribute("input"), affectedConnections); + addConnectionToSet(connectionProcess.getRelationshipAttribute("output"), affectedConnections); + + entityStore.deleteById(connectionProcessVertex.getProperty("__guid", String.class)); + } catch (AtlasBaseException e) { + if (!e.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)) { + throw e; } + } + } - Set affectedConnections = new HashSet<>(); - - // Process each connection process - for (String connectionProcessQn : connectionProcessQNs) { - if (!checkIfMoreChildProcessExistForConnectionProcess(connectionProcessQn)) { - AtlasObjectId atlasObjectId = new AtlasObjectId(); - atlasObjectId.setTypeName(CONNECTION_PROCESS_ENTITY_TYPE); - atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME, connectionProcessQn)); - - try { - // Get connection process before deletion to track affected connections - AtlasVertex connectionProcessVertex = entityRetriever.getEntityVertex(atlasObjectId); - AtlasEntity connectionProcess = entityRetriever.toAtlasEntity(connectionProcessVertex); - - // Get all input and output connections - List inputConnQNs = getConnectionQualifiedNames(connectionProcess, "inputs"); - List outputConnQNs = getConnectionQualifiedNames(connectionProcess, "outputs"); - - // Add all connections to affected set - affectedConnections.addAll(inputConnQNs); - affectedConnections.addAll(outputConnQNs); - - // Delete the connection process - entityStore.deleteById(connectionProcessVertex.getProperty("__guid", String.class)); - } catch (AtlasBaseException exp) { - if (!exp.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)) { - throw exp; - } - } + private void addConnectionToSet(Object connectionAttr, Set connections) { + if (connectionAttr instanceof AtlasObjectId) { + AtlasObjectId connObjectId = (AtlasObjectId) connectionAttr; + Map uniqueAttributes = connObjectId.getUniqueAttributes(); + if (uniqueAttributes != null) { + String qn = (String) uniqueAttributes.get(QUALIFIED_NAME); + if (StringUtils.isNotEmpty(qn)) { + connections.add(qn); } } - - // Combine involved and affected connections - Set connectionsToCheck = new HashSet<>(); - connectionsToCheck.addAll(involvedConnections); - connectionsToCheck.addAll(affectedConnections); - - // Check and update hasLineage for all connections involved - for (String connectionQN : connectionsToCheck) { - checkAndUpdateConnectionLineage(connectionQN); - } - } finally { - RequestContext.get().endMetricRecord(metricRecorder); } } From 571635802a417b8e6ef6905f699d5a6fbb32078f Mon Sep 17 00:00:00 2001 From: Pavan Manish Date: Thu, 28 Nov 2024 17:44:36 +0530 Subject: [PATCH 100/241] FT-800: Added better message logging for unhandled exceptions --- .../atlas/web/errors/ExceptionMapperUtil.java | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java index e2511e5d50..50b09f11ee 100644 --- a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java +++ b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java @@ -19,15 +19,73 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; public class ExceptionMapperUtil { protected static final Logger LOGGER = LoggerFactory.getLogger(ExceptionMapperUtil.class); @SuppressWarnings("UnusedParameters") protected static String formatErrorMessage(long id, Exception exception) { - return String.format("There was an error processing your request. It has been logged (ID %016x).", id); + StringBuilder response = new StringBuilder(); + + // Add error ID and general error message + response.append("{\n") + .append(String.format(" \"errorId\": \"%016x\",\n", id)) + .append(" \"message\": \"There was an error processing your request.\",\n") + .append(" \"causes\": [\n"); + + // Traverse through the chain of causes + List causes = new ArrayList<>(); + Throwable currentException = exception; + while (currentException != null) { + causes.add(formatCause(currentException)); + currentException =currentException .getCause(); + } + + // Add all formatted causes to the response + for (int i = 0; i < causes.size(); i++) { + response.append(causes.get(i)); + if (i < causes.size() - 1) { + response.append(",\n"); + } + } + + // Close the JSON structure + response.append("\n ]\n") + .append("}"); + + return response.toString(); + } + + // Helper method to format a single exception cause + private static String formatCause(Throwable exception) { + StringBuilder cause = new StringBuilder(); + + // Extract location details from the first stack trace element + StackTraceElement[] stackTrace = exception.getStackTrace(); + String location = "Unavailable"; + if (stackTrace != null && stackTrace.length > 0) { + StackTraceElement element = stackTrace[0]; + location = String.format("%s.%s (%s:%d)", + element.getClassName(), + element.getMethodName(), + element.getFileName(), + element.getLineNumber()); + } + + // Build JSON object for this cause + cause.append(" {\n") + .append(" \"errorType\": \"").append(exception.getClass().getName()).append("\",\n") + .append(" \"errorMessage\": \"").append(exception.getMessage() != null ? exception.getMessage() : "No additional information provided").append("\",\n") + .append(" \"location\": \"").append(location).append("\"\n") + .append(" }"); + + return cause.toString(); } + + protected static void logException(long id, Exception exception) { LOGGER.error(formatLogMessage(id, exception), exception); } From 71631238aed20e19fd15dcc3b3b58d21286470eb Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 28 Nov 2024 14:05:29 +0530 Subject: [PATCH 101/241] use preloaded properties --- .../store/graph/v2/EntityGraphRetriever.java | 79 +++++++++++++++---- 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 34d3036042..f0b608d029 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -67,6 +67,10 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.janusgraph.core.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -88,10 +92,7 @@ import java.util.Objects; import java.util.Queue; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; +import java.util.concurrent.*; import java.util.stream.Collectors; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; @@ -131,6 +132,7 @@ @Component public class EntityGraphRetriever { private static final Logger LOG = LoggerFactory.getLogger(EntityGraphRetriever.class); + private static final int BATCH_SIZE = 300; private static final String GLOSSARY_TERM_DISPLAY_NAME_ATTR = "name"; public static final String TERM_RELATION_NAME = "AtlasGlossarySemanticAssignment"; @@ -1009,17 +1011,45 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } + private Map preloadProperties(AtlasVertex entityVertex) { + Map propertiesMap = new HashMap<>(); + + // Execute the traversal to fetch properties + GraphTraversal> traversal = graph.V(entityVertex.getId()).properties(); + + // Iterate through the resulting VertexProperty objects + while (traversal.hasNext()) { + VertexProperty property = traversal.next(); + + if (property.isPresent()) { // Ensure the property exists + propertiesMap.put(property.key(), property.value()); + } + } + + return propertiesMap; + } + private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); AtlasEntityHeader ret = new AtlasEntityHeader(); try { - String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); - String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); - Boolean isIncomplete = isEntityIncomplete(entityVertex); + //pre-fetching the properties + Map properties = preloadProperties(entityVertex); + + String typeName = (String) properties.get(Constants.TYPE_NAME_PROPERTY_KEY); + String guid = (String) properties.get(Constants.GUID_PROPERTY_KEY); + + Integer value = (Integer)properties.get(Constants.IS_INCOMPLETE_PROPERTY_KEY); + Boolean isIncomplete = value != null && value.equals(INCOMPLETE_ENTITY_VALUE) ? Boolean.TRUE : Boolean.FALSE; ret.setTypeName(typeName); ret.setGuid(guid); ret.setStatus(GraphHelper.getStatus(entityVertex)); + + String state = (String)properties.get(Constants.STATE_PROPERTY_KEY); + Id.EntityState entityState = state == null ? null : Id.EntityState.valueOf(state); + ret.setStatus((entityState == Id.EntityState.DELETED) ? AtlasEntity.Status.DELETED : AtlasEntity.Status.ACTIVE); + RequestContext context = RequestContext.get(); boolean includeClassifications = context.includeClassifications(); boolean includeClassificationNames = context.isIncludeClassificationNames(); @@ -1031,10 +1061,10 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, ret.setIsIncomplete(isIncomplete); ret.setLabels(getLabels(entityVertex)); - ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); - ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); - ret.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); - ret.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); + ret.setCreatedBy(properties.get(CREATED_BY_KEY) != null ? (String) properties.get(CREATED_BY_KEY) : null); + ret.setUpdatedBy(properties.get(MODIFIED_BY_KEY) != null ? (String) properties.get(MODIFIED_BY_KEY) : null); + ret.setCreateTime(new Date((Long)properties.get(TIMESTAMP_PROPERTY_KEY))); + ret.setUpdateTime(new Date((Long)properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY))); if(RequestContext.get().includeMeanings()) { List termAssignmentHeaders = mapAssignedTerms(entityVertex); @@ -1043,7 +1073,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) .collect(Collectors.toList())); } - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); // this is not costly if (entityType != null) { for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { @@ -1054,12 +1084,15 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, } } - Object displayText = getDisplayText(entityVertex, entityType); - - if (displayText != null) { - ret.setDisplayText(displayText.toString()); + if(properties.get(NAME) != null){ + ret.setDisplayText(properties.get(NAME).toString()); + } else if(properties.get(DISPLAY_NAME) != null) { + ret.setDisplayText(properties.get(DISPLAY_NAME).toString()); + } else if(properties.get(QUALIFIED_NAME) != null) { + ret.setDisplayText(properties.get(QUALIFIED_NAME).toString()); } + //attributes = only the attributes of entityType if (CollectionUtils.isNotEmpty(attributes)) { for (String attrName : attributes) { AtlasAttribute attribute = entityType.getAttribute(attrName); @@ -1078,7 +1111,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, } } - Object attrValue = getVertexAttribute(entityVertex, attribute); + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, properties); //use prefetch cache if (attrValue != null) { ret.setAttribute(attrName, attrValue); @@ -1709,6 +1742,18 @@ public Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) t return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null; } + public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute attribute, Map properties) throws AtlasBaseException { + if (vertex == null || attribute == null) { + return null; + } + + if (properties.get(attribute) != null) { + return properties.get(attribute); + } + + return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null; + } + private Object getVertexAttributeIgnoreInactive(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException { return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false, true, true) : null; } From ec800d6bda4096c23ab5a896263328c9fc8cf918 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Thu, 28 Nov 2024 23:10:50 +0530 Subject: [PATCH 102/241] preload janus graph vertex properties --- .github/workflows/maven.yml | 1 + .../store/graph/v2/EntityGraphRetriever.java | 20 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..ab8b455bbd 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - janusprefetchwith0.6 jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index f0b608d029..6f3bfd1342 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -70,7 +70,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; -import org.janusgraph.core.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -132,7 +131,6 @@ @Component public class EntityGraphRetriever { private static final Logger LOG = LoggerFactory.getLogger(EntityGraphRetriever.class); - private static final int BATCH_SIZE = 300; private static final String GLOSSARY_TERM_DISPLAY_NAME_ATTR = "name"; public static final String TERM_RELATION_NAME = "AtlasGlossarySemanticAssignment"; @@ -1036,7 +1034,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, //pre-fetching the properties Map properties = preloadProperties(entityVertex); - String typeName = (String) properties.get(Constants.TYPE_NAME_PROPERTY_KEY); + String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); //properties.get returns null String guid = (String) properties.get(Constants.GUID_PROPERTY_KEY); Integer value = (Integer)properties.get(Constants.IS_INCOMPLETE_PROPERTY_KEY); @@ -1044,7 +1042,6 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, ret.setTypeName(typeName); ret.setGuid(guid); - ret.setStatus(GraphHelper.getStatus(entityVertex)); String state = (String)properties.get(Constants.STATE_PROPERTY_KEY); Id.EntityState entityState = state == null ? null : Id.EntityState.valueOf(state); @@ -1063,8 +1060,8 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, ret.setCreatedBy(properties.get(CREATED_BY_KEY) != null ? (String) properties.get(CREATED_BY_KEY) : null); ret.setUpdatedBy(properties.get(MODIFIED_BY_KEY) != null ? (String) properties.get(MODIFIED_BY_KEY) : null); - ret.setCreateTime(new Date((Long)properties.get(TIMESTAMP_PROPERTY_KEY))); - ret.setUpdateTime(new Date((Long)properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY))); + ret.setCreateTime(properties.get(TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)properties.get(TIMESTAMP_PROPERTY_KEY)) : null); + ret.setUpdateTime(properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY)) : null); if(RequestContext.get().includeMeanings()) { List termAssignmentHeaders = mapAssignedTerms(entityVertex); @@ -1077,7 +1074,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, if (entityType != null) { for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { - Object attrValue = getVertexAttribute(entityVertex, headerAttribute); + Object attrValue = getVertexAttributePreFetchCache(entityVertex, headerAttribute, properties); if (attrValue != null) { ret.setAttribute(headerAttribute.getName(), attrValue); @@ -1111,7 +1108,8 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, } } - Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, properties); //use prefetch cache + //this is a call to cassandra + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, properties); //use prefetch cache if (attrValue != null) { ret.setAttribute(attrName, attrValue); @@ -1747,11 +1745,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return null; } - if (properties.get(attribute) != null) { - return properties.get(attribute); + if (properties.get(attribute.getName()) != null) { + return properties.get(attribute.getName()); } - return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null; + return null; } private Object getVertexAttributeIgnoreInactive(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException { From 33f90073d164ae701d0b87188c0a3e6e9168a004 Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 29 Nov 2024 08:15:44 +0530 Subject: [PATCH 103/241] add telemetry --- .../src/main/java/org/apache/atlas/AtlasConfiguration.java | 1 + .../repository/store/graph/v2/EntityGraphRetriever.java | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index df2bca7860..58cd14040d 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -117,6 +117,7 @@ public enum AtlasConfiguration { ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), + ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", false), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 6f3bfd1342..4c7ac69830 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1749,6 +1749,13 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return properties.get(attribute.getName()); } + if (AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean() && !attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE)) { + AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); + Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); + properties.put(attribute.getName(), mappedVertex); + RequestContext.get().endMetricRecord(nonPrimitiveAttributes); + } + return null; } From e060f617370440ea78695f52fdc804241cf990ad Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 29 Nov 2024 09:22:04 +0530 Subject: [PATCH 104/241] fix return type --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 4c7ac69830..ac16006f03 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1754,6 +1754,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); properties.put(attribute.getName(), mappedVertex); RequestContext.get().endMetricRecord(nonPrimitiveAttributes); + return mappedVertex; } return null; From 7f3148e3963e5c0a0c8e159a790b019b603e83b0 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:05:35 +0530 Subject: [PATCH 105/241] feat: clear tasks in executor --- .../main/java/org/apache/atlas/tasks/TaskQueueWatcher.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java index 9214cbfbf9..68dc122954 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java @@ -108,6 +108,7 @@ public void run() { } else { redisService.releaseDistributedLock(ATLAS_TASK_LOCK); } + fetcher.clearTasks(); Thread.sleep(pollInterval); } catch (InterruptedException interruptedException) { LOG.error("TaskQueueWatcher: Interrupted: thread is terminated, new tasks will not be loaded into the queue until next restart"); @@ -165,6 +166,10 @@ public void run() { public List getTasks() { return tasks; } + + public void clearTasks() { + this.tasks.clear(); + } } @PreDestroy From be801da3340a2a94d50b3083c5e5e2ab3a71b7a3 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 29 Nov 2024 13:11:42 +0530 Subject: [PATCH 106/241] feat: remove redundant thread, add commit for task status --- .../org/apache/atlas/tasks/TaskQueueWatcher.java | 14 ++++---------- .../java/org/apache/atlas/tasks/TaskRegistry.java | 3 +++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java index 68dc122954..435741a7ec 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java @@ -87,18 +87,12 @@ public void run() { LOG.debug("TaskQueueWatcher: running {}:{}", Thread.currentThread().getName(), Thread.currentThread().getId()); } while (shouldRun.get()) { + TasksFetcher fetcher = new TasksFetcher(registry); try { if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); continue; } - - TasksFetcher fetcher = new TasksFetcher(registry); - - Thread tasksFetcherThread = new Thread(fetcher); - tasksFetcherThread.start(); - tasksFetcherThread.join(); - List tasks = fetcher.getTasks(); if (CollectionUtils.isNotEmpty(tasks)) { final CountDownLatch latch = new CountDownLatch(tasks.size()); @@ -108,7 +102,6 @@ public void run() { } else { redisService.releaseDistributedLock(ATLAS_TASK_LOCK); } - fetcher.clearTasks(); Thread.sleep(pollInterval); } catch (InterruptedException interruptedException) { LOG.error("TaskQueueWatcher: Interrupted: thread is terminated, new tasks will not be loaded into the queue until next restart"); @@ -117,6 +110,7 @@ public void run() { LOG.error("TaskQueueWatcher: Exception occurred " + e.getMessage(), e); } finally { redisService.releaseDistributedLock(ATLAS_TASK_LOCK); + fetcher.clearTasks(); } } } @@ -146,7 +140,7 @@ private void submitAll(List tasks, CountDownLatch latch) { } } - static class TasksFetcher implements Runnable { + static class TasksFetcher { private TaskRegistry registry; private List tasks = new ArrayList<>(); @@ -154,7 +148,6 @@ public TasksFetcher(TaskRegistry registry) { this.registry = registry; } - @Override public void run() { if (LOG.isDebugEnabled()) { LOG.debug("TasksFetcher: Fetching tasks for queuing"); @@ -164,6 +157,7 @@ public void run() { } public List getTasks() { + run(); return tasks; } diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index 8c08a26121..bffa4c8672 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -430,6 +430,7 @@ public List getTasksForReQueueIndexSearch() { } else { LOG.warn(String.format("There is a mismatch on tasks status between ES and Cassandra for guid: %s", atlasTask.getGuid())); + vertex.setProperty(Constants.TASK_STATUS, atlasTask.getStatus().toString()); } } else { LOG.warn("Null vertex while re-queuing tasks at index {}", fetched); @@ -446,6 +447,8 @@ public List getTasksForReQueueIndexSearch() { } } catch (Exception e){ break; + } finally { + graph.commit(); } } From 7dd964217226df920bca60e576cd1ce038884cf5 Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 29 Nov 2024 14:37:53 +0530 Subject: [PATCH 107/241] add guard rails --- .../apache/atlas/utils/AtlasPerfMetrics.java | 19 ++- .../org/apache/atlas/AtlasConfiguration.java | 3 +- .../store/graph/v2/EntityGraphRetriever.java | 120 ++++++++++++++++++ .../repository/util/AccessControlUtils.java | 5 + .../java/org/apache/atlas/RequestContext.java | 7 + .../apache/atlas/web/rest/DiscoveryREST.java | 2 + 6 files changed, 154 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java index dd8a101ad5..f72cac2abf 100644 --- a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java +++ b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java @@ -50,6 +50,24 @@ public void recordMetric(MetricRecorder recorder) { } } + public void recordMetricWithInvocations(MetricRecorder recorder, long count) { + if (recorder != null) { + final String name = recorder.name; + final long timeTaken = recorder.getElapsedTime(); + + Metric metric = metrics.get(name); + + if (metric == null) { + metric = new Metric(name); + + metrics.put(name, metric); + } + + metric.invocations += count; + metric.totalTimeMSecs += timeTaken; + } + } + public void clear() { metrics.clear(); } @@ -129,7 +147,6 @@ public String getName() { public long getInvocations() { return invocations; } - public void setTotalTimeMSecs(long totalTimeMSecs) { this.totalTimeMSecs = totalTimeMSecs; } diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 58cd14040d..d97d4cd4a8 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -117,8 +117,9 @@ public enum AtlasConfiguration { ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), + ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", false), - + ATLAS_INDEXSEARCH_ATTRIBUTES_MIN_LIMIT("atlas.indexsearch.attributes.min.limit", 8), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), ATLAS_UD_RELATIONSHIPS_MAX_COUNT("atlas.ud.relationship.max.count", 100); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index ac16006f03..ac0dd6a140 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -49,6 +49,7 @@ import org.apache.atlas.repository.graphdb.AtlasElement; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.util.AccessControlUtils; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; import org.apache.atlas.type.AtlasEntityType; @@ -1027,9 +1028,128 @@ private Map preloadProperties(AtlasVertex entityVertex) { return propertiesMap; } + + private boolean isPolicyAttribute(Set attributes) { + Set exclusionSet = new HashSet<>(Arrays.asList(AccessControlUtils.ATTR_POLICY_TYPE, + AccessControlUtils.ATTR_POLICY_USERS, + AccessControlUtils.ATTR_POLICY_GROUPS, + AccessControlUtils.ATTR_POLICY_ROLES, + AccessControlUtils.ATTR_POLICY_ACTIONS, + AccessControlUtils.ATTR_POLICY_CATEGORY, + AccessControlUtils.ATTR_POLICY_SUB_CATEGORY, + AccessControlUtils.ATTR_POLICY_RESOURCES, + AccessControlUtils.ATTR_POLICY_IS_ENABLED, + AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY, + AccessControlUtils.ATTR_POLICY_SERVICE_NAME, + AccessControlUtils.ATTR_POLICY_PRIORITY, + AccessControlUtils.REL_ATTR_POLICIES, + AccessControlUtils.ATTR_SERVICE_SERVICE_TYPE, + AccessControlUtils.ATTR_SERVICE_TAG_SERVICE, + AccessControlUtils.ATTR_SERVICE_IS_ENABLED, + AccessControlUtils.ATTR_SERVICE_LAST_SYNC)); + + return exclusionSet.stream().anyMatch(attributes::contains); + } + private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { + boolean shouldPrefetch = attributes.size() > AtlasConfiguration.ATLAS_INDEXSEARCH_ATTRIBUTES_MIN_LIMIT.getInt() + && !isPolicyAttribute(attributes) + && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); + + if (shouldPrefetch) { + return mapVertexToAtlasEntityHeaderWithPrefetch(entityVertex, attributes); + } else { + return mapVertexToAtlasEntityHeaderWithoutPrefetch(entityVertex, attributes); + } + } + + private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithoutPrefetch(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); AtlasEntityHeader ret = new AtlasEntityHeader(); + try { + String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); + String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); + Boolean isIncomplete = isEntityIncomplete(entityVertex); + + ret.setTypeName(typeName); + ret.setGuid(guid); + ret.setStatus(GraphHelper.getStatus(entityVertex)); + RequestContext context = RequestContext.get(); + boolean includeClassifications = context.includeClassifications(); + boolean includeClassificationNames = context.isIncludeClassificationNames(); + if(includeClassifications){ + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } else if (!includeClassifications && includeClassificationNames) { + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } + ret.setIsIncomplete(isIncomplete); + ret.setLabels(getLabels(entityVertex)); + + ret.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); + ret.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); + ret.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); + ret.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); + + if(RequestContext.get().includeMeanings()) { + List termAssignmentHeaders = mapAssignedTerms(entityVertex); + ret.setMeanings(termAssignmentHeaders); + ret.setMeaningNames( + termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) + .collect(Collectors.toList())); + } + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + + if (entityType != null) { + for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { + Object attrValue = getVertexAttribute(entityVertex, headerAttribute); + + if (attrValue != null) { + ret.setAttribute(headerAttribute.getName(), attrValue); + } + } + + Object displayText = getDisplayText(entityVertex, entityType); + + if (displayText != null) { + ret.setDisplayText(displayText.toString()); + } + + if (CollectionUtils.isNotEmpty(attributes)) { + for (String attrName : attributes) { + AtlasAttribute attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attrName = toNonQualifiedName(attrName); + + if (ret.hasAttribute(attrName)) { + continue; + } + + attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attribute = entityType.getRelationshipAttribute(attrName, null); + } + } + + Object attrValue = getVertexAttribute(entityVertex, attribute); + + if (attrValue != null) { + ret.setAttribute(attrName, attrValue); + } + } + } + } + } + finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + return ret; + } + + private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeaderWithPrefetch"); + AtlasEntityHeader ret = new AtlasEntityHeader(); try { //pre-fetching the properties Map properties = preloadProperties(entityVertex); diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index 7101868e30..3040a19401 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -59,6 +59,11 @@ public final class AccessControlUtils { public static final String ATTR_PERSONA_USERS = "personaUsers"; public static final String ATTR_PERSONA_GROUPS = "personaGroups"; + public static final String ATTR_SERVICE_SERVICE_TYPE = "authServiceType"; + public static final String ATTR_SERVICE_TAG_SERVICE = "tagService"; + public static final String ATTR_SERVICE_IS_ENABLED = "authServiceIsEnabled"; + public static final String ATTR_SERVICE_LAST_SYNC = "authServicePolicyLastSync"; + public static final String ATTR_PURPOSE_CLASSIFICATIONS = "purposeClassifications"; public static final String ATTR_POLICY_TYPE = "policyType"; diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 45e63641c5..c4a150a112 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -657,6 +657,13 @@ public void endMetricRecord(MetricRecorder recorder) { } } + public void endMetricRecordWithInvocations(MetricRecorder recorder, long invocationCount) { + if (metrics != null && recorder != null) { + metrics.recordMetricWithInvocations(recorder, invocationCount); + } + } + + public void recordEntityGuidUpdate(AtlasEntity entity, String guidInRequest) { recordEntityGuidUpdate(new EntityGuidPair(entity, guidInRequest)); } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index ba94ff263c..dd737dffe1 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -401,6 +401,8 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.indexSearch(" + parameters + ")"); } + RequestContext.get().endMetricRecordWithInvocations(RequestContext.get().startMetricRecord("dslSize"), parameters.getQuerySize()); + RequestContext.get().endMetricRecordWithInvocations(RequestContext.get().startMetricRecord("attributeSize"), parameters.getAttributes().size()); if (AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_API_LIMIT.getBoolean() && parameters.getQuerySize() > AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong()) { if(CollectionUtils.isEmpty(parameters.getUtmTags())) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong())); From 65b35cf2e66ccc8c1100a0ad03e8c1dd170467df Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 29 Nov 2024 15:11:06 +0530 Subject: [PATCH 108/241] Revert to ealier way to fetching changed policies Policy object in audit log was querying all policies one by one --- .../atlas/plugin/model/RangerPolicyDelta.java | 4 +-- .../plugin/service/RangerBasePlugin.java | 2 +- .../atlas/plugin/util/PolicyRefresher.java | 3 ++ .../CachePolicyTransformerImpl.java | 34 ++++++++----------- .../org/apache/atlas/web/rest/AuthREST.java | 13 +++---- 5 files changed, 24 insertions(+), 32 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index def6c6c0dd..b5f9b58233 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -19,8 +19,8 @@ package org.apache.atlas.plugin.model; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonIgnore; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index fb44584e44..76ee3c3e1f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -314,7 +314,7 @@ public void setPolicies(ServicePolicies policies) { Boolean hasPolicyDeltas = RangerPolicyDeltaUtil.hasPolicyDeltas(policies); if (hasPolicyDeltas == null) { - LOG.info("Downloaded policies do not require policy change !! [" + policies + "]"); + LOG.info("Downloaded policies do not require policy change !! [" + (policies.getPolicies() != null ? policies.getPolicies().size() : 0) + "]"); if (this.policyEngine == null) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b9331a547c..5b806fd08e 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -324,7 +324,9 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading all policies for first time"); RangerRESTUtils restUtils = new RangerRESTUtils(); CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); @@ -332,6 +334,7 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); } else { + LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTiemInMillis); svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 3edd214e3b..c0a8894cd5 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -22,10 +22,8 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.audit.AuditSearchParams; import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; -import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -42,7 +40,6 @@ import org.apache.atlas.plugin.model.RangerServiceDef; import org.apache.atlas.plugin.model.RangerValiditySchedule; import org.apache.atlas.plugin.util.ServicePolicies.TagPolicies; -import org.apache.atlas.repository.audit.ESBasedAuditRepository; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -97,12 +94,13 @@ public class CachePolicyTransformerImpl { private static final String ATTR_POLICY_GROUPS = "policyGroups"; private static final String ATTR_POLICY_USERS = "policyUsers"; private static final String ATTR_POLICY_ROLES = "policyRoles"; - private static final String ATTR_POLICY_VALIDITY = "policyValiditySchedule"; - private static final String ATTR_POLICY_CONDITIONS = "policyConditions"; - private static final String ATTR_POLICY_MASK_TYPE = "policyMaskType"; + public static final String ATTR_POLICY_VALIDITY = "policyValiditySchedule"; + public static final String ATTR_POLICY_CONDITIONS = "policyConditions"; + public static final String ATTR_POLICY_MASK_TYPE = "policyMaskType"; private static final String RESOURCE_SERVICE_DEF_PATH = "/service-defs/"; private static final String RESOURCE_SERVICE_DEF_PATTERN = RESOURCE_SERVICE_DEF_PATH + "atlas-servicedef-%s.json"; + public static final int POLICY_BATCH_SIZE = 250; private EntityDiscoveryService discoveryService; private AtlasGraph graph; @@ -140,7 +138,7 @@ public AtlasEntityHeader getService() { return service; } - public ServicePolicies createPoliciesWithDelta(String serviceName, Map policyChanges, List atlasPolicies) { + public ServicePolicies getPoliciesDelta(String serviceName, Map policyChanges) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); ServicePolicies servicePolicies = new ServicePolicies(); @@ -159,7 +157,7 @@ public ServicePolicies createPoliciesWithDelta(String serviceName, Map policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasPolicies); + List policiesDelta = getRangerPolicyDelta(service, policyChanges); servicePolicies.setPolicyDeltas(policiesDelta); @@ -215,7 +213,7 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long servicePolicies.setPolicyUpdateTime(new Date()); if (service != null) { - List allPolicies = getServicePolicies(service, 250); + List allPolicies = getServicePolicies(service, POLICY_BATCH_SIZE); servicePolicies.setServiceName(serviceName); servicePolicies.setServiceId(service.getGuid()); @@ -282,7 +280,7 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getRangerPolicyDelta." + serviceName); @@ -292,6 +290,9 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } + ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); + List atlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); + List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -308,7 +309,7 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } - public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List events) { + public Map createPolicyChangeMap(String serviceName, List events) { Map policyChanges = new HashMap<>(); for (EntityAuditEventV2 event : events) { if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policyChanges.containsKey(event.getEntityId())) { @@ -316,16 +317,9 @@ public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List atlasPolicies = new ArrayList<>(); - for (EntityAuditEventV2 event : events) { - AtlasEntityHeader policy = event.getEntityHeader(); - if (policy != null) { - atlasPolicies.add(policy); - } - } - LOG.info("PolicyDelta: {}: Found {} policy changes with {} policies", serviceName, policyChanges.size(), atlasPolicies.size()); + LOG.info("PolicyDelta: {}: Found {} policy changes in {} policies", serviceName, events.size(), policyChanges.size()); - return createPoliciesWithDelta(serviceName, policyChanges, atlasPolicies); + return policyChanges; } private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 14ed3b1101..e4ac2552bd 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -154,7 +154,8 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); - ret = policyTransformer.extractAndTransformPolicyDelta(serviceName, auditEvents); + Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); + ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges); } else { if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; @@ -174,8 +175,6 @@ private List getPolicyAuditLogs(String serviceName, long las List entityUpdateToWatch = new ArrayList<>(); entityUpdateToWatch.add(POLICY_ENTITY_TYPE); - entityUpdateToWatch.add(PERSONA_ENTITY_TYPE); - entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); AuditSearchParams parameters = new AuditSearchParams(); Map dsl = getMap("size", 100); @@ -187,16 +186,12 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - parameters.setDsl(dsl); - parameters.setAttributes(new HashSet<>(Arrays.asList( - "qualifiedName", - "name" - ))); List events = new ArrayList<>(); try { - EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + String query = parameters.getQueryString(); + EntityAuditSearchResult result = auditRepository.searchEvents(query); // attributes are not getting passed in query if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { events = result.getEntityAudits(); } From 0befe573eea428d377dcb66836745a2681a8a3d4 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 29 Nov 2024 16:11:57 +0530 Subject: [PATCH 109/241] Added repair method for Task ES Docs mismatch --- .../janus/AtlasElasticsearchQuery.java | 41 +++++++++++++++ .../org/apache/atlas/tasks/TaskRegistry.java | 52 +++++++++++++++++-- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index dd182e3040..93b34ea4e4 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -174,6 +174,19 @@ private Map runQueryWithLowLevelClient(String query) throws Atla } } + private Map runUpdateByQueryWithLowLevelClient(String query) throws AtlasBaseException { + try { + String responseString = performDirectUpdateByQuery(query); + + Map responseMap = AtlasType.fromJson(responseString, Map.class); + return responseMap; + + } catch (IOException e) { + LOG.error("Failed to execute direct query on ES {}", e.getMessage()); + throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED, e.getMessage()); + } + } + private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchParams) throws AtlasBaseException, IOException { AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("performAsyncDirectIndexQuery"); DirectIndexQueryResult result = null; @@ -444,6 +457,30 @@ private String performDirectIndexQuery(String query, boolean source) throws Atla return EntityUtils.toString(response.getEntity()); } + private String performDirectUpdateByQuery(String query) throws AtlasBaseException, IOException { + HttpEntity entity = new NStringEntity(query, ContentType.APPLICATION_JSON); + String endPoint; + + endPoint = index + "/_update_by_query"; + + Request request = new Request("POST", endPoint); + request.setEntity(entity); + + Response response; + try { + response = lowLevelRestClient.performRequest(request); + } catch (ResponseException rex) { + if (rex.getResponse().getStatusLine().getStatusCode() == 404) { + LOG.warn(String.format("ES index with name %s not found", index)); + throw new AtlasBaseException(INDEX_NOT_FOUND, index); + } else { + throw new AtlasBaseException(String.format("Error in executing elastic query: %s", EntityUtils.toString(entity)), rex); + } + } + + return EntityUtils.toString(response.getEntity()); + } + private DirectIndexQueryResult getResultFromResponse(String responseString, boolean async) throws IOException { Map responseMap = AtlasType.fromJson(responseString, Map.class); return getResultFromResponse(responseMap.get("response")); @@ -495,6 +532,10 @@ public Map directIndexQuery(String query) throws AtlasBaseExcept return runQueryWithLowLevelClient(query); } + public Map directUpdateByQuery(String query) throws AtlasBaseException { + return runUpdateByQueryWithLowLevelClient(query); + } + @Override public Iterator> vertices() { SearchRequest searchRequest = getSearchRequest(index, sourceBuilder); diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index bffa4c8672..5f0285bb36 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.tasks; +import com.datastax.oss.driver.shaded.fasterxml.jackson.databind.ObjectMapper; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.GraphTransaction; @@ -31,6 +32,7 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; +import org.apache.atlas.repository.graphdb.janus.AtlasElasticsearchQuery; import org.apache.atlas.type.AtlasType; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -49,6 +51,7 @@ import java.util.List; import java.util.Arrays; import java.util.Map; +import java.util.LinkedHashMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -61,6 +64,7 @@ public class TaskRegistry { private static final Logger LOG = LoggerFactory.getLogger(TaskRegistry.class); public static final int TASK_FETCH_BATCH_SIZE = 100; public static final List> SORT_ARRAY = Collections.singletonList(mapOf(Constants.TASK_CREATED_TIME, mapOf("order", "asc"))); + public static final String JANUSGRAPH_VERTEX_INDEX = "janusgraph_vertex_index"; private AtlasGraph graph; private TaskService taskService; @@ -430,7 +434,7 @@ public List getTasksForReQueueIndexSearch() { } else { LOG.warn(String.format("There is a mismatch on tasks status between ES and Cassandra for guid: %s", atlasTask.getGuid())); - vertex.setProperty(Constants.TASK_STATUS, atlasTask.getStatus().toString()); + repairMismatchedTask(atlasTask); } } else { LOG.warn("Null vertex while re-queuing tasks at index {}", fetched); @@ -447,14 +451,56 @@ public List getTasksForReQueueIndexSearch() { } } catch (Exception e){ break; - } finally { - graph.commit(); } } return ret; } + private void repairMismatchedTask(AtlasTask atlasTask) { + AtlasElasticsearchQuery indexQuery = null; + + try { + // Create a map for the fields to be updated + Map fieldsToUpdate = new HashMap<>(); + fieldsToUpdate.put("__task_endTime", atlasTask.getEndTime().getTime()); + fieldsToUpdate.put("__task_timeTakenInSeconds", atlasTask.getTimeTakenInSeconds()); + fieldsToUpdate.put("__task_status", atlasTask.getStatus().toString()); + fieldsToUpdate.put("__task_modificationTimestamp", atlasTask.getUpdatedTime().getTime()); // Set current timestamp + + // Convert fieldsToUpdate map to JSON using Jackson + ObjectMapper objectMapper = new ObjectMapper(); + String fieldsToUpdateJson = objectMapper.writeValueAsString(fieldsToUpdate); + + // Construct the Elasticsearch update by query DSL + String queryDsl = "{" + + "\"script\": {" + + " \"source\": \"for (entry in params.fields.entrySet()) { ctx._source[entry.getKey()] = entry.getValue() }\"," + + " \"params\": {" + + " \"fields\": " + fieldsToUpdateJson + + " }" + + "}," + + "\"query\": {" + + " \"match\": {" + + " \"__task_guid\": \"" + atlasTask.getGuid() + "\"" + + " }" + + "}" + + "}"; + + // Execute the Elasticsearch query + indexQuery = (AtlasElasticsearchQuery) graph.elasticsearchQuery(JANUSGRAPH_VERTEX_INDEX); + Map result = indexQuery.directUpdateByQuery(queryDsl); + + if (result != null) { + LOG.info("Elasticsearch UpdateByQuery Result: " + result); + } else { + LOG.info("No documents updated in Elasticsearch for guid: " + atlasTask.getGuid()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + public void commit() { this.graph.commit(); } From 3b2e7d3600a55216f921055f5f207ccf119d9dcb Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 29 Nov 2024 16:13:02 +0530 Subject: [PATCH 110/241] Added branch for auto-image creation github action --- .github/workflows/maven.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..ee574c61aa 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - lineageondemand + - plt529 jobs: build: @@ -64,7 +64,7 @@ jobs: - name: Build with Maven run: | branch_name=${{ steps.get_branch.outputs.branch }} - if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]] + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'plt529' ]] then echo "build without dashboard" chmod +x ./build.sh && ./build.sh build_without_dashboard From 7878bafe663ed7d27f7a12658b569c5fd38a4f83 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 29 Nov 2024 16:19:34 +0530 Subject: [PATCH 111/241] Added branch for auto-image creation github action --- .github/workflows/maven.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index ee574c61aa..9a7174a37d 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - plt529 + - plt529test jobs: build: @@ -64,7 +64,7 @@ jobs: - name: Build with Maven run: | branch_name=${{ steps.get_branch.outputs.branch }} - if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'plt529' ]] + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'plt529test' ]] then echo "build without dashboard" chmod +x ./build.sh && ./build.sh build_without_dashboard From 9729887c9997b65f74b520b20ba5ccc7fe7cb3f0 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 29 Nov 2024 18:38:31 +0530 Subject: [PATCH 112/241] Added a doc targetting fix. instead of using guid using doc_id. --- .../apache/atlas/tasks/TaskQueueWatcher.java | 28 +++++++++++++++---- .../org/apache/atlas/tasks/TaskRegistry.java | 10 ++++--- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java index 435741a7ec..a1360938f6 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java @@ -78,43 +78,61 @@ public void shutdown() { public void run() { boolean isMaintenanceMode = AtlasConfiguration.ATLAS_MAINTENANCE_MODE.getBoolean(); if (isMaintenanceMode) { - LOG.info("TaskQueueWatcher: Maintenance mode is enabled, new tasks will not be loaded into the queue until next restart"); + LOG.info("TaskQueueWatcher: Maintenance mode is enabled. New tasks will not be loaded into the queue until next restart."); return; } shouldRun.set(true); if (LOG.isDebugEnabled()) { - LOG.debug("TaskQueueWatcher: running {}:{}", Thread.currentThread().getName(), Thread.currentThread().getId()); + LOG.debug("TaskQueueWatcher: Starting thread {}:{}", Thread.currentThread().getName(), Thread.currentThread().getId()); } while (shouldRun.get()) { + LOG.info("TaskQueueWatcher: Starting a new iteration of task fetching and processing."); TasksFetcher fetcher = new TasksFetcher(registry); try { + LOG.debug("TaskQueueWatcher: Attempting to acquire distributed lock: {}", ATLAS_TASK_LOCK); if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { + LOG.warn("TaskQueueWatcher: Could not acquire lock: {}. Retrying after {} ms.", ATLAS_TASK_LOCK, AtlasConstants.TASK_WAIT_TIME_MS); Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); continue; } + LOG.info("TaskQueueWatcher: Acquired distributed lock: {}", ATLAS_TASK_LOCK); + List tasks = fetcher.getTasks(); + LOG.info("TaskQueueWatcher: Fetched {} tasks for processing.", CollectionUtils.isNotEmpty(tasks) ? tasks.size() : 0); + if (CollectionUtils.isNotEmpty(tasks)) { final CountDownLatch latch = new CountDownLatch(tasks.size()); + LOG.info("TaskQueueWatcher: Submitting {} tasks to the queue.", tasks.size()); submitAll(tasks, latch); - LOG.info("Submitted {} tasks to the queue", tasks.size()); + + LOG.info("TaskQueueWatcher: Waiting for submitted tasks to complete."); waitForTasksToComplete(latch); + LOG.info("TaskQueueWatcher: All tasks have been processed."); } else { + LOG.info("TaskQueueWatcher: No tasks fetched. Releasing distributed lock: {}", ATLAS_TASK_LOCK); redisService.releaseDistributedLock(ATLAS_TASK_LOCK); } + + LOG.info("TaskQueueWatcher: Sleeping for {} ms before the next poll.", pollInterval); Thread.sleep(pollInterval); } catch (InterruptedException interruptedException) { - LOG.error("TaskQueueWatcher: Interrupted: thread is terminated, new tasks will not be loaded into the queue until next restart"); + LOG.error("TaskQueueWatcher: Interrupted. Thread is terminating. New tasks will not be loaded into the queue until next restart.", interruptedException); break; } catch (Exception e) { - LOG.error("TaskQueueWatcher: Exception occurred " + e.getMessage(), e); + LOG.error("TaskQueueWatcher: Exception occurred during task processing: {}", e.getMessage(), e); } finally { + LOG.info("TaskQueueWatcher: Releasing distributed lock: {}", ATLAS_TASK_LOCK); redisService.releaseDistributedLock(ATLAS_TASK_LOCK); fetcher.clearTasks(); + LOG.debug("TaskQueueWatcher: Cleared tasks from the fetcher."); } } + + LOG.info("TaskQueueWatcher: Thread has stopped. shouldRun is now set to false."); } + private void waitForTasksToComplete(final CountDownLatch latch) throws InterruptedException { if (latch.getCount() != 0) { LOG.info("TaskQueueWatcher: Waiting on Latch, current count: {}", latch.getCount()); diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index 5f0285bb36..a642e81605 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -37,6 +37,7 @@ import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections4.ListUtils; +import org.janusgraph.util.encoding.LongEncoding; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -434,7 +435,8 @@ public List getTasksForReQueueIndexSearch() { } else { LOG.warn(String.format("There is a mismatch on tasks status between ES and Cassandra for guid: %s", atlasTask.getGuid())); - repairMismatchedTask(atlasTask); + String docId = LongEncoding.encode(Long.parseLong(vertex.getIdForDisplay())); + repairMismatchedTask(atlasTask, docId); } } else { LOG.warn("Null vertex while re-queuing tasks at index {}", fetched); @@ -457,7 +459,7 @@ public List getTasksForReQueueIndexSearch() { return ret; } - private void repairMismatchedTask(AtlasTask atlasTask) { + private void repairMismatchedTask(AtlasTask atlasTask, String docId) { AtlasElasticsearchQuery indexQuery = null; try { @@ -481,8 +483,8 @@ private void repairMismatchedTask(AtlasTask atlasTask) { + " }" + "}," + "\"query\": {" - + " \"match\": {" - + " \"__task_guid\": \"" + atlasTask.getGuid() + "\"" + + " \"term\": {" + + " \"_id\": \"" + docId + "\"" + " }" + "}" + "}"; From 5ad0253e350c3b6e92f5055178878870c90f7b47 Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 29 Nov 2024 19:04:57 +0530 Subject: [PATCH 113/241] debug policy issues --- .../atlas/policytransformer/CachePolicyTransformerImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 4c6b9621fa..7c4f484b41 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -334,6 +334,8 @@ private void setPolicyItems(RangerPolicy rangerPolicy, AtlasEntityHeader atlasPo List users = (List) atlasPolicy.getAttribute("policyUsers"); List groups = (List) atlasPolicy.getAttribute("policyGroups"); + + LOG.info("PolicyRoles: {}", atlasPolicy.getAttribute("policyRoles")); List roles = (List) atlasPolicy.getAttribute("policyRoles"); List accesses = new ArrayList<>(); From 5920f1af6ca9ef920d0a79e001035d65aeb7889b Mon Sep 17 00:00:00 2001 From: hr2904 Date: Sat, 30 Nov 2024 02:45:53 +0530 Subject: [PATCH 114/241] Fixed logging --- .../java/org/apache/atlas/tasks/TaskQueueWatcher.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java index a1360938f6..f6165a639d 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java @@ -90,7 +90,7 @@ public void run() { LOG.info("TaskQueueWatcher: Starting a new iteration of task fetching and processing."); TasksFetcher fetcher = new TasksFetcher(registry); try { - LOG.debug("TaskQueueWatcher: Attempting to acquire distributed lock: {}", ATLAS_TASK_LOCK); + LOG.info("TaskQueueWatcher: Attempting to acquire distributed lock: {}", ATLAS_TASK_LOCK); if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { LOG.warn("TaskQueueWatcher: Could not acquire lock: {}. Retrying after {} ms.", ATLAS_TASK_LOCK, AtlasConstants.TASK_WAIT_TIME_MS); Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); @@ -117,15 +117,15 @@ public void run() { LOG.info("TaskQueueWatcher: Sleeping for {} ms before the next poll.", pollInterval); Thread.sleep(pollInterval); } catch (InterruptedException interruptedException) { - LOG.error("TaskQueueWatcher: Interrupted. Thread is terminating. New tasks will not be loaded into the queue until next restart.", interruptedException); + LOG.info("TaskQueueWatcher: Interrupted. Thread is terminating. New tasks will not be loaded into the queue until next restart.", interruptedException); break; } catch (Exception e) { - LOG.error("TaskQueueWatcher: Exception occurred during task processing: {}", e.getMessage(), e); + LOG.info("TaskQueueWatcher: Exception occurred during task processing: {}", e.getMessage(), e); } finally { LOG.info("TaskQueueWatcher: Releasing distributed lock: {}", ATLAS_TASK_LOCK); redisService.releaseDistributedLock(ATLAS_TASK_LOCK); fetcher.clearTasks(); - LOG.debug("TaskQueueWatcher: Cleared tasks from the fetcher."); + LOG.info("TaskQueueWatcher: Cleared tasks from the fetcher."); } } From 340fd369e1b04d64a5fc745d27ff1a1e75f101f0 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Sat, 30 Nov 2024 03:10:45 +0530 Subject: [PATCH 115/241] Fixed logging #2 --- .../org/apache/atlas/tasks/TaskRegistry.java | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index a642e81605..8ddda944e5 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -385,38 +385,47 @@ public List getTasksForReQueueIndexSearch() { DirectIndexQueryResult indexQueryResult = null; List ret = new ArrayList<>(); - int size = 1000; - int from = 0; + int size = 1000; // Batch size for fetching tasks + int from = 0; // Pagination offset + int totalFetched = 0; // Tracks the total number of tasks fetched + + LOG.info("Starting fetch of PENDING and IN_PROGRESS tasks. Queue size limit: {}", queueSize); IndexSearchParams indexSearchParams = new IndexSearchParams(); - List statusClauseList = new ArrayList(); + // Build query for PENDING and IN_PROGRESS tasks + List> statusClauseList = new ArrayList<>(); statusClauseList.add(mapOf("match", mapOf(TASK_STATUS, AtlasTask.Status.IN_PROGRESS.toString()))); statusClauseList.add(mapOf("match", mapOf(TASK_STATUS, AtlasTask.Status.PENDING.toString()))); - Map dsl = mapOf("query", mapOf("bool", mapOf("should", statusClauseList))); + Map dsl = mapOf( + "query", mapOf("bool", mapOf("should", statusClauseList)) + ); dsl.put("sort", Collections.singletonList(mapOf(Constants.TASK_CREATED_TIME, mapOf("order", "asc")))); dsl.put("size", size); - int totalFetched = 0; + while (true) { int fetched = 0; try { if (totalFetched + size > queueSize) { - size = queueSize - totalFetched; + size = queueSize - totalFetched; // Adjust size to not exceed queue size + LOG.info("Adjusting fetch size to {} based on queue size constraint.", size); } dsl.put("from", from); dsl.put("size", size); + LOG.debug("DSL Query for iteration: {}", dsl); indexSearchParams.setDsl(dsl); + LOG.info("Executing Elasticsearch query with from: {} and size: {}", from, size); AtlasIndexQuery indexQuery = graph.elasticsearchQuery(Constants.VERTEX_INDEX, indexSearchParams); try { indexQueryResult = indexQuery.vertices(indexSearchParams); + LOG.info("Query executed successfully for from: {} with size: {}", from, size); } catch (AtlasBaseException e) { - LOG.error("Failed to fetch pending/in-progress task vertices to re-que"); - e.printStackTrace(); + LOG.error("Failed to fetch PENDING/IN_PROGRESS task vertices. Exiting loop.", e); break; } @@ -428,34 +437,47 @@ public List getTasksForReQueueIndexSearch() { if (vertex != null) { AtlasTask atlasTask = toAtlasTask(vertex); + + LOG.debug("Processing fetched task: {}", atlasTask); if (atlasTask.getStatus().equals(AtlasTask.Status.PENDING) || - atlasTask.getStatus().equals(AtlasTask.Status.IN_PROGRESS) ){ - LOG.info(String.format("Fetched task from index search: %s", atlasTask.toString())); + atlasTask.getStatus().equals(AtlasTask.Status.IN_PROGRESS)) { + LOG.info("Adding task to the result list: {}", atlasTask); ret.add(atlasTask); - } - else { - LOG.warn(String.format("There is a mismatch on tasks status between ES and Cassandra for guid: %s", atlasTask.getGuid())); + } else { + LOG.warn("Status mismatch for task with guid: {}. Expected PENDING/IN_PROGRESS but found: {}", + atlasTask.getGuid(), atlasTask.getStatus()); + + // Repair mismatched task String docId = LongEncoding.encode(Long.parseLong(vertex.getIdForDisplay())); + LOG.info("Repairing mismatched task with docId: {}", docId); repairMismatchedTask(atlasTask, docId); } } else { - LOG.warn("Null vertex while re-queuing tasks at index {}", fetched); + LOG.warn("Null vertex encountered while re-queuing tasks at index {}", fetched); } fetched++; } + LOG.info("Fetched {} tasks in this iteration.", fetched); + } else { + LOG.warn("Index query result is null for from: {} and size: {}", from, size); } totalFetched += fetched; + LOG.info("Total tasks fetched so far: {}. Incrementing offset by {}.", totalFetched, size); + from += size; if (fetched < size || totalFetched >= queueSize) { + LOG.info("Breaking loop. Fetched fewer tasks ({}) than requested size ({}) or reached queue size limit ({}).", fetched, size, queueSize); break; } - } catch (Exception e){ + } catch (Exception e) { + LOG.error("Exception occurred during task fetching process. Exiting loop.", e); break; } } + LOG.info("Fetch process completed. Total tasks fetched: {}.", totalFetched); return ret; } From a2ebbb0ec95510fc0e1f31a147993859b5f6eb75 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 10:34:55 +0530 Subject: [PATCH 116/241] add logs --- .../atlas/policytransformer/CachePolicyTransformerImpl.java | 1 - .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 7c4f484b41..2dcbe62ab5 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -335,7 +335,6 @@ private void setPolicyItems(RangerPolicy rangerPolicy, AtlasEntityHeader atlasPo List users = (List) atlasPolicy.getAttribute("policyUsers"); List groups = (List) atlasPolicy.getAttribute("policyGroups"); - LOG.info("PolicyRoles: {}", atlasPolicy.getAttribute("policyRoles")); List roles = (List) atlasPolicy.getAttribute("policyRoles"); List accesses = new ArrayList<>(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index ac0dd6a140..9053a4f6bb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1871,6 +1871,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute if (AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean() && !attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE)) { AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); + LOG.info("Fetching non primitive attributes for entity: {}", attribute.getName()); Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); properties.put(attribute.getName(), mappedVertex); RequestContext.get().endMetricRecord(nonPrimitiveAttributes); From c88de8e631e070573a94665f0cf085e8a2a963ad Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 2 Dec 2024 10:51:56 +0530 Subject: [PATCH 117/241] Refatoring code for PR --- .../apache/atlas/tasks/TaskQueueWatcher.java | 26 ++--------- .../org/apache/atlas/tasks/TaskRegistry.java | 46 +++++-------------- 2 files changed, 17 insertions(+), 55 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java index f6165a639d..c212aad09e 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java @@ -78,61 +78,45 @@ public void shutdown() { public void run() { boolean isMaintenanceMode = AtlasConfiguration.ATLAS_MAINTENANCE_MODE.getBoolean(); if (isMaintenanceMode) { - LOG.info("TaskQueueWatcher: Maintenance mode is enabled. New tasks will not be loaded into the queue until next restart."); + LOG.info("TaskQueueWatcher: Maintenance mode is enabled, new tasks will not be loaded into the queue until next restart"); return; } shouldRun.set(true); if (LOG.isDebugEnabled()) { - LOG.debug("TaskQueueWatcher: Starting thread {}:{}", Thread.currentThread().getName(), Thread.currentThread().getId()); + LOG.debug("TaskQueueWatcher: running {}:{}", Thread.currentThread().getName(), Thread.currentThread().getId()); } while (shouldRun.get()) { - LOG.info("TaskQueueWatcher: Starting a new iteration of task fetching and processing."); TasksFetcher fetcher = new TasksFetcher(registry); try { - LOG.info("TaskQueueWatcher: Attempting to acquire distributed lock: {}", ATLAS_TASK_LOCK); if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { - LOG.warn("TaskQueueWatcher: Could not acquire lock: {}. Retrying after {} ms.", ATLAS_TASK_LOCK, AtlasConstants.TASK_WAIT_TIME_MS); Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); continue; } LOG.info("TaskQueueWatcher: Acquired distributed lock: {}", ATLAS_TASK_LOCK); List tasks = fetcher.getTasks(); - LOG.info("TaskQueueWatcher: Fetched {} tasks for processing.", CollectionUtils.isNotEmpty(tasks) ? tasks.size() : 0); - if (CollectionUtils.isNotEmpty(tasks)) { final CountDownLatch latch = new CountDownLatch(tasks.size()); - LOG.info("TaskQueueWatcher: Submitting {} tasks to the queue.", tasks.size()); submitAll(tasks, latch); - - LOG.info("TaskQueueWatcher: Waiting for submitted tasks to complete."); + LOG.info("Submitted {} tasks to the queue", tasks.size()); waitForTasksToComplete(latch); - LOG.info("TaskQueueWatcher: All tasks have been processed."); } else { - LOG.info("TaskQueueWatcher: No tasks fetched. Releasing distributed lock: {}", ATLAS_TASK_LOCK); redisService.releaseDistributedLock(ATLAS_TASK_LOCK); } - - LOG.info("TaskQueueWatcher: Sleeping for {} ms before the next poll.", pollInterval); Thread.sleep(pollInterval); } catch (InterruptedException interruptedException) { - LOG.info("TaskQueueWatcher: Interrupted. Thread is terminating. New tasks will not be loaded into the queue until next restart.", interruptedException); + LOG.error("TaskQueueWatcher: Interrupted: thread is terminated, new tasks will not be loaded into the queue until next restart"); break; } catch (Exception e) { - LOG.info("TaskQueueWatcher: Exception occurred during task processing: {}", e.getMessage(), e); + LOG.error("TaskQueueWatcher: Exception occurred " + e.getMessage(), e); } finally { - LOG.info("TaskQueueWatcher: Releasing distributed lock: {}", ATLAS_TASK_LOCK); redisService.releaseDistributedLock(ATLAS_TASK_LOCK); fetcher.clearTasks(); - LOG.info("TaskQueueWatcher: Cleared tasks from the fetcher."); } } - - LOG.info("TaskQueueWatcher: Thread has stopped. shouldRun is now set to false."); } - private void waitForTasksToComplete(final CountDownLatch latch) throws InterruptedException { if (latch.getCount() != 0) { LOG.info("TaskQueueWatcher: Waiting on Latch, current count: {}", latch.getCount()); diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index 8ddda944e5..d405a900c6 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -385,47 +385,38 @@ public List getTasksForReQueueIndexSearch() { DirectIndexQueryResult indexQueryResult = null; List ret = new ArrayList<>(); - int size = 1000; // Batch size for fetching tasks - int from = 0; // Pagination offset - int totalFetched = 0; // Tracks the total number of tasks fetched - - LOG.info("Starting fetch of PENDING and IN_PROGRESS tasks. Queue size limit: {}", queueSize); + int size = 1000; + int from = 0; IndexSearchParams indexSearchParams = new IndexSearchParams(); - // Build query for PENDING and IN_PROGRESS tasks - List> statusClauseList = new ArrayList<>(); + List statusClauseList = new ArrayList(); statusClauseList.add(mapOf("match", mapOf(TASK_STATUS, AtlasTask.Status.IN_PROGRESS.toString()))); statusClauseList.add(mapOf("match", mapOf(TASK_STATUS, AtlasTask.Status.PENDING.toString()))); - Map dsl = mapOf( - "query", mapOf("bool", mapOf("should", statusClauseList)) - ); + Map dsl = mapOf("query", mapOf("bool", mapOf("should", statusClauseList))); dsl.put("sort", Collections.singletonList(mapOf(Constants.TASK_CREATED_TIME, mapOf("order", "asc")))); dsl.put("size", size); - + int totalFetched = 0; while (true) { int fetched = 0; try { if (totalFetched + size > queueSize) { - size = queueSize - totalFetched; // Adjust size to not exceed queue size - LOG.info("Adjusting fetch size to {} based on queue size constraint.", size); + size = queueSize - totalFetched; } dsl.put("from", from); dsl.put("size", size); - LOG.debug("DSL Query for iteration: {}", dsl); indexSearchParams.setDsl(dsl); - LOG.info("Executing Elasticsearch query with from: {} and size: {}", from, size); AtlasIndexQuery indexQuery = graph.elasticsearchQuery(Constants.VERTEX_INDEX, indexSearchParams); try { indexQueryResult = indexQuery.vertices(indexSearchParams); - LOG.info("Query executed successfully for from: {} with size: {}", from, size); } catch (AtlasBaseException e) { - LOG.error("Failed to fetch PENDING/IN_PROGRESS task vertices. Exiting loop.", e); + LOG.error("Failed to fetch pending/in-progress task vertices to re-que"); + e.printStackTrace(); break; } @@ -437,47 +428,34 @@ public List getTasksForReQueueIndexSearch() { if (vertex != null) { AtlasTask atlasTask = toAtlasTask(vertex); - - LOG.debug("Processing fetched task: {}", atlasTask); if (atlasTask.getStatus().equals(AtlasTask.Status.PENDING) || - atlasTask.getStatus().equals(AtlasTask.Status.IN_PROGRESS)) { - LOG.info("Adding task to the result list: {}", atlasTask); + atlasTask.getStatus().equals(AtlasTask.Status.IN_PROGRESS) ){ + LOG.info(String.format("Fetched task from index search: %s", atlasTask.toString())); ret.add(atlasTask); } else { LOG.warn("Status mismatch for task with guid: {}. Expected PENDING/IN_PROGRESS but found: {}", atlasTask.getGuid(), atlasTask.getStatus()); - - // Repair mismatched task String docId = LongEncoding.encode(Long.parseLong(vertex.getIdForDisplay())); - LOG.info("Repairing mismatched task with docId: {}", docId); repairMismatchedTask(atlasTask, docId); } } else { - LOG.warn("Null vertex encountered while re-queuing tasks at index {}", fetched); + LOG.warn("Null vertex while re-queuing tasks at index {}", fetched); } fetched++; } - LOG.info("Fetched {} tasks in this iteration.", fetched); - } else { - LOG.warn("Index query result is null for from: {} and size: {}", from, size); } totalFetched += fetched; - LOG.info("Total tasks fetched so far: {}. Incrementing offset by {}.", totalFetched, size); - from += size; if (fetched < size || totalFetched >= queueSize) { - LOG.info("Breaking loop. Fetched fewer tasks ({}) than requested size ({}) or reached queue size limit ({}).", fetched, size, queueSize); break; } - } catch (Exception e) { - LOG.error("Exception occurred during task fetching process. Exiting loop.", e); + } catch (Exception e){ break; } } - LOG.info("Fetch process completed. Total tasks fetched: {}.", totalFetched); return ret; } From 0da6164de4ac8af03611c9108faa175c8b0937da Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 2 Dec 2024 10:58:29 +0530 Subject: [PATCH 118/241] Refatoring code for PR --- .github/workflows/maven.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 9a7174a37d..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - plt529test + - lineageondemand jobs: build: @@ -64,7 +64,7 @@ jobs: - name: Build with Maven run: | branch_name=${{ steps.get_branch.outputs.branch }} - if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'plt529test' ]] + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]] then echo "build without dashboard" chmod +x ./build.sh && ./build.sh build_without_dashboard From dd30b6ce3f6e89525b55c9fb9f3afa54f64f28f9 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 12:10:08 +0530 Subject: [PATCH 119/241] fix --- .github/workflows/maven.yml | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index ab8b455bbd..310e03b925 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,7 @@ on: - development - master - lineageondemand - - janusprefetchwith0.6 + - janusoptimisation jobs: build: @@ -35,12 +35,12 @@ jobs: steps: - uses: actions/checkout@v2 - + - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: java-version: 1.8 - + - name: Cache Maven packages uses: actions/cache@v2 with: @@ -49,8 +49,9 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: get_branch + run: | + echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + echo BRANCH_NAME=${GITHUB_REF#refs/heads/} - name: Create Maven Settings uses: s4u/maven-settings-action@v2.8.0 @@ -64,7 +65,7 @@ jobs: - name: Build with Maven run: | - branch_name=${{ steps.get_branch.outputs.branch }} + branch_name=${{ env.BRANCH_NAME }} if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]] then echo "build without dashboard" @@ -74,19 +75,26 @@ jobs: chmod +x ./build.sh && ./build.sh fi - - run: echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV + - name: Get Repository Name + run: echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" - id: get_version + # run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: | + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | head -n 1)abcd" + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" >> $GITHUB_ENV + + - name: Get commit ID + run: echo "COMMIT_ID=$(echo ${GITHUB_SHA} | cut -c1-7)abcd" >> $GITHUB_ENV - - name: Set up Buildx + - name: Set up Buildx id: buildx uses: docker/setup-buildx-action@v1 - name: Login to GitHub Registry - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: registry: ghcr.io username: $GITHUB_ACTOR @@ -103,8 +111,8 @@ jobs: provenance: true push: true tags: | - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:latest + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:${{ env.COMMIT_ID }} - name: Scan Image uses: aquasecurity/trivy-action@master @@ -117,4 +125,4 @@ jobs: - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v2.1.33 with: - sarif_file: 'trivy-image-results.sarif' + sarif_file: 'trivy-image-results.sarif' \ No newline at end of file From d3ae7c4d42bed47e675883906b14212519e630bf Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 12:12:29 +0530 Subject: [PATCH 120/241] add branch --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 310e03b925..284b9e81a5 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,6 +27,7 @@ on: - master - lineageondemand - janusoptimisation + - janusprefetchwith0.6 jobs: build: From 5047e77ce984ad3cbd634cbd5df92a79a8206e7f Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 2 Dec 2024 12:46:01 +0530 Subject: [PATCH 121/241] Added more logs. --- .../apache/atlas/tasks/TaskQueueWatcher.java | 22 +++++++++++-- .../org/apache/atlas/tasks/TaskRegistry.java | 31 ++++++++++++++----- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java index c212aad09e..c4ccdc358e 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java @@ -87,36 +87,52 @@ public void run() { LOG.debug("TaskQueueWatcher: running {}:{}", Thread.currentThread().getName(), Thread.currentThread().getId()); } while (shouldRun.get()) { + LOG.info("TaskQueueWatcher: Starting a new iteration of task fetching and processing."); TasksFetcher fetcher = new TasksFetcher(registry); try { + LOG.info("TaskQueueWatcher: Attempting to acquire distributed lock: {}", ATLAS_TASK_LOCK); if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { + LOG.warn("TaskQueueWatcher: Could not acquire lock: {}. Retrying after {} ms.", ATLAS_TASK_LOCK, AtlasConstants.TASK_WAIT_TIME_MS); Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); continue; } LOG.info("TaskQueueWatcher: Acquired distributed lock: {}", ATLAS_TASK_LOCK); List tasks = fetcher.getTasks(); + LOG.info("TaskQueueWatcher: Fetched {} tasks for processing.", CollectionUtils.isNotEmpty(tasks) ? tasks.size() : 0); + if (CollectionUtils.isNotEmpty(tasks)) { final CountDownLatch latch = new CountDownLatch(tasks.size()); + LOG.info("TaskQueueWatcher: Submitting {} tasks to the queue.", tasks.size()); submitAll(tasks, latch); - LOG.info("Submitted {} tasks to the queue", tasks.size()); + + LOG.info("TaskQueueWatcher: Waiting for submitted tasks to complete."); waitForTasksToComplete(latch); + LOG.info("TaskQueueWatcher: All tasks have been processed."); } else { + LOG.info("TaskQueueWatcher: No tasks fetched. Releasing distributed lock: {}", ATLAS_TASK_LOCK); redisService.releaseDistributedLock(ATLAS_TASK_LOCK); } + + LOG.info("TaskQueueWatcher: Sleeping for {} ms before the next poll.", pollInterval); Thread.sleep(pollInterval); } catch (InterruptedException interruptedException) { - LOG.error("TaskQueueWatcher: Interrupted: thread is terminated, new tasks will not be loaded into the queue until next restart"); + LOG.info("TaskQueueWatcher: Interrupted. Thread is terminating. New tasks will not be loaded into the queue until next restart.", interruptedException); break; } catch (Exception e) { - LOG.error("TaskQueueWatcher: Exception occurred " + e.getMessage(), e); + LOG.info("TaskQueueWatcher: Exception occurred during task processing: {}", e.getMessage(), e); } finally { + LOG.info("TaskQueueWatcher: Releasing distributed lock: {}", ATLAS_TASK_LOCK); redisService.releaseDistributedLock(ATLAS_TASK_LOCK); fetcher.clearTasks(); + LOG.info("TaskQueueWatcher: Cleared tasks from the fetcher."); } } + + LOG.info("TaskQueueWatcher: Thread has stopped. shouldRun is now set to false."); } + private void waitForTasksToComplete(final CountDownLatch latch) throws InterruptedException { if (latch.getCount() != 0) { LOG.info("TaskQueueWatcher: Waiting on Latch, current count: {}", latch.getCount()); diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index d405a900c6..014532c794 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -402,21 +402,24 @@ public List getTasksForReQueueIndexSearch() { int fetched = 0; try { if (totalFetched + size > queueSize) { - size = queueSize - totalFetched; + size = queueSize - totalFetched; // Adjust size to not exceed queue size + LOG.info("Adjusting fetch size to {} based on queue size constraint.", size); } dsl.put("from", from); dsl.put("size", size); + LOG.info("DSL Query for iteration: {}", dsl); indexSearchParams.setDsl(dsl); + LOG.info("Executing Elasticsearch query with from: {} and size: {}", from, size); AtlasIndexQuery indexQuery = graph.elasticsearchQuery(Constants.VERTEX_INDEX, indexSearchParams); try { indexQueryResult = indexQuery.vertices(indexSearchParams); + LOG.info("Query executed successfully for from: {} with size: {}", from, size); } catch (AtlasBaseException e) { - LOG.error("Failed to fetch pending/in-progress task vertices to re-que"); - e.printStackTrace(); + LOG.error("Failed to fetch PENDING/IN_PROGRESS task vertices. Exiting loop.", e); break; } @@ -428,34 +431,46 @@ public List getTasksForReQueueIndexSearch() { if (vertex != null) { AtlasTask atlasTask = toAtlasTask(vertex); + + LOG.info("Processing fetched task: {}", atlasTask); if (atlasTask.getStatus().equals(AtlasTask.Status.PENDING) || - atlasTask.getStatus().equals(AtlasTask.Status.IN_PROGRESS) ){ - LOG.info(String.format("Fetched task from index search: %s", atlasTask.toString())); + atlasTask.getStatus().equals(AtlasTask.Status.IN_PROGRESS)) { + LOG.info("Adding task to the result list: {}", atlasTask); ret.add(atlasTask); } else { LOG.warn("Status mismatch for task with guid: {}. Expected PENDING/IN_PROGRESS but found: {}", atlasTask.getGuid(), atlasTask.getStatus()); + // Repair mismatched task String docId = LongEncoding.encode(Long.parseLong(vertex.getIdForDisplay())); + LOG.info("Repairing mismatched task with docId: {}", docId); repairMismatchedTask(atlasTask, docId); } } else { - LOG.warn("Null vertex while re-queuing tasks at index {}", fetched); + LOG.warn("Null vertex encountered while re-queuing tasks at index {}", fetched); } fetched++; } + LOG.info("Fetched {} tasks in this iteration.", fetched); + } else { + LOG.warn("Index query result is null for from: {} and size: {}", from, size); } totalFetched += fetched; + LOG.info("Total tasks fetched so far: {}. Incrementing offset by {}.", totalFetched, size); + from += size; if (fetched < size || totalFetched >= queueSize) { + LOG.info("Breaking loop. Fetched fewer tasks ({}) than requested size ({}) or reached queue size limit ({}).", fetched, size, queueSize); break; } - } catch (Exception e){ + } catch (Exception e) { + LOG.error("Exception occurred during task fetching process. Exiting loop.", e); break; } } + LOG.info("Fetch process completed. Total tasks fetched: {}.", totalFetched); return ret; } @@ -494,7 +509,7 @@ private void repairMismatchedTask(AtlasTask atlasTask, String docId) { Map result = indexQuery.directUpdateByQuery(queryDsl); if (result != null) { - LOG.info("Elasticsearch UpdateByQuery Result: " + result); + LOG.info("Elasticsearch UpdateByQuery Result: " + result + "\nfor task : " + atlasTask.getGuid()); } else { LOG.info("No documents updated in Elasticsearch for guid: " + atlasTask.getGuid()); } From f036ad4947acf96353bcb19c8389aa5225108a2e Mon Sep 17 00:00:00 2001 From: abhijeet-atlan Date: Mon, 2 Dec 2024 14:20:14 +0530 Subject: [PATCH 122/241] Revert "DG-1924 : added params for phase1 tag propogation scaling" --- .../apache/atlas/repository/Constants.java | 2 - .../apache/atlas/model/tasks/AtlasTask.java | 27 ---- .../graph/GraphBackedSearchIndexer.java | 2 - .../store/graph/v2/EntityGraphMapper.java | 152 ------------------ .../org/apache/atlas/tasks/TaskRegistry.java | 4 - 5 files changed, 187 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 917bc71936..7460115fc8 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -363,8 +363,6 @@ public final class Constants { public static final String TASK_CLASSIFICATION_ID = encodePropertyKey(TASK_PREFIX + "classificationId"); public static final String TASK_ENTITY_GUID = encodePropertyKey(TASK_PREFIX + "entityGuid"); public static final String TASK_CLASSIFICATION_TYPENAME = encodePropertyKey(TASK_PREFIX + "classificationTypeName"); - public static final String TASK_ASSET_COUNT_TO_PROPAGATE = encodePropertyKey(TASK_PREFIX + "assetsCountToPropagate"); - public static final String TASK_ASSET_COUNT_PROPAGATED = encodePropertyKey(TASK_PREFIX + "assetsCountPropagated"); public static final String ACTIVE_STATE_VALUE = "ACTIVE"; public static final String TASK_HEADER_ATLAN_AGENT = "x-atlan-agent"; public static final String TASK_HEADER_ATLAN_AGENT_ID = "x-atlan-agent-id"; diff --git a/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java b/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java index 463da44df9..e4b87620d9 100644 --- a/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java +++ b/intg/src/main/java/org/apache/atlas/model/tasks/AtlasTask.java @@ -95,8 +95,6 @@ public static Status from(String s) { private String classificationId; private String entityGuid; private String classificationTypeName; - private Long assetsCountToPropagate; - private Long assetsCountPropagated; public AtlasTask() { } @@ -115,22 +113,6 @@ public AtlasTask(String type, String createdBy, Map parameters, this.entityGuid = entityGuid; } - public AtlasTask(String type, String createdBy, Map parameters, String classificationId, - String entityGuid, Long assetsCountToPropagate, Long assetsCountPropagated) { - this.guid = UUID.randomUUID().toString(); - this.type = type; - this.createdBy = createdBy; - this.createdTime = new Date(); - this.updatedTime = this.createdTime; - this.parameters = parameters; - this.status = Status.PENDING; - this.attemptCount = 0; - this.classificationId = classificationId; - this.entityGuid = entityGuid; - this.assetsCountToPropagate = assetsCountToPropagate; - this.assetsCountPropagated = assetsCountPropagated; - } - public String getGuid() { return guid; } @@ -257,13 +239,6 @@ public String getEntityGuid() { return entityGuid; } - public void setAssetsCountToPropagate(Long assetsCount) { - this.assetsCountToPropagate = assetsCount ; - } - public Long getAssetsCountToPropagate() { - return assetsCountToPropagate ; - } - @JsonIgnore public void start() { this.setStatus(Status.IN_PROGRESS); @@ -295,8 +270,6 @@ public String toString() { ", attemptCount=" + attemptCount + ", errorMessage='" + errorMessage + '\'' + ", status=" + status + - ", assetsCountToPropagate=" + assetsCountToPropagate + - ", assetsCountPropagated=" + assetsCountPropagated + '}'; } } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 7b2c3da7ac..a8bacbbb5f 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -407,8 +407,6 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep createCommonVertexIndex(management, TASK_ENTITY_GUID, UniqueKind.NONE, String.class, SINGLE, false, false, true); createCommonVertexIndex(management, TASK_ERROR_MESSAGE, UniqueKind.NONE, String.class, SINGLE, false, false); createCommonVertexIndex(management, TASK_ATTEMPT_COUNT, UniqueKind.NONE, Integer.class, SINGLE, false, false); - createCommonVertexIndex(management, TASK_ASSET_COUNT_TO_PROPAGATE, UniqueKind.NONE, Long.class, SINGLE, false, false); - createCommonVertexIndex(management, TASK_ASSET_COUNT_PROPAGATED, UniqueKind.NONE, Long.class, SINGLE, false, false); createCommonVertexIndex(management, TASK_UPDATED_TIME, UniqueKind.NONE, Long.class, SINGLE, false, false); createCommonVertexIndex(management, TASK_TIME_TAKEN_IN_SECONDS, UniqueKind.NONE, Long.class, SINGLE, false, false); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index dc7122a00a..76f0be44cb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3227,123 +3227,6 @@ public void cleanUpClassificationPropagation(String classificationName, int batc LOG.info("Completed cleaning up classification {}", classificationName); } - - -// Huge change to get affected edge count, to be reviewed hence keeping in comments for future review. -// public void cleanUpClassificationPropagation(String classificationName, int batchLimit) { -// int CLEANUP_MAX = batchLimit <= 0 ? CLEANUP_BATCH_SIZE : batchLimit * CLEANUP_BATCH_SIZE; -// int cleanedUpCount = 0; -// long classificationEdgeCount = 0; // To hold the total count -// long classificationEdgeInMemoryCount = 0; -// -// // Cache vertices and their classification edges during the initial iteration -// List allTagVertices = new ArrayList<>(); -// Map vertexEdgeCounts = new HashMap<>(); -// -// Iterator tagVerticesIterator = GraphHelper.getClassificationVertices(graph, classificationName, CLEANUP_BATCH_SIZE); -// -// while (tagVerticesIterator != null && tagVerticesIterator.hasNext()) { -// AtlasVertex tagVertex = tagVerticesIterator.next(); -// long edgeCount = GraphHelper.getClassificationEdges(tagVertex, null, classificationName).size(); -// classificationEdgeCount += edgeCount; -// -// // Cache the vertex and its edge count -// allTagVertices.add(tagVertex); -// vertexEdgeCounts.put(tagVertex, edgeCount); -// } -// -// LOG.info("Total classification edges to be cleaned for {}: {}", classificationName, classificationEdgeCount); -// -// // Now use the cached vertices for cleanup -// List tagVerticesProcessed = new ArrayList<>(); -// List currentAssetVerticesBatch = new ArrayList<>(); -// -// Iterator cachedVerticesIterator = allTagVertices.iterator(); -// while (cachedVerticesIterator.hasNext()) { -// if (cleanedUpCount >= CLEANUP_MAX) { -// return; -// } -// -// while (cachedVerticesIterator.hasNext() && currentAssetVerticesBatch.size() < CLEANUP_BATCH_SIZE) { -// AtlasVertex tagVertex = cachedVerticesIterator.next(); -// -// int availableSlots = CLEANUP_BATCH_SIZE - currentAssetVerticesBatch.size(); -// long assetCountForCurrentTagVertex = GraphHelper.getAssetsCountOfClassificationVertex(tagVertex); -// currentAssetVerticesBatch.addAll(GraphHelper.getAllAssetsWithClassificationVertex(tagVertex, availableSlots)); -// LOG.info("Available slots : {}, assetCountForCurrentTagVertex : {}, queueSize : {}", availableSlots, assetCountForCurrentTagVertex, currentAssetVerticesBatch.size()); -// if (assetCountForCurrentTagVertex <= availableSlots) { -// tagVerticesProcessed.add(tagVertex); -// } -// } -// -// int currentAssetsBatchSize = currentAssetVerticesBatch.size(); -// if (currentAssetsBatchSize > 0) { -// LOG.info("To clean up tag {} from {} entities", classificationName, currentAssetsBatchSize); -// int offset = 0; -// do { -// try { -// int toIndex = Math.min((offset + CHUNK_SIZE), currentAssetsBatchSize); -// List entityVertices = currentAssetVerticesBatch.subList(offset, toIndex); -// for (AtlasVertex vertex : entityVertices) { -// List deletedClassifications = new ArrayList<>(); -// GraphTransactionInterceptor.lockObjectAndReleasePostCommit(graphHelper.getGuid(vertex)); -// List classificationEdges = GraphHelper.getClassificationEdges(vertex, null, classificationName); -// int batchSize = CHUNK_SIZE; -// for (int i = 0; i < classificationEdges.size(); i += batchSize) { -// int end = Math.min(i + batchSize, classificationEdges.size()); -// List batch = classificationEdges.subList(i, end); -// for (AtlasEdge edge : batch) { -// try { -// AtlasClassification classification = entityRetriever.toAtlasClassification(edge.getInVertex()); -// deletedClassifications.add(classification); -// deleteDelegate.getHandler().deleteEdgeReference(edge, TypeCategory.CLASSIFICATION, false, true, null, vertex); -// classificationEdgeInMemoryCount++; -// } catch (IllegalStateException | AtlasBaseException e) { -// e.printStackTrace(); -// } -// } -// if (classificationEdgeInMemoryCount >= CHUNK_SIZE) { -// transactionInterceptHelper.intercept(); -// classificationEdgeInMemoryCount = 0; -// } -// } -// try { -// AtlasEntity entity = repairClassificationMappings(vertex); -// entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassifications); -// } catch (IllegalStateException | AtlasBaseException e) { -// e.printStackTrace(); -// } -// } -// -// transactionInterceptHelper.intercept(); -// -// offset += CHUNK_SIZE; -// } finally { -// LOG.info("For offset {} , cleaned up classification edges.", offset); -// LOG.info("Cleaned up {} entities for classification {}", offset, classificationName); -// } -// -// } while (offset < currentAssetsBatchSize); -// -// for (AtlasVertex classificationVertex : tagVerticesProcessed) { -// try { -// deleteDelegate.getHandler().deleteClassificationVertex(classificationVertex, true); -// } catch (IllegalStateException e) { -// e.printStackTrace(); -// } -// } -// transactionInterceptHelper.intercept(); -// -// cleanedUpCount += currentAssetsBatchSize; -// currentAssetVerticesBatch.clear(); -// tagVerticesProcessed.clear(); -// } -// } - -// LOG.info("Completed cleaning up classification {}", classificationName); -// } - - public AtlasEntity repairClassificationMappings(AtlasVertex entityVertex) throws AtlasBaseException { String guid = GraphHelper.getGuid(entityVertex); AtlasEntity entity = instanceConverter.getEntity(guid, ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES); @@ -3577,14 +3460,6 @@ public List propagateClassification(String entityGuid, String classifica Boolean toExclude = propagationMode == CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ? true:false; List impactedVertices = entityRetriever.getIncludedImpactedVerticesV2(entityVertex, relationshipGuid, classificationVertexId, edgeLabelsToCheck,toExclude); - // update the 'assetsCountToPropagate' on in memory java object. - AtlasTask currentTask = RequestContext.get().getCurrentTask(); - currentTask.setAssetsCountToPropagate((long) impactedVertices.size()); - - //update the 'assetsCountToPropagate' in the current task vertex. - AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); - currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, impactedVertices.size()); - if (CollectionUtils.isEmpty(impactedVertices)) { LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", entityGuid, classificationVertexId); @@ -4176,15 +4051,6 @@ public void updateClassificationTextPropagation(String classificationVertexId) t AtlasClassification classification = entityRetriever.toAtlasClassification(classificationVertex); LOG.info("Fetched classification : {} ", classification.toString()); List impactedVertices = graphHelper.getAllPropagatedEntityVertices(classificationVertex); - - // update the 'assetsCountToPropagate' on in memory java object. - AtlasTask currentTask = RequestContext.get().getCurrentTask(); - currentTask.setAssetsCountToPropagate((long) impactedVertices.size()); - - //update the 'assetsCountToPropagate' in the current task vertex. - AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); - currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, impactedVertices.size()); - LOG.info("impactedVertices : {}", impactedVertices.size()); int batchSize = 100; for (int i = 0; i < impactedVertices.size(); i += batchSize) { @@ -4230,14 +4096,6 @@ public List deleteClassificationPropagation(String entityGuid, String cl int propagatedEdgesSize = propagatedEdges.size(); - // update the 'assetsCountToPropagate' on in memory java object. - AtlasTask currentTask = RequestContext.get().getCurrentTask(); - currentTask.setAssetsCountToPropagate((long) propagatedEdgesSize); - - //update the 'assetsCountToPropagate' in the current task vertex. - AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); - currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, propagatedEdgesSize); - LOG.info(String.format("Number of edges to be deleted : %s for classification vertex with id : %s", propagatedEdgesSize, classificationVertexId)); List deletedPropagationsGuid = processClassificationEdgeDeletionInChunk(classification, propagatedEdges); @@ -4397,16 +4255,6 @@ public void classificationRefreshPropagation(String classificationId) throws Atl .filter(vertex -> vertex != null) .collect(Collectors.toList()); - Integer taskCount = verticesToRemove.size() + verticesToAddClassification.size(); - - // update the 'assetsCountToPropagate' on in memory java object. - AtlasTask currentTask = RequestContext.get().getCurrentTask(); - currentTask.setAssetsCountToPropagate((long) taskCount); - - //update the 'assetsCountToPropagate' in the current task vertex. - AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); - currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, taskCount); - //Remove classifications from unreachable vertices processPropagatedClassificationDeletionFromVertices(verticesToRemove, currentClassificationVertex, classification); diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index 5add393781..014532c794 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -624,10 +624,6 @@ public static AtlasTask toAtlasTask(AtlasVertex v) { ret.setErrorMessage(errorMessage); } - long countToPropagate = v.getProperty(Constants.TASK_ASSET_COUNT_TO_PROPAGATE, Long.class); - if (errorMessage != null) { - ret.setAssetsCountToPropagate(countToPropagate); - } return ret; } From 4fe9fd8b1a11776aa52bf14ca3405daeca4bb5b6 Mon Sep 17 00:00:00 2001 From: Abhijeet Kumar Date: Mon, 2 Dec 2024 14:24:17 +0530 Subject: [PATCH 123/241] custom image creation workflow --- .github/workflows/maven.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index afec7cf4e5..3af835f4fa 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - dg1908 + - revert3773dg1924 jobs: build: @@ -64,7 +65,7 @@ jobs: - name: Build with Maven run: | branch_name=${{ steps.get_branch.outputs.branch }} - if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'dg1908' ]] + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'revert3773dg1924' ]] then echo "build without dashboard" chmod +x ./build.sh && ./build.sh build_without_dashboard From 1ab776e08b3e2e9b0305db2bff69a245da634a95 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 2 Dec 2024 14:51:33 +0530 Subject: [PATCH 124/241] Use last policy updated at time as last update time --- .../atlas/plugin/model/RangerPolicy.java | 4 +- .../atlas/plugin/util/PolicyRefresher.java | 7 +-- .../atlas/plugin/util/ServicePolicies.java | 44 ++++++++++++++++--- .../CachePolicyTransformerImpl.java | 11 +++-- .../org/apache/atlas/web/rest/AuthREST.java | 3 +- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java index e1a6bda610..3a4e52fd3b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -19,8 +19,9 @@ package org.apache.atlas.plugin.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.collections.CollectionUtils; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -529,6 +530,7 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) { this.isDenyAllElse = isDenyAllElse == null ? Boolean.FALSE : isDenyAllElse; } + @JsonIgnore public String getAtlasGuid() { if (getGuid().length() > 36) { return getGuid().substring(0, 36); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 5b806fd08e..b74741cfad 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -22,11 +22,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; -import org.apache.atlas.repository.audit.ESBasedAuditRepository; -import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -281,7 +278,7 @@ private void loadPolicy() { serviceDefSetInPlugin = false; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; - lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; + lastUpdatedTiemInMillis = svcPolicies.getLatestUpdateTime() != null ? svcPolicies.getLatestUpdateTime().getTime() : -1L; } else { if (!policiesSetInPlugin && !serviceDefSetInPlugin) { plugIn.setPolicies(null); @@ -402,7 +399,7 @@ private ServicePolicies loadFromCache() { } lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue(); - lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); + lastUpdatedTiemInMillis = policies.getLatestUpdateTime() == null ? -1 : policies.getLatestUpdateTime().getTime(); } } catch (Exception excp) { LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 1a4dd81f29..2fb01d9e5f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -20,10 +20,11 @@ package org.apache.atlas.plugin.util; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.collections.MapUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import org.apache.atlas.plugin.model.RangerPolicy; import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.model.RangerServiceDef; @@ -33,12 +34,7 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @JsonInclude(JsonInclude.Include.NON_NULL) @XmlRootElement @@ -180,6 +176,40 @@ public String toString() { + "securityZones=" + securityZones ; } + + @JsonIgnore + public Date getLatestUpdateTime() { + Date lastestUpdateTime = new Date(0L); + + if (policies != null && !policies.isEmpty()) { + for (RangerPolicy policy : policies) { + if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { + lastestUpdateTime = policy.getUpdateTime(); + } + } + } + + if (tagPolicies != null && tagPolicies.getPolicies() != null) { + for (RangerPolicy policy : tagPolicies.getPolicies()) { + if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { + lastestUpdateTime = policy.getUpdateTime(); + } + } + } + + if (policyDeltas != null && !policyDeltas.isEmpty()) { + for (RangerPolicyDelta delta : policyDeltas) { + if (delta.getPolicy() != null && delta.getPolicy().getUpdateTime() != null && delta.getPolicy().getUpdateTime().after(lastestUpdateTime)) { + lastestUpdateTime = delta.getPolicy().getUpdateTime(); + } + } + } + if (Objects.equals(lastestUpdateTime, new Date(0L))) { + lastestUpdateTime = null; + } + return lastestUpdateTime; + } + public List getPolicyDeltas() { return this.policyDeltas; } public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index c0a8894cd5..8d4edad4e5 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -138,17 +138,18 @@ public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPoliciesDelta(String serviceName, Map policyChanges) { + public ServicePolicies getPoliciesDelta(String serviceName, Map policyChanges, long lastAuditEventTime) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); ServicePolicies servicePolicies = new ServicePolicies(); try { - servicePolicies.setServiceName(serviceName); - service = getServiceEntity(serviceName); + servicePolicies.setServiceName(serviceName); servicePolicies.setPolicyVersion(-1L); - servicePolicies.setPolicyUpdateTime(new Date()); + + Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); + servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { servicePolicies.setServiceName(serviceName); @@ -678,6 +679,8 @@ private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String servi policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); policy.setCreateTime(atlasPolicy.getCreateTime()); + policy.setUpdatedBy(atlasPolicy.getUpdatedBy()); + policy.setUpdateTime(atlasPolicy.getUpdateTime()); policy.setIsEnabled(getIsPolicyEnabled(atlasPolicy)); policy.setConditions(getPolicyConditions(atlasPolicy)); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index e4ac2552bd..3f552331a9 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -154,8 +154,9 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); + long lastEventTime = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); - ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges); + ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); } else { if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; From 8bfe37784b6b2a7c257ada140bb8a8648a491590 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 2 Dec 2024 15:50:07 +0530 Subject: [PATCH 125/241] Use current time as last update time for policy refresher In local it was creating problem as all policies are updated on startup and query fetches policies which has greater or equal to that timestamp which include all of them. --- .../java/org/apache/atlas/plugin/util/PolicyRefresher.java | 3 ++- .../atlas/policytransformer/CachePolicyTransformerImpl.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b74741cfad..b8d7daf4df 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -37,6 +37,7 @@ import java.io.FileWriter; import java.io.Reader; import java.io.Writer; +import java.util.Date; import java.util.Timer; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -278,7 +279,7 @@ private void loadPolicy() { serviceDefSetInPlugin = false; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; - lastUpdatedTiemInMillis = svcPolicies.getLatestUpdateTime() != null ? svcPolicies.getLatestUpdateTime().getTime() : -1L; + lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; } else { if (!policiesSetInPlugin && !serviceDefSetInPlugin) { plugIn.setPolicies(null); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 8d4edad4e5..f8eef3c8e0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -148,7 +148,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map 0 ? new Date(lastAuditEventTime) : new Date(); + // Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); + Date policyUpdateTime = new Date(); servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { From e93359b6c506d4c02ee55e351c98c62995ee090f Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 2 Dec 2024 16:36:27 +0530 Subject: [PATCH 126/241] PR refactor --- .../main/java/org/apache/atlas/tasks/TaskRegistry.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index d405a900c6..5544c035e9 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.tasks; +import com.datastax.oss.driver.shaded.fasterxml.jackson.core.JsonProcessingException; import com.datastax.oss.driver.shaded.fasterxml.jackson.databind.ObjectMapper; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.RequestContext; @@ -498,9 +499,12 @@ private void repairMismatchedTask(AtlasTask atlasTask, String docId) { } else { LOG.info("No documents updated in Elasticsearch for guid: " + atlasTask.getGuid()); } - } catch (Exception e) { - e.printStackTrace(); + } catch (JsonProcessingException e) { + LOG.error("Error converting fieldsToUpdate to JSON for task with guid: {} and docId: {}. Error: {}", atlasTask.getGuid(), docId, e.getMessage(), e); } + catch (AtlasBaseException e) { + LOG.error("Error executing Elasticsearch query for task with guid: {} and docId: {}. Error: {}", atlasTask.getGuid(), docId, e.getMessage(), e); + } } public void commit() { From 817fc5baa84bea8889b7888d8f55f07747099703 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 17:24:34 +0530 Subject: [PATCH 127/241] some optimizations --- .../store/graph/v2/EntityGraphRetriever.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 9053a4f6bb..72615037c5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1132,6 +1132,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithoutPrefetch(AtlasVerte } } + Object attrValue = getVertexAttribute(entityVertex, attribute); if (attrValue != null) { @@ -1865,13 +1866,21 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return null; } - if (properties.get(attribute.getName()) != null) { + LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); + + if (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || + attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ENUM)) { + return properties.get(attribute.getName()); + } + + if (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY) && + (attribute.getAttributeType()).getTypeCategory().getClass().getComponentType().toString().equals("class java.lang.String")) { return properties.get(attribute.getName()); } - if (AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean() && !attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE)) { + Set ENRICH_PROPERTY_TYPES = new HashSet<>(Arrays.asList(TypeCategory.STRUCT, TypeCategory.OBJECT_ID_TYPE, TypeCategory.MAP)); + if (ENRICH_PROPERTY_TYPES.contains(attribute.getAttributeType().getTypeCategory())) { AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); - LOG.info("Fetching non primitive attributes for entity: {}", attribute.getName()); Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); properties.put(attribute.getName(), mappedVertex); RequestContext.get().endMetricRecord(nonPrimitiveAttributes); From 53a05976d556af0d51c8d8b5ee6937d547e1c900 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 18:40:49 +0530 Subject: [PATCH 128/241] fix prop types --- .../store/graph/v2/EntityGraphRetriever.java | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 72615037c5..d6c7a4e6e3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1021,7 +1021,20 @@ private Map preloadProperties(AtlasVertex entityVertex) { VertexProperty property = traversal.next(); if (property.isPresent()) { // Ensure the property exists - propertiesMap.put(property.key(), property.value()); + if (propertiesMap.containsKey(property.key())) { + Object prevValue = propertiesMap.get(property.key()); + if (prevValue instanceof List) { + ((List) prevValue).add(property.value()); + } else { + List values = new ArrayList<>(); + values.add(prevValue); + values.add(property.value()); + propertiesMap.put(property.key(), values); + } + }else { + propertiesMap.put(property.key(), property.value()); + } + } } @@ -1869,17 +1882,14 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); if (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || - attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ENUM)) { - return properties.get(attribute.getName()); - } - - if (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY) && - (attribute.getAttributeType()).getTypeCategory().getClass().getComponentType().toString().equals("class java.lang.String")) { + attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ENUM)|| + attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY)) { return properties.get(attribute.getName()); } Set ENRICH_PROPERTY_TYPES = new HashSet<>(Arrays.asList(TypeCategory.STRUCT, TypeCategory.OBJECT_ID_TYPE, TypeCategory.MAP)); if (ENRICH_PROPERTY_TYPES.contains(attribute.getAttributeType().getTypeCategory())) { + LOG.info("capturing excluded property set category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); properties.put(attribute.getName(), mappedVertex); From 580e70242dc883e9d7d6a0e78fe28ce40131937a Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 19:59:59 +0530 Subject: [PATCH 129/241] fix null attributes --- .../store/graph/v2/EntityGraphRetriever.java | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index d6c7a4e6e3..a69c4807af 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1010,7 +1010,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } - private Map preloadProperties(AtlasVertex entityVertex) { + private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes) { Map propertiesMap = new HashMap<>(); // Execute the traversal to fetch properties @@ -1020,24 +1020,28 @@ private Map preloadProperties(AtlasVertex entityVertex) { while (traversal.hasNext()) { VertexProperty property = traversal.next(); - if (property.isPresent()) { // Ensure the property exists - if (propertiesMap.containsKey(property.key())) { - Object prevValue = propertiesMap.get(property.key()); - if (prevValue instanceof List) { - ((List) prevValue).add(property.value()); + AtlasAttribute attribute = entityType.getAttribute(property.key()); + TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; + TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + + if (propertiesMap.get(property.key()) == null) { + propertiesMap.put(property.key(), property.value()); + } else { + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { + Object value = propertiesMap.get(property.key()); + if (value instanceof List) { + ((List) value).add(property.value()); } else { List values = new ArrayList<>(); - values.add(prevValue); + values.add(value); values.add(property.value()); propertiesMap.put(property.key(), values); } - }else { + } else { propertiesMap.put(property.key(), property.value()); } - } } - return propertiesMap; } @@ -1166,9 +1170,10 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e AtlasEntityHeader ret = new AtlasEntityHeader(); try { //pre-fetching the properties - Map properties = preloadProperties(entityVertex); - String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); //properties.get returns null + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); // this is not costly + Map properties = preloadProperties(entityVertex, typeRegistry.getEntityTypeByName(GraphHelper.getTypeName(entityVertex)), attributes); + String guid = (String) properties.get(Constants.GUID_PROPERTY_KEY); Integer value = (Integer)properties.get(Constants.IS_INCOMPLETE_PROPERTY_KEY); @@ -1204,7 +1209,6 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) .collect(Collectors.toList())); } - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); // this is not costly if (entityType != null) { for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { @@ -1878,15 +1882,23 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute if (vertex == null || attribute == null) { return null; } - LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); - if (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || - attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ENUM)|| - attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY)) { + if ((properties.get(attribute.getName()) != null) && + (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || + attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ENUM) || + attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY) + )) { return properties.get(attribute.getName()); } + TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); + TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + + if (elementTypeCategory == TypeCategory.PRIMITIVE) { + return new ArrayList<>(); + } + Set ENRICH_PROPERTY_TYPES = new HashSet<>(Arrays.asList(TypeCategory.STRUCT, TypeCategory.OBJECT_ID_TYPE, TypeCategory.MAP)); if (ENRICH_PROPERTY_TYPES.contains(attribute.getAttributeType().getTypeCategory())) { LOG.info("capturing excluded property set category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); From 729e859b04a46e30f3178e4465a561261f377572 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 21:17:59 +0530 Subject: [PATCH 130/241] add log line --- .../repository/store/graph/v2/EntityGraphRetriever.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index a69c4807af..2154352c62 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1884,11 +1884,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); - if ((properties.get(attribute.getName()) != null) && - (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || - attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ENUM) || - attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY) - )) { + if (properties.get(attribute.getName()) != null) { return properties.get(attribute.getName()); } @@ -1896,6 +1892,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; if (elementTypeCategory == TypeCategory.PRIMITIVE) { + LOG.info("capturing null array attributes - {}: {} : {}", attribute.getName(), elementTypeCategory, properties.get(attribute.getName())); return new ArrayList<>(); } From 85692a9608450f26005d8a8c9ad56ac4f3ca36f6 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 22:03:24 +0530 Subject: [PATCH 131/241] explicitly set meanings --- .../store/graph/v2/EntityGraphRetriever.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 2154352c62..9512959a77 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1227,6 +1227,8 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e ret.setDisplayText(properties.get(QUALIFIED_NAME).toString()); } + + //attributes = only the attributes of entityType if (CollectionUtils.isNotEmpty(attributes)) { for (String attrName : attributes) { @@ -1254,9 +1256,10 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e } } } + ret.setAttribute("meanings", ret.getMeanings()); + ret.setAttribute("meaningNames", ret.getMeaningNames()); } - } - finally { + } finally { RequestContext.get().endMetricRecord(metricRecorder); } return ret; @@ -1884,6 +1887,10 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); + if (properties.get(attribute.getName()) == "meanings") { + LOG.info("meanings type: {}", ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory()); + } + if (properties.get(attribute.getName()) != null) { return properties.get(attribute.getName()); } From 7012726478b16ebca388964d9e5bd51a59407c82 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 22:22:20 +0530 Subject: [PATCH 132/241] fix --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 9512959a77..276659f6b8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1887,7 +1887,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); - if (properties.get(attribute.getName()) == "meanings") { + if (attribute.getName().equals( "meanings")){ LOG.info("meanings type: {}", ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory()); } From ea3a5bfdfb9fee48acbe741db869e4ba53d86bb3 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 22:27:09 +0530 Subject: [PATCH 133/241] add prop --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 276659f6b8..5c2e8428e3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1256,8 +1256,6 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e } } } - ret.setAttribute("meanings", ret.getMeanings()); - ret.setAttribute("meaningNames", ret.getMeaningNames()); } } finally { RequestContext.get().endMetricRecord(metricRecorder); @@ -1903,8 +1901,8 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return new ArrayList<>(); } - Set ENRICH_PROPERTY_TYPES = new HashSet<>(Arrays.asList(TypeCategory.STRUCT, TypeCategory.OBJECT_ID_TYPE, TypeCategory.MAP)); - if (ENRICH_PROPERTY_TYPES.contains(attribute.getAttributeType().getTypeCategory())) { + + if (AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { LOG.info("capturing excluded property set category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); From fcd01168649199bfef97df712b9ed9c8a41624e7 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 2 Dec 2024 23:25:10 +0530 Subject: [PATCH 134/241] add different types --- .../store/graph/v2/EntityGraphRetriever.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 5c2e8428e3..6a7be70343 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1885,22 +1885,21 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); - if (attribute.getName().equals( "meanings")){ - LOG.info("meanings type: {}", ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory()); - } - if (properties.get(attribute.getName()) != null) { + if (properties.get(attribute.getName()) != null || + attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE)) { return properties.get(attribute.getName()); } - TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); - TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - - if (elementTypeCategory == TypeCategory.PRIMITIVE) { - LOG.info("capturing null array attributes - {}: {} : {}", attribute.getName(), elementTypeCategory, properties.get(attribute.getName())); + if (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY)) { + LOG.info("capturing null array attributes - {} : {}", attribute.getName(), properties.get(attribute.getName())); return new ArrayList<>(); } + if (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.MAP)) { + LOG.info("capturing null map attributes - {}: {}", attribute.getName(), properties.get(attribute.getName())); + return new HashMap<>(); + } if (AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { LOG.info("capturing excluded property set category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); From 2ae76fd326835a46be388f8369a3da5fe7cfb5fe Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 00:11:58 +0530 Subject: [PATCH 135/241] improve performance --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 6a7be70343..966a06f419 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1901,7 +1901,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return new HashMap<>(); } - if (AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { + if (properties.get(attribute.getName()) != null && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { LOG.info("capturing excluded property set category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); From 067f5ef9d484681c45a6f6dc70f305bfe30ad0ee Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 01:00:57 +0530 Subject: [PATCH 136/241] add checks --- .../store/graph/v2/EntityGraphRetriever.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 966a06f419..b66d1f9bbb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1885,22 +1885,21 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); + TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); + TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ?((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - if (properties.get(attribute.getName()) != null || - attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE)) { + + if (properties.get(attribute.getName()) != null && + (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { + LOG.info("capturing non null attributes - {} : {} : {} ", attribute.getName(), properties.get(attribute.getName()), attribute.getAttributeType().getTypeCategory()); return properties.get(attribute.getName()); } - if (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY)) { + if (properties.get(attribute.getName()) == null && attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY)) { LOG.info("capturing null array attributes - {} : {}", attribute.getName(), properties.get(attribute.getName())); return new ArrayList<>(); } - if (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.MAP)) { - LOG.info("capturing null map attributes - {}: {}", attribute.getName(), properties.get(attribute.getName())); - return new HashMap<>(); - } - if (properties.get(attribute.getName()) != null && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { LOG.info("capturing excluded property set category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); From ddd5f186df763be70db7548ce71cd18f5132412e Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 09:43:34 +0530 Subject: [PATCH 137/241] add telemetry --- .../store/graph/v2/EntityGraphRetriever.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index b66d1f9bbb..4bbe25daa8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1011,6 +1011,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) } private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes) { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("preloadProperties"); Map propertiesMap = new HashMap<>(); // Execute the traversal to fetch properties @@ -1042,6 +1043,8 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt } } } + + RequestContext.get().endMetricRecord(metricRecorder); return propertiesMap; } @@ -1172,7 +1175,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e //pre-fetching the properties String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); //properties.get returns null AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); // this is not costly - Map properties = preloadProperties(entityVertex, typeRegistry.getEntityTypeByName(GraphHelper.getTypeName(entityVertex)), attributes); + Map properties = preloadProperties(entityVertex, entityType, attributes); String guid = (String) properties.get(Constants.GUID_PROPERTY_KEY); @@ -1880,6 +1883,7 @@ public Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) t } public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute attribute, Map properties) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getVertexAttributePreFetchCache"); if (vertex == null || attribute == null) { return null; } @@ -1892,11 +1896,13 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute if (properties.get(attribute.getName()) != null && (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { LOG.info("capturing non null attributes - {} : {} : {} ", attribute.getName(), properties.get(attribute.getName()), attribute.getAttributeType().getTypeCategory()); + RequestContext.get().endMetricRecord(metricRecorder); return properties.get(attribute.getName()); } if (properties.get(attribute.getName()) == null && attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY)) { LOG.info("capturing null array attributes - {} : {}", attribute.getName(), properties.get(attribute.getName())); + RequestContext.get().endMetricRecord(metricRecorder); return new ArrayList<>(); } @@ -1906,9 +1912,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); properties.put(attribute.getName(), mappedVertex); RequestContext.get().endMetricRecord(nonPrimitiveAttributes); + RequestContext.get().endMetricRecord(metricRecorder); return mappedVertex; } + RequestContext.get().endMetricRecord(metricRecorder); return null; } From 2d026ba22950cd7891fc8d0593d7dd0f5b9c29e7 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 10:17:19 +0530 Subject: [PATCH 138/241] remove if --- .../repository/store/graph/v2/EntityGraphRetriever.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 4bbe25daa8..c2cc6daadd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1889,12 +1889,13 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); - TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); - TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ?((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; +// TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); +// TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ?((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - if (properties.get(attribute.getName()) != null && - (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { + if (properties.get(attribute.getName()) != null ) { +// && +// (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { LOG.info("capturing non null attributes - {} : {} : {} ", attribute.getName(), properties.get(attribute.getName()), attribute.getAttributeType().getTypeCategory()); RequestContext.get().endMetricRecord(metricRecorder); return properties.get(attribute.getName()); From b75a2b73c65694ce15df863dc3bab143b795f339 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 10:45:19 +0530 Subject: [PATCH 139/241] remove telemeytry --- .../store/graph/v2/EntityGraphRetriever.java | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index c2cc6daadd..976f25d690 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1883,27 +1883,24 @@ public Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) t } public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute attribute, Map properties) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getVertexAttributePreFetchCache"); + // AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getVertexAttributePreFetchCache"); if (vertex == null || attribute == null) { return null; } - LOG.info("capturing property its category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); -// TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); -// TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ?((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); + TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ?((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - if (properties.get(attribute.getName()) != null ) { -// && -// (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { - LOG.info("capturing non null attributes - {} : {} : {} ", attribute.getName(), properties.get(attribute.getName()), attribute.getAttributeType().getTypeCategory()); - RequestContext.get().endMetricRecord(metricRecorder); + + if (properties.get(attribute.getName()) != null && + (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { + //RequestContext.get().endMetricRecord(metricRecorder); return properties.get(attribute.getName()); } if (properties.get(attribute.getName()) == null && attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY)) { - LOG.info("capturing null array attributes - {} : {}", attribute.getName(), properties.get(attribute.getName())); - RequestContext.get().endMetricRecord(metricRecorder); + // RequestContext.get().endMetricRecord(metricRecorder); return new ArrayList<>(); } @@ -1913,11 +1910,10 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); properties.put(attribute.getName(), mappedVertex); RequestContext.get().endMetricRecord(nonPrimitiveAttributes); - RequestContext.get().endMetricRecord(metricRecorder); + // RequestContext.get().endMetricRecord(metricRecorder); return mappedVertex; } - RequestContext.get().endMetricRecord(metricRecorder); return null; } From a13c25f22cdf3ba964904d1025f2a79229857168 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 11:28:19 +0530 Subject: [PATCH 140/241] Remove too much telemetry --- .../org/apache/atlas/AtlasConfiguration.java | 10 ++++++++++ .../store/graph/v2/EntityGraphRetriever.java | 16 ++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index d97d4cd4a8..69b1a5c5a5 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -118,7 +118,17 @@ public enum AtlasConfiguration { ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), + + /*** + * This configuration is used to enable fetching non primitive attributes in index search + */ ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", false), + + /** + * decides which flow of indexsearch to be used + * if no of attributes requested is less than configured invoke old flow + * else invoke new flow + */ ATLAS_INDEXSEARCH_ATTRIBUTES_MIN_LIMIT("atlas.indexsearch.attributes.min.limit", 8), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 976f25d690..9879ab0c38 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1010,8 +1010,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } - private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes) { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("preloadProperties"); + private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType) { Map propertiesMap = new HashMap<>(); // Execute the traversal to fetch properties @@ -1044,7 +1043,6 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt } } - RequestContext.get().endMetricRecord(metricRecorder); return propertiesMap; } @@ -1175,7 +1173,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e //pre-fetching the properties String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); //properties.get returns null AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); // this is not costly - Map properties = preloadProperties(entityVertex, entityType, attributes); + Map properties = preloadProperties(entityVertex, entityType); String guid = (String) properties.get(Constants.GUID_PROPERTY_KEY); @@ -1883,7 +1881,6 @@ public Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) t } public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute attribute, Map properties) throws AtlasBaseException { - // AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getVertexAttributePreFetchCache"); if (vertex == null || attribute == null) { return null; } @@ -1892,25 +1889,24 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ?((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - + // if element is primitive or array of primitives, return the value from properties if (properties.get(attribute.getName()) != null && (attribute.getAttributeType().getTypeCategory().equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { - //RequestContext.get().endMetricRecord(metricRecorder); return properties.get(attribute.getName()); } + // if array is empty && element is array of primitives, return the value from properties if (properties.get(attribute.getName()) == null && attribute.getAttributeType().getTypeCategory().equals(TypeCategory.ARRAY)) { - // RequestContext.get().endMetricRecord(metricRecorder); return new ArrayList<>(); } + // if element is non-primitive, fetch the value from the vertex if (properties.get(attribute.getName()) != null && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { - LOG.info("capturing excluded property set category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); + LOG.debug("capturing excluded property set category and value - {}: {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName())); AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); properties.put(attribute.getName(), mappedVertex); RequestContext.get().endMetricRecord(nonPrimitiveAttributes); - // RequestContext.get().endMetricRecord(metricRecorder); return mappedVertex; } From b48451c58c61f58887154b8908218e2f07fbd52c Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 13:13:42 +0530 Subject: [PATCH 141/241] revoke janus --- .../store/graph/v2/EntityGraphRetriever.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 9879ab0c38..539be25881 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1070,15 +1070,15 @@ private boolean isPolicyAttribute(Set attributes) { } private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { - boolean shouldPrefetch = attributes.size() > AtlasConfiguration.ATLAS_INDEXSEARCH_ATTRIBUTES_MIN_LIMIT.getInt() - && !isPolicyAttribute(attributes) - && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); - - if (shouldPrefetch) { - return mapVertexToAtlasEntityHeaderWithPrefetch(entityVertex, attributes); - } else { +// boolean shouldPrefetch = attributes.size() > AtlasConfiguration.ATLAS_INDEXSEARCH_ATTRIBUTES_MIN_LIMIT.getInt() +// && !isPolicyAttribute(attributes) +// && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); +// +// if (shouldPrefetch) { +// return mapVertexToAtlasEntityHeaderWithPrefetch(entityVertex, attributes); +// } else { return mapVertexToAtlasEntityHeaderWithoutPrefetch(entityVertex, attributes); - } + //} } private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithoutPrefetch(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { From 62c251bf1a91408327222d228516ed52db6c2ceb Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 15:36:19 +0530 Subject: [PATCH 142/241] fix preload properties --- .../store/graph/v2/EntityGraphRetriever.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 9879ab0c38..5c5834bcee 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1024,23 +1024,21 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - if (propertiesMap.get(property.key()) == null) { - propertiesMap.put(property.key(), property.value()); - } else { + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { Object value = propertiesMap.get(property.key()); if (value instanceof List) { ((List) value).add(property.value()); } else { List values = new ArrayList<>(); - values.add(value); values.add(property.value()); propertiesMap.put(property.key(), values); } } else { - propertiesMap.put(property.key(), property.value()); + if (propertiesMap.get(property.key()) == null) { + propertiesMap.put(property.key(), property.value()); + } } - } } return propertiesMap; From 4116a4e172ecd0d3b931a3e1be798f38d90f2ec1 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 15:45:35 +0530 Subject: [PATCH 143/241] remove flag --- .../src/main/java/org/apache/atlas/AtlasConfiguration.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 69b1a5c5a5..70771e1a7e 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -123,13 +123,6 @@ public enum AtlasConfiguration { * This configuration is used to enable fetching non primitive attributes in index search */ ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", false), - - /** - * decides which flow of indexsearch to be used - * if no of attributes requested is less than configured invoke old flow - * else invoke new flow - */ - ATLAS_INDEXSEARCH_ATTRIBUTES_MIN_LIMIT("atlas.indexsearch.attributes.min.limit", 8), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), ATLAS_UD_RELATIONSHIPS_MAX_COUNT("atlas.ud.relationship.max.count", 100); From 3ba45f6f8c8b7feb1ed93cdae5f77280e084c3f3 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 15:46:12 +0530 Subject: [PATCH 144/241] fix condition --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 5c5834bcee..edde78ab33 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1068,8 +1068,7 @@ private boolean isPolicyAttribute(Set attributes) { } private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { - boolean shouldPrefetch = attributes.size() > AtlasConfiguration.ATLAS_INDEXSEARCH_ATTRIBUTES_MIN_LIMIT.getInt() - && !isPolicyAttribute(attributes) + boolean shouldPrefetch = !isPolicyAttribute(attributes) && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); if (shouldPrefetch) { From b8cc3c2d2c2ffd4a1886498aedcbc0b03723d177 Mon Sep 17 00:00:00 2001 From: Pavan Manish Date: Tue, 3 Dec 2024 15:53:22 +0530 Subject: [PATCH 145/241] FT-800: Handle circular references in exception chain in formatErrorMessage method --- .../apache/atlas/web/errors/ExceptionMapperUtil.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java index 50b09f11ee..d8e45cea1e 100644 --- a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java +++ b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java @@ -35,12 +35,18 @@ protected static String formatErrorMessage(long id, Exception exception) { .append(" \"message\": \"There was an error processing your request.\",\n") .append(" \"causes\": [\n"); - // Traverse through the chain of causes + // Traverse through the chain of causes, avoiding cycles List causes = new ArrayList<>(); + List visited = new ArrayList<>(); Throwable currentException = exception; while (currentException != null) { + if (visited.contains(currentException)) { + causes.add(" {\n \"errorType\": \"CircularReferenceDetected\",\n \"errorMessage\": \"A circular reference was detected in the exception chain.\",\n \"location\": \"Unavailable\"\n }"); + break; + } + visited.add(currentException); causes.add(formatCause(currentException)); - currentException =currentException .getCause(); + currentException = currentException.getCause(); } // Add all formatted causes to the response From 079cf4133ce65b3aa23358b4e40fdd0f289c054b Mon Sep 17 00:00:00 2001 From: Pavan Manish Date: Tue, 3 Dec 2024 16:04:02 +0530 Subject: [PATCH 146/241] FT-800: Handle null exceptions gracefully in formatErrorMessage method --- .../org/apache/atlas/web/errors/ExceptionMapperUtil.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java index d8e45cea1e..03df167d5d 100644 --- a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java +++ b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java @@ -27,6 +27,15 @@ public class ExceptionMapperUtil { @SuppressWarnings("UnusedParameters") protected static String formatErrorMessage(long id, Exception exception) { + if (exception == null) { + // If the exception is null, return a minimal error message + return "{\n" + + String.format(" \"errorId\": \"%016x\",\n", id) + + " \"message\": \"No exception provided\",\n" + + " \"causes\": []\n" + + "}"; + } + StringBuilder response = new StringBuilder(); // Add error ID and general error message From 5635dbc9fc2ff4ceeed09b4130409e59e2fd1421 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 3 Dec 2024 16:19:51 +0530 Subject: [PATCH 147/241] add isPresent --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index edde78ab33..a1eb7cd350 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1024,7 +1024,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - + if (property.isPresent()) { if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { Object value = propertiesMap.get(property.key()); if (value instanceof List) { @@ -1039,6 +1039,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt propertiesMap.put(property.key(), property.value()); } } + } } return propertiesMap; From 1fba289d96b6bef730a83b66e272041eb846f798 Mon Sep 17 00:00:00 2001 From: Pavan Manish Date: Tue, 3 Dec 2024 16:43:05 +0530 Subject: [PATCH 148/241] FT-800: Use AtlasType.toJson() for JSON serialization --- .../atlas/web/errors/ExceptionMapperUtil.java | 69 ++++++++----------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java index 03df167d5d..f10029266a 100644 --- a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java +++ b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java @@ -17,10 +17,13 @@ */ package org.apache.atlas.web.errors; +import org.apache.atlas.type.AtlasType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.HashMap; public class ExceptionMapperUtil { protected static final Logger LOGGER = LoggerFactory.getLogger(ExceptionMapperUtil.class); @@ -29,28 +32,31 @@ public class ExceptionMapperUtil { protected static String formatErrorMessage(long id, Exception exception) { if (exception == null) { // If the exception is null, return a minimal error message - return "{\n" - + String.format(" \"errorId\": \"%016x\",\n", id) - + " \"message\": \"No exception provided\",\n" - + " \"causes\": []\n" - + "}"; + Map errorDetails = new HashMap<>(); + errorDetails.put("errorId", String.format("%016x", id)); + errorDetails.put("message", "No exception provided."); + errorDetails.put("causes", new ArrayList<>()); + return AtlasType.toJson(errorDetails); } - StringBuilder response = new StringBuilder(); + // Prepare data for error message + Map errorDetails = new HashMap<>(); + errorDetails.put("errorId", String.format("%016x", id)); + errorDetails.put("message", "There was an error processing your request."); - // Add error ID and general error message - response.append("{\n") - .append(String.format(" \"errorId\": \"%016x\",\n", id)) - .append(" \"message\": \"There was an error processing your request.\",\n") - .append(" \"causes\": [\n"); - - // Traverse through the chain of causes, avoiding cycles - List causes = new ArrayList<>(); + // Create a list of causes + List> causes = new ArrayList<>(); List visited = new ArrayList<>(); Throwable currentException = exception; + while (currentException != null) { if (visited.contains(currentException)) { - causes.add(" {\n \"errorType\": \"CircularReferenceDetected\",\n \"errorMessage\": \"A circular reference was detected in the exception chain.\",\n \"location\": \"Unavailable\"\n }"); + // If circular reference detected, add special entry + Map circularCause = new HashMap<>(); + circularCause.put("errorType", "CircularReferenceDetected"); + circularCause.put("errorMessage", "A circular reference was detected in the exception chain."); + circularCause.put("location", "Unavailable"); + causes.add(circularCause); break; } visited.add(currentException); @@ -58,24 +64,14 @@ protected static String formatErrorMessage(long id, Exception exception) { currentException = currentException.getCause(); } - // Add all formatted causes to the response - for (int i = 0; i < causes.size(); i++) { - response.append(causes.get(i)); - if (i < causes.size() - 1) { - response.append(",\n"); - } - } - - // Close the JSON structure - response.append("\n ]\n") - .append("}"); + errorDetails.put("causes", causes); - return response.toString(); + return AtlasType.toJson(errorDetails); } // Helper method to format a single exception cause - private static String formatCause(Throwable exception) { - StringBuilder cause = new StringBuilder(); + private static Map formatCause(Throwable exception) { + Map cause = new HashMap<>(); // Extract location details from the first stack trace element StackTraceElement[] stackTrace = exception.getStackTrace(); @@ -89,18 +85,14 @@ private static String formatCause(Throwable exception) { element.getLineNumber()); } - // Build JSON object for this cause - cause.append(" {\n") - .append(" \"errorType\": \"").append(exception.getClass().getName()).append("\",\n") - .append(" \"errorMessage\": \"").append(exception.getMessage() != null ? exception.getMessage() : "No additional information provided").append("\",\n") - .append(" \"location\": \"").append(location).append("\"\n") - .append(" }"); + // Populate the cause map + cause.put("errorType", exception.getClass().getName()); + cause.put("errorMessage", exception.getMessage() != null ? exception.getMessage() : "No additional information provided"); + cause.put("location", location); - return cause.toString(); + return cause; } - - protected static void logException(long id, Exception exception) { LOGGER.error(formatLogMessage(id, exception), exception); } @@ -109,5 +101,4 @@ protected static void logException(long id, Exception exception) { protected static String formatLogMessage(long id, Throwable exception) { return String.format("Error handling a request: %016x", id); } - } From cf76260c78575f14fa3e90a978567ff7a7316b58 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 3 Dec 2024 17:11:55 +0530 Subject: [PATCH 149/241] Use pagination while fetching audit logs --- .../CachePolicyTransformerImpl.java | 12 +++++---- .../org/apache/atlas/web/rest/AuthREST.java | 26 ++++++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index f8eef3c8e0..7439fb9aec 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -169,7 +169,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map tagRangerPolicies = getServicePolicies(tagService, 0); +// List tagRangerPolicies = getServicePolicies(tagService, 0); TagPolicies tagPolicies = new TagPolicies(); @@ -180,15 +180,14 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map getAtlasPolicies(String serviceName, int batchSi Map dsl = getMap("size", 0); List> mustClauseList = new ArrayList<>(); - mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); if (!policyGuids.isEmpty()) { mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); + } else { + // no service filter required if guids are provided + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); } dsl.put("query", getMap("bool", getMap("must", mustClauseList))); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 3f552331a9..25e144344c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -163,6 +163,10 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s } ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); } + LOG.info("downloadPolicies: serviceName={}, lastUpdatedTime={}, policies fetched={} delta fetched={}", serviceName, + lastUpdatedTime > 0 ? new Date(lastUpdatedTime) : lastUpdatedTime, + ret != null && ret.getPolicies() != null ? ret.getPolicies().size() : 0, + ret != null && ret.getPolicyDeltas() != null ? ret.getPolicyDeltas().size() : 0); updateLastSync(serviceName); return ret; @@ -184,18 +188,26 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - parameters.setDsl(dsl); + + int from = 0; + int size = 100; List events = new ArrayList<>(); try { - String query = parameters.getQueryString(); - EntityAuditSearchResult result = auditRepository.searchEvents(query); // attributes are not getting passed in query - if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { - events = result.getEntityAudits(); - } + do { + dsl.put("from", from); + dsl.put("size", size); + parameters.setDsl(dsl); + String query = parameters.getQueryString(); + EntityAuditSearchResult result = auditRepository.searchEvents(query); // attributes are not getting passed in query + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + from += size; + } while (events.size() == size); } catch (AtlasBaseException e) { LOG.error("ERROR in getPolicyAuditLogs while fetching entity audits {}: ", e.getMessage()); } finally { From 863d6c0a5dbc2abe16118ba1c0e623fe7511fde3 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:22:31 +0530 Subject: [PATCH 150/241] fix: emmit audits in case of policy update by accesscontrol resource update --- .../graph/v2/preprocessor/AuthPolicyPreProcessor.java | 7 +++++++ .../v2/preprocessor/accesscontrol/PersonaPreProcessor.java | 1 + .../v2/preprocessor/accesscontrol/PurposePreProcessor.java | 2 ++ 3 files changed, 10 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index 58fb516564..c87d1bc6e1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -139,6 +139,9 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { policy.setAttribute(ATTR_POLICY_USERS, new ArrayList<>()); policy.setAttribute(ATTR_POLICY_GROUPS, new ArrayList<>()); + if(parentEntity != null) { + policy.setAttribute(ATTR_POLICY_IS_ENABLED, getIsAccessControlEnabled(parentEntity)); + } //create ES alias aliasStore.updateAlias(parent, policy); @@ -155,6 +158,10 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { policy.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + if(parentEntity != null) { + policy.setAttribute(ATTR_POLICY_IS_ENABLED, getIsAccessControlEnabled(parentEntity)); + } + //create ES alias aliasStore.updateAlias(parent, policy); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index 8a2c8b1945..e6f815e36a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -205,6 +205,7 @@ private void updatePoliciesIsEnabledAttr(EntityMutationContext context, AtlasEnt policyToBeUpdated.setAttribute(ATTR_POLICY_IS_ENABLED, enable); context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + RequestContext.get().cacheDifferentialEntity(policyToBeUpdated); } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java index ca8fcb803f..3dbf342feb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java @@ -167,6 +167,7 @@ private void processUpdatePurpose(EntityMutationContext context, policyToBeUpdated.setAttribute(ATTR_POLICY_RESOURCES, newTagsResources); context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + RequestContext.get().cacheDifferentialEntity(policyToBeUpdated); existingPurposeExtInfo.addReferredEntity(policyToBeUpdated); } @@ -199,6 +200,7 @@ private void updatePoliciesIsEnabledAttr(EntityMutationContext context, AtlasEnt policyToBeUpdated.setAttribute(ATTR_POLICY_IS_ENABLED, enable); context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + RequestContext.get().cacheDifferentialEntity(policyToBeUpdated); } } } From b698f1397822c635044e7f6e809f5b637b9ef50c Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 3 Dec 2024 17:59:38 +0530 Subject: [PATCH 151/241] Use policy udpate time as last refresh time --- .../policytransformer/CachePolicyTransformerImpl.java | 4 ++-- .../src/main/java/org/apache/atlas/web/rest/AuthREST.java | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 7439fb9aec..5623120dc0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -148,8 +148,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map 0 ? new Date(lastAuditEventTime) : new Date(); - Date policyUpdateTime = new Date(); + Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); +// Date policyUpdateTime = new Date(); servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 25e144344c..8d8ebaf036 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -154,7 +154,8 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); - long lastEventTime = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); + long lastEventTime = auditEvents.isEmpty() ? 0 : auditEvents.get(auditEvents.size() - 1).getCreated(); + long lastEventTime2 = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); } else { @@ -192,6 +193,10 @@ private List getPolicyAuditLogs(String serviceName, long las dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + List> sortClause = new ArrayList<>(); + sortClause.add(getMap("created", getMap("order", "asc"))); + dsl.put("sort", sortClause); + int from = 0; int size = 100; From 22759b2304181aa6d9c4fd164fd75b56b70136ca Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Tue, 3 Dec 2024 18:26:22 +0530 Subject: [PATCH 152/241] surrond preload properties with try/cath. show actual exception in logs --- .../store/graph/v2/EntityGraphRetriever.java | 41 +++++++++++-------- .../atlas/web/errors/ExceptionMapperUtil.java | 2 +- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index a1eb7cd350..4bbb8d7c4f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1018,27 +1018,32 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt // Iterate through the resulting VertexProperty objects while (traversal.hasNext()) { - VertexProperty property = traversal.next(); - - AtlasAttribute attribute = entityType.getAttribute(property.key()); - TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; - TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - - if (property.isPresent()) { - if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { - Object value = propertiesMap.get(property.key()); - if (value instanceof List) { - ((List) value).add(property.value()); + try { + VertexProperty property = traversal.next(); + + AtlasAttribute attribute = entityType.getAttribute(property.key()); + TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; + TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + + if (property.isPresent()) { + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { + Object value = propertiesMap.get(property.key()); + if (value instanceof List) { + ((List) value).add(property.value()); + } else { + List values = new ArrayList<>(); + values.add(property.value()); + propertiesMap.put(property.key(), values); + } } else { - List values = new ArrayList<>(); - values.add(property.value()); - propertiesMap.put(property.key(), values); - } - } else { - if (propertiesMap.get(property.key()) == null) { - propertiesMap.put(property.key(), property.value()); + if (propertiesMap.get(property.key()) == null) { + propertiesMap.put(property.key(), property.value()); + } } } + } catch (RuntimeException e) { + LOG.error("Error preloading properties for entity vertex: {}", entityVertex, e); + throw e; // Re-throw the exception after logging it } } diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java index e2511e5d50..debae45902 100644 --- a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java +++ b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java @@ -34,7 +34,7 @@ protected static void logException(long id, Exception exception) { @SuppressWarnings("UnusedParameters") protected static String formatLogMessage(long id, Throwable exception) { - return String.format("Error handling a request: %016x", id); + return String.format("Error handling a request: %016x , exception is: %s", id, exception.getMessage()); } } From 9e1c9e3bdc6f66a0f63f88d4caffcd793949c89d Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Tue, 3 Dec 2024 19:36:58 +0530 Subject: [PATCH 153/241] print stacktrace --- .../org/apache/atlas/web/errors/ExceptionMapperUtil.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java index debae45902..20adce06e7 100644 --- a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java +++ b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java @@ -34,7 +34,11 @@ protected static void logException(long id, Exception exception) { @SuppressWarnings("UnusedParameters") protected static String formatLogMessage(long id, Throwable exception) { - return String.format("Error handling a request: %016x , exception is: %s", id, exception.getMessage()); + StringBuilder stackTrace = new StringBuilder(); + for (StackTraceElement element : exception.getStackTrace()) { + stackTrace.append(element.toString()).append("\n"); + } + return String.format("Error handling a request: %016x , exception is: %s\nStack trace:\n%s", id, exception.getMessage(), stackTrace.toString()); } } From c7ad8ab5410327d15caaba6c46a36de881485f89 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Tue, 3 Dec 2024 20:16:14 +0530 Subject: [PATCH 154/241] print stacktrace --- .../src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index dd737dffe1..94a6e7fb67 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -442,6 +442,7 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, } throw abe; } catch (Exception e) { + e.printStackTrace(); AtlasBaseException abe = new AtlasBaseException(e.getMessage(), e.getCause()); if (enableSearchLogging && parameters.isSaveSearchLog()) { logSearchLog(parameters, servletRequest, abe, System.currentTimeMillis() - startTime); From a87bbfc74cb2abdf96f6eaafcdec87c269596782 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Tue, 3 Dec 2024 23:43:07 +0530 Subject: [PATCH 155/241] fix the NPE --- .../org/apache/atlas/web/errors/ExceptionMapperUtil.java | 6 +----- .../main/java/org/apache/atlas/web/rest/DiscoveryREST.java | 5 ++--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java index 20adce06e7..debae45902 100644 --- a/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java +++ b/webapp/src/main/java/org/apache/atlas/web/errors/ExceptionMapperUtil.java @@ -34,11 +34,7 @@ protected static void logException(long id, Exception exception) { @SuppressWarnings("UnusedParameters") protected static String formatLogMessage(long id, Throwable exception) { - StringBuilder stackTrace = new StringBuilder(); - for (StackTraceElement element : exception.getStackTrace()) { - stackTrace.append(element.toString()).append("\n"); - } - return String.format("Error handling a request: %016x , exception is: %s\nStack trace:\n%s", id, exception.getMessage(), stackTrace.toString()); + return String.format("Error handling a request: %016x , exception is: %s", id, exception.getMessage()); } } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 94a6e7fb67..2d075a5e02 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -402,7 +402,7 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, } RequestContext.get().endMetricRecordWithInvocations(RequestContext.get().startMetricRecord("dslSize"), parameters.getQuerySize()); - RequestContext.get().endMetricRecordWithInvocations(RequestContext.get().startMetricRecord("attributeSize"), parameters.getAttributes().size()); + RequestContext.get().endMetricRecordWithInvocations(RequestContext.get().startMetricRecord("attributeSize"), CollectionUtils.isNotEmpty(parameters.getAttributes()) ? parameters.getAttributes().size() : 0); if (AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_API_LIMIT.getBoolean() && parameters.getQuerySize() > AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong()) { if(CollectionUtils.isEmpty(parameters.getUtmTags())) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong())); @@ -442,8 +442,7 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, } throw abe; } catch (Exception e) { - e.printStackTrace(); - AtlasBaseException abe = new AtlasBaseException(e.getMessage(), e.getCause()); + AtlasBaseException abe = new AtlasBaseException(e.getMessage(), e); if (enableSearchLogging && parameters.isSaveSearchLog()) { logSearchLog(parameters, servletRequest, abe, System.currentTimeMillis() - startTime); } From 343fef441b8692374d179670e35f1b6032ba809e Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 4 Dec 2024 16:56:20 +0530 Subject: [PATCH 156/241] Added perf loggers --- .../atlas/discovery/EntityLineageService.java | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index a873105a6d..1919e6df99 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -658,18 +658,23 @@ private static boolean isInputDirection(AtlasLineageListContext lineageListConte } private boolean checkForOffset(AtlasEdge atlasEdge, AtlasVertex entityVertex, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret) { - String entityGuid = getGuid(entityVertex); - LineageOnDemandConstraints entityConstraints = getAndValidateLineageConstraintsByGuid(entityGuid, atlasLineageOnDemandContext); - LineageInfoOnDemand entityLineageInfo = ret.getRelationsOnDemand().containsKey(entityGuid) ? ret.getRelationsOnDemand().get(entityGuid) : new LineageInfoOnDemand(entityConstraints); - - if (entityConstraints.getFrom() != 0 && entityLineageInfo.getFromCounter() < entityConstraints.getFrom()) { - if (! lineageContainsSkippedEdgeV2(ret, atlasEdge)) { - addEdgeToSkippedEdges(ret, atlasEdge); - entityLineageInfo.incrementFromCounter(); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("checkForOffset"); + try { + String entityGuid = getGuid(entityVertex); + LineageOnDemandConstraints entityConstraints = getAndValidateLineageConstraintsByGuid(entityGuid, atlasLineageOnDemandContext); + LineageInfoOnDemand entityLineageInfo = ret.getRelationsOnDemand().containsKey(entityGuid) ? ret.getRelationsOnDemand().get(entityGuid) : new LineageInfoOnDemand(entityConstraints); + + if (entityConstraints.getFrom() != 0 && entityLineageInfo.getFromCounter() < entityConstraints.getFrom()) { + if (! lineageContainsSkippedEdgeV2(ret, atlasEdge)) { + addEdgeToSkippedEdges(ret, atlasEdge); + entityLineageInfo.incrementFromCounter(); + } + return true; } - return true; + return false; + } finally { + RequestContext.get().endMetricRecord(metric); } - return false; } private boolean checkOffsetAndSkipEntity(AtlasLineageListContext atlasLineageListContext, AtlasLineageListInfo ret) { @@ -996,17 +1001,22 @@ private int getLineageMaxNodeAllowedCount() { } private String getEdgeLabel(AtlasEdge edge) { - AtlasVertex inVertex = edge.getInVertex(); - AtlasVertex outVertex = edge.getOutVertex(); - String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); - String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex); - String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(edge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); - boolean isInputEdge = edge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); - - if (isLineageOnDemandEnabled()) { - return getEdgeLabelFromGuids(isInputEdge, inGuid, outGuid); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getEdgeLabel"); + try { + AtlasVertex inVertex = edge.getInVertex(); + AtlasVertex outVertex = edge.getOutVertex(); + String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); + String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex); + String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(edge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class); + boolean isInputEdge = edge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); + + if (isLineageOnDemandEnabled()) { + return getEdgeLabelFromGuids(isInputEdge, inGuid, outGuid); + } + return relationGuid; + } finally { + RequestContext.get().endMetricRecord(metric); } - return relationGuid; } private String getEdgeLabelFromGuids(boolean isInputEdge, String inGuid, String outGuid) { @@ -1309,7 +1319,12 @@ private List getEdgesOfProcess(boolean isInput, AtlasLineageContext l } private boolean vertexMatchesEvaluation(AtlasVertex currentVertex, AtlasLineageContext lineageContext) { - return currentVertex.equals(lineageContext.getStartDatasetVertex()) || lineageContext.evaluate(currentVertex); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("vertexMatchesEvaluation"); + try { + return currentVertex.equals(lineageContext.getStartDatasetVertex()) || lineageContext.evaluate(currentVertex); + } finally { + RequestContext.get().endMetricRecord(metric); + } } private boolean vertexMatchesEvaluation(AtlasVertex currentVertex, AtlasLineageOnDemandContext atlasLineageOnDemandContext) { @@ -1384,9 +1399,11 @@ private boolean lineageContainsSkippedEdgeV2(AtlasLineageOnDemandInfo lineageInf } private void addEdgeToSkippedEdges(AtlasLineageOnDemandInfo lineageInfo, AtlasEdge edge) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("addEdgeToSkippedEdges"); if (lineageInfo.getSkippedEdges() != null) { lineageInfo.getSkippedEdges().add(getEdgeLabel(edge)); } + RequestContext.get().endMetricRecord(metric); } private AtlasLineageInfo initializeLineageInfo(String guid, LineageDirection direction, int depth, int limit, int offset) { From 557fece9a30ad4b2f3e797b4b87daec0c669e9d9 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Wed, 4 Dec 2024 17:09:55 +0530 Subject: [PATCH 157/241] Handle delete policy event --- .../atlas/plugin/util/PolicyRefresher.java | 2 +- .../plugin/util/RangerPolicyDeltaUtil.java | 9 +++-- .../CachePolicyTransformerImpl.java | 40 ++++++++++++------- .../org/apache/atlas/web/rest/AuthREST.java | 7 +++- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b8d7daf4df..5f9351be26 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -108,7 +108,7 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); - LOG.info("PolicyRefresher(serviceName=" + serviceName + ") - delta based policy refresh is enabled"); + LOG.info("PolicyRefresher(serviceName=" + serviceName + ") - delta based policy refresh enabled="+this.enableDeltaBasedRefresh); setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index e63345c10a..0c8267b5c7 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -91,7 +91,10 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List 1) { - LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray())); + if (CollectionUtils.isEmpty(deletedPolicies)) { + LOG.warn("Unexpected: found no policy for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray())); } break; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 5623120dc0..1edf210162 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -60,17 +60,7 @@ import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_PRIORITY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SUB_CATEGORY; -import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_DATAMESH; -import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PERSONA; -import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PURPOSE; -import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; -import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyCategory; +import static org.apache.atlas.repository.util.AccessControlUtils.*; @Component public class CachePolicyTransformerImpl { @@ -304,7 +294,29 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); policyDeltas.add(delta); } - LOG.info("PolicyDelta: {}: atlas policies found = {}, delta created = {}", serviceName, atlasPolicies.size(), policyDeltas.size()); + + // handle delete changes separately as they won't be present in atlas policies + List deletedPolicyDeltas = new ArrayList<>(); + for (String policyGuid : policyGuids) { + if (policyChanges.get(policyGuid) == EntityAuditActionV2.ENTITY_DELETE) { + RangerPolicy deletedPolicy = new RangerPolicy(); + deletedPolicy.setGuid(policyGuid); + deletedPolicy.setService(serviceName); + deletedPolicy.setServiceType(serviceType); + RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta( + deletedPolicy.getId(), + auditEventToDeltaChangeType.get(EntityAuditActionV2.ENTITY_DELETE), + deletedPolicy.getVersion(), + deletedPolicy + ); + deletedPolicyDeltas.add(deletedPolicyDelta); + } + } + + policyDeltas.addAll(deletedPolicyDeltas); + + LOG.info("PolicyDelta: {}: atlas policies found={}, delta created={}, including deleted policies={}", + serviceName, atlasPolicies.size(), policyDeltas.size(), deletedPolicyDeltas.size()); RequestContext.get().endMetricRecord(recorder); return policyDeltas; @@ -585,15 +597,13 @@ private List getAtlasPolicies(String serviceName, int batchSi Map dsl = getMap("size", 0); List> mustClauseList = new ArrayList<>(); - - mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); if (!policyGuids.isEmpty()) { mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); } else { - // no service filter required if guids are provided mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); } dsl.put("query", getMap("bool", getMap("must", mustClauseList))); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 8d8ebaf036..c668aa5c90 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -155,7 +155,10 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); long lastEventTime = auditEvents.isEmpty() ? 0 : auditEvents.get(auditEvents.size() - 1).getCreated(); - long lastEventTime2 = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); + LOG.info("PolicyDelta: serviceName={}, lastUpdatedTime={}, audit events found={}", serviceName, lastEventTime, auditEvents.size()); + if (auditEvents.isEmpty()) { + return null; + } Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); } else { @@ -189,7 +192,7 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("created", getMap("gt", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); From 795bbb1f67ef458f115950a5110090530edbdf58 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 4 Dec 2024 17:53:10 +0530 Subject: [PATCH 158/241] change default config --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 70771e1a7e..363dafaaee 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -122,7 +122,7 @@ public enum AtlasConfiguration { /*** * This configuration is used to enable fetching non primitive attributes in index search */ - ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", false), + ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", true), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), ATLAS_UD_RELATIONSHIPS_MAX_COUNT("atlas.ud.relationship.max.count", 100); From f04603bdd15099e8b5291690c6859d5bec04a3f8 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Wed, 4 Dec 2024 17:54:41 +0530 Subject: [PATCH 159/241] Added perf loggers for vertex + edge filters --- .../atlas/discovery/EntityLineageService.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 1919e6df99..a8841eb1aa 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -1328,11 +1328,21 @@ private boolean vertexMatchesEvaluation(AtlasVertex currentVertex, AtlasLineageC } private boolean vertexMatchesEvaluation(AtlasVertex currentVertex, AtlasLineageOnDemandContext atlasLineageOnDemandContext) { - return atlasLineageOnDemandContext.evaluate(currentVertex); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("vertexMatchesEvaluation"); + try { + return atlasLineageOnDemandContext.evaluate(currentVertex); + } finally { + RequestContext.get().endMetricRecord(metric); + } } private boolean edgeMatchesEvaluation(AtlasEdge currentEdge, AtlasLineageOnDemandContext atlasLineageOnDemandContext) { - return atlasLineageOnDemandContext.evaluate(currentEdge); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("edgeMatchesEvaluation"); + try { + return atlasLineageOnDemandContext.evaluate(currentEdge); + } finally { + RequestContext.get().endMetricRecord(metric); + } } private boolean shouldProcessEdge(AtlasLineageContext lineageContext, AtlasEdge edge) { From b84869817d95d8a7bb5b2574fe95a31ffe7244e2 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Wed, 21 Aug 2024 18:22:46 +0530 Subject: [PATCH 160/241] Merge pull request #3395 from atlanhq/ns/DG-1709-refresh-delta-policies DG-1709 | Delta based policies refresh --- .../atlas/plugin/model/RangerPolicyDelta.java | 4 + .../policyengine/RangerPolicyRepository.java | 34 +-- .../plugin/service/RangerBasePlugin.java | 1 + .../atlas/plugin/util/PolicyRefresher.java | 38 ++- .../plugin/util/RangerPolicyDeltaUtil.java | 12 +- .../atlas/plugin/util/ServicePolicies.java | 2 +- .../CachePolicyTransformerImpl.java | 237 ++++++++++++++++-- .../apache/atlas/ApplicationProperties.java | 1 + .../org/apache/atlas/web/rest/AuthREST.java | 40 ++- 9 files changed, 299 insertions(+), 70 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index adfe24af1d..def6c6c0dd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -73,6 +73,9 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol @JsonIgnore public Long getPolicyId() { return policy != null ? policy.getId() : null; } + @JsonIgnore + public String getPolicyGuid() { return policy != null ? policy.getGuid() : null; } + @JsonIgnore public String getZoneName() { return policy != null ? policy.getZoneName() : null; } @@ -94,6 +97,7 @@ public String toString() { + ", serviceType:" + getServiceType() + ", policyType:" + getPolicyType() + ", policyId:[" + getPolicyId() + "]" + + ", policyGuid:[" + getPolicyGuid() + "]" + ", policy:[" + policy +"]"; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java index 5b3b469fb0..4324ccb2cd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java @@ -90,7 +90,7 @@ enum AuditModeEnum { private List dataMaskPolicyEvaluators; private List rowFilterPolicyEvaluators; private final List auditPolicyEvaluators; - private Map policyEvaluatorsMap; + private Map policyEvaluatorsMap; private boolean isContextEnrichersShared = false; private boolean isPreCleaned = false; @@ -654,9 +654,9 @@ public List getLikelyMatchPolicyEvaluators(RangerAccessRe } - public Map getPolicyEvaluatorsMap() { return policyEvaluatorsMap; } + public Map getPolicyEvaluatorsMap() { return policyEvaluatorsMap; } - RangerPolicyEvaluator getPolicyEvaluator(Long id) { + RangerPolicyEvaluator getPolicyEvaluator(String id) { return policyEvaluatorsMap.get(id); } @@ -1252,17 +1252,17 @@ private void removeEvaluatorFromTrie(RangerPolicyEvaluator oldEvaluator, RangerR } } - private Map createPolicyEvaluatorsMap() { - Map tmpPolicyEvaluatorMap = new HashMap<>(); + private Map createPolicyEvaluatorsMap() { + Map tmpPolicyEvaluatorMap = new HashMap<>(); for (RangerPolicyEvaluator evaluator : getPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } for (RangerPolicyEvaluator evaluator : getDataMaskPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } for (RangerPolicyEvaluator evaluator : getRowFilterPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } return tmpPolicyEvaluatorMap; @@ -1294,7 +1294,7 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) { } if (!RangerPolicy.POLICY_TYPE_AUDIT.equals(policy.getPolicyType())) { - policyEvaluatorsMap.put(policy.getId(), ret); + policyEvaluatorsMap.put(policy.getGuid(), ret); } } } @@ -1306,22 +1306,22 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) { return ret; } - private void removePolicy(Long id) { + private void removePolicy(String guid) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerPolicyRepository.removePolicy(" + id +")"); + LOG.debug("==> RangerPolicyRepository.removePolicy(" + guid +")"); } Iterator iterator = policies.iterator(); while (iterator.hasNext()) { - if (id.equals(iterator.next().getId())) { + if (guid.equals(iterator.next().getGuid())) { iterator.remove(); //break; } } - policyEvaluatorsMap.remove(id); + policyEvaluatorsMap.remove(guid); if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerPolicyRepository.removePolicy(" + id +")"); + LOG.debug("<== RangerPolicyRepository.removePolicy(" + guid +")"); } } @@ -1355,13 +1355,13 @@ private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) { } private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) { - + LOG.info("PolicyDelta: RangerPolicyRepository.update is called, policyGuid: "+delta.getPolicyGuid()); if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")"); } Integer changeType = delta.getChangeType(); String policyType = delta.getPolicyType(); - Long policyId = delta.getPolicyId(); + String policyId = delta.getPolicyGuid(); RangerPolicy policy = delta.getPolicy(); @@ -1472,7 +1472,7 @@ private void updateResourceTrie(List deltas) { for (RangerPolicyDelta delta : deltas) { final Integer changeType = delta.getChangeType(); final String serviceType = delta.getServiceType(); - final Long policyId = delta.getPolicyId(); + final String policyId = delta.getPolicyGuid(); final String policyType = delta.getPolicyType(); if (!serviceType.equals(this.serviceDef.getName())) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index b224cccc7e..fb44584e44 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -379,6 +379,7 @@ public void setPolicies(ServicePolicies policies) { RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine; newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies); + //TODO: this looks like a mistake, second arg should be servicePolicies which has the applied delta } if (newPolicyEngine != null) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index aae09a7d26..b18906dde4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -21,8 +21,12 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; +import org.apache.atlas.repository.audit.ESBasedAuditRepository; +import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,6 +44,8 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import static org.apache.atlas.ApplicationProperties.DELTA_BASED_REFRESH; + public class PolicyRefresher extends Thread { private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); @@ -64,6 +70,9 @@ public class PolicyRefresher extends Thread { private long lastActivationTimeInMillis; private boolean policiesSetInPlugin; private boolean serviceDefSetInPlugin; + private Configuration atlasConfig; + private boolean enableDeltaBasedRefresh; + private ESBasedAuditRepository auditRepository; public PolicyRefresher(RangerBasePlugin plugIn) { @@ -104,6 +113,16 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.userStoreProvider = new RangerUserStoreProvider(getServiceType(), appId, getServiceName(), atlasAuthAdminClient, cacheDir, pluginConfig); this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); + try { + this.atlasConfig = ApplicationProperties.get(); + this.auditRepository = new ESBasedAuditRepository(atlasConfig); + this.auditRepository.start(); + this.enableDeltaBasedRefresh = this.atlasConfig.getBoolean(DELTA_BASED_REFRESH, false); + } catch (AtlasException e) { + LOG.error("PolicyDelta: Error while reading atlas configuration", e); + this.enableDeltaBasedRefresh = false; + } + setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); if(LOG.isDebugEnabled()) { @@ -316,13 +335,20 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { - if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null) { RangerRESTUtils restUtils = new RangerRESTUtils(); - CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); - - svcPolicies = transformer.getPolicies(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis); + CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); + if (lastUpdatedTiemInMillis == -1) { + svcPolicies = transformer.getPoliciesAll(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis, null); + } else if (this.enableDeltaBasedRefresh) { + svcPolicies = transformer.getPoliciesDelta(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis); + } else { + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); + } } else { svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index e3cb0f3697..b92ae004d4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -79,7 +79,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List applyDeltas(List policies, List deltas, String compo for (RangerPolicyDelta delta : deltas) { final Integer changeType = delta.getChangeType(); - final Long policyId = delta.getPolicyId(); + final String policyGuid = delta.getPolicyGuid(); if (changeType == null) { isValid = false; @@ -171,7 +167,7 @@ public static boolean isValidDeltas(List deltas, String compo && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { isValid = false; - } else if (policyId == null) { + } else if (policyGuid == null) { isValid = false; } else { final String serviceType = delta.getServiceType(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 547349c8f5..1a4dd81f29 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -390,7 +390,7 @@ static public TagPolicies copyHeader(TagPolicies source, String componentService return ret; } - public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { + public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { ServicePolicies ret = copyHeader(servicePolicies); List oldResourcePolicies = policyEngine.getResourcePolicies(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 4c6b9621fa..f092a488a6 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -18,14 +18,19 @@ package org.apache.atlas.policytransformer; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; +import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; +import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.util.ServicePolicies; import org.apache.atlas.plugin.model.RangerPolicy; import org.apache.atlas.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; @@ -37,6 +42,7 @@ import org.apache.atlas.plugin.model.RangerServiceDef; import org.apache.atlas.plugin.model.RangerValiditySchedule; import org.apache.atlas.plugin.util.ServicePolicies.TagPolicies; +import org.apache.atlas.repository.audit.ESBasedAuditRepository; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -53,20 +59,11 @@ import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.apache.atlas.repository.Constants.NAME; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.SERVICE_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; @@ -116,11 +113,15 @@ public class CachePolicyTransformerImpl { private PurposeCachePolicyTransformer purposeTransformer; private AtlasEntityHeader service; + private final ESBasedAuditRepository auditRepository; + + private final Map auditEventToDeltaChangeType; @Inject - public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry, ESBasedAuditRepository auditRepository) throws AtlasBaseException { this.graph = new AtlasJanusGraph(); this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); + this.auditRepository = auditRepository; personaTransformer = new PersonaCachePolicyTransformer(entityRetriever); purposeTransformer = new PurposeCachePolicyTransformer(entityRetriever); @@ -131,13 +132,81 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa LOG.error("Failed to initialize discoveryService"); throw new AtlasBaseException(e.getCause()); } + + this.auditEventToDeltaChangeType = new HashMap<>(); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPolicies(String serviceName, String pluginId, Long lastUpdatedTime) { + public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Long lastUpdatedTime) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); + + ServicePolicies servicePolicies = new ServicePolicies(); + + try { + servicePolicies.setServiceName(serviceName); + + service = getServiceEntity(serviceName); + servicePolicies.setPolicyVersion(-1L); + servicePolicies.setPolicyUpdateTime(new Date()); + + if (service != null) { + servicePolicies.setServiceName(serviceName); + servicePolicies.setServiceId(service.getGuid()); + + String serviceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, serviceName); + servicePolicies.setServiceDef(getResourceAsObject(serviceDefName, RangerServiceDef.class)); + + List policiesDelta = getServicePoliciesDelta(service, 250, lastUpdatedTime); + servicePolicies.setPolicyDeltas(policiesDelta); + + + //Process tag based policies + String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); + if (StringUtils.isNotEmpty(tagServiceName)) { + AtlasEntityHeader tagService = getServiceEntity(tagServiceName); + + if (tagService != null) { + List tagRangerPolicies = getServicePolicies(tagService, 0, null); + + TagPolicies tagPolicies = new TagPolicies(); + + tagPolicies.setServiceName(tagServiceName); + tagPolicies.setPolicyUpdateTime(new Date()); + tagPolicies.setServiceId(tagService.getGuid()); + tagPolicies.setPolicyVersion(-1L); + + String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); + tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); + + servicePolicies.setTagPolicies(tagPolicies); + servicePolicies.getTagPolicies().setPolicies(tagRangerPolicies); + LOG.info("PolicyDelta: {}: Found tag policies - {}", serviceName, tagRangerPolicies.size()); + } + } + + + + LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); + LOG.info("PolicyDelta: Found and set {} policies as delta and {} tag policies", servicePolicies.getPolicyDeltas().size(), servicePolicies.getTagPolicies().getPolicies().size()); + } + + } catch (Exception e) { + LOG.error("PolicyDelta: {}: ERROR in getPoliciesDelta {}: {}", serviceName, e.getMessage(), e); + return null; + } + + RequestContext.get().endMetricRecord(recorder); + return servicePolicies; + } + + + public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime, Date latestEditTime) { //TODO: return only if updated AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPolicies." + serviceName); @@ -151,7 +220,7 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las servicePolicies.setPolicyUpdateTime(new Date()); if (service != null) { - List allPolicies = getServicePolicies(service, 250); + List allPolicies = getServicePolicies(service, 250, latestEditTime); servicePolicies.setServiceName(serviceName); servicePolicies.setServiceId(service.getGuid()); @@ -165,7 +234,7 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las AtlasEntityHeader tagService = getServiceEntity(tagServiceName); if (tagService != null) { - allPolicies.addAll(getServicePolicies(tagService, 0)); + allPolicies.addAll(getServicePolicies(tagService, 0, latestEditTime)); TagPolicies tagPolicies = new TagPolicies(); @@ -195,7 +264,7 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las } } catch (Exception e) { - LOG.error("ERROR in getPolicies {}: ", e); + LOG.error("ERROR in getPolicies: ", e); return null; } @@ -203,14 +272,30 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las return servicePolicies; } - private List getServicePolicies(AtlasEntityHeader service, int batchSize) throws AtlasBaseException, IOException { + private List getServicePolicies(AtlasEntityHeader service, int batchSize, Date latestEditTime) throws AtlasBaseException, IOException, InterruptedException { List servicePolicies = new ArrayList<>(); + List atlasPolicies = new ArrayList<>(); String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); - List atlasPolicies = getAtlasPolicies(serviceName, batchSize); + int maxAttempts = 5; + int sleepFor = 500; + for (int attempt = 0; attempt <= maxAttempts; attempt++) { + try { + atlasPolicies = getAtlasPolicies(serviceName, batchSize, latestEditTime, new ArrayList<>()); + break; + } catch (AtlasBaseException e) { + LOG.error("ES_SYNC_FIX: {}: ERROR in getServicePolicies: {}", serviceName, e.getMessage()); + TimeUnit.MILLISECONDS.sleep(sleepFor); + if (attempt == maxAttempts) { + throw e; + } + sleepFor *= 2; + } + } + LOG.info("ES_SYNC_FIX: {}: Moving to transform policies, size: {}", serviceName, atlasPolicies.size()); if (CollectionUtils.isNotEmpty(atlasPolicies)) { //transform policies servicePolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -218,6 +303,83 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } + private List getServicePoliciesDelta(AtlasEntityHeader service, int batchSize, Long lastUpdatedTime) throws AtlasBaseException, IOException { + + String serviceName = (String) service.getAttribute("name"); + String serviceType = (String) service.getAttribute("authServiceType"); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getServicePoliciesWithDelta." + serviceName); + + List policyDeltas = new ArrayList<>(); + + // TODO: when getServicePolicies (without delta) is removed, merge the pagination for audit logs and policy fetch into one + List auditEvents = queryPoliciesAuditLogs(serviceName, lastUpdatedTime, batchSize); + Map policiesWithChangeType = new HashMap<>(); + for (EntityAuditEventV2 event : auditEvents) { + if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policiesWithChangeType.containsKey(event.getEntityId())) { + policiesWithChangeType.put(event.getEntityId(), event.getAction()); + } + } + LOG.info("PolicyDelta: {}: Total audit logs found = {}, events for {} ({}) = {}", serviceName, auditEvents.size(), POLICY_ENTITY_TYPE, policiesWithChangeType.size(), policiesWithChangeType); + if (policiesWithChangeType.isEmpty()) { + return policyDeltas; + } + + ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, null, policyGuids); + + List rangerPolicies = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(atlasPolicies)) { + rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); + } + + for (RangerPolicy policy : rangerPolicies) { + Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getGuid())); + RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); + policyDeltas.add(delta); + } + LOG.info("PolicyDelta: {}: atlas policies found = {}, delta created = {}", serviceName, atlasPolicies.size(), policyDeltas.size()); + RequestContext.get().endMetricRecord(recorder); + + return policyDeltas; + } + + private List queryPoliciesAuditLogs(String serviceName, Long afterTime, int batchSize) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.queryPoliciesAuditLogs." + serviceName); + + List entityUpdateToWatch = new ArrayList<>(); + entityUpdateToWatch.add(POLICY_ENTITY_TYPE); + entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); + + AuditSearchParams parameters = new AuditSearchParams(); + Map dsl = getMap("size", batchSize); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); + afterTime = afterTime == -1 ? 0 : afterTime; + mustClauseList.add(getMap("range", getMap("created", getMap("gte", afterTime)))); + + List> sortList = new ArrayList<>(0); + sortList.add(getMap("created", getMap("order", "desc"))); + dsl.put("sort", sortList); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + parameters.setDsl(dsl); + + List events = new ArrayList<>(); + try { + EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + } catch (AtlasBaseException e) { + LOG.error("ERROR in queryPoliciesAuditLogs while fetching entity audits {}: ", e.getMessage(), e); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + return events; + } + private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, String serviceType, String serviceName) throws IOException, AtlasBaseException { @@ -417,13 +579,13 @@ private List getPolicyConditions(AtlasEntityHeader at return null; } - List conditions = (List) atlasPolicy.getAttribute("policyConditions"); + List> conditions = (List>) atlasPolicy.getAttribute("policyConditions"); - for (AtlasStruct condition : conditions) { + for (HashMap condition : conditions) { RangerPolicyItemCondition rangerCondition = new RangerPolicyItemCondition(); - rangerCondition.setType((String) condition.getAttribute("policyConditionType")); - rangerCondition.setValues((List) condition.getAttribute("policyConditionValues")); + rangerCondition.setType((String) condition.get("policyConditionType")); + rangerCondition.setValues((List) condition.get("policyConditionValues")); ret.add(rangerCondition); } @@ -452,7 +614,7 @@ private List getPolicyValiditySchedule(AtlasEntityHeader return ret; } - private List getAtlasPolicies(String serviceName, int batchSize) throws AtlasBaseException { + private List getAtlasPolicies(String serviceName, int batchSize, Date latestEditTime, List policyGuids) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl."+service+".getAtlasPolicies"); List ret = new ArrayList<>(); @@ -481,8 +643,13 @@ private List getAtlasPolicies(String serviceName, int batchSi List> mustClauseList = new ArrayList<>(); mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); + if (!policyGuids.isEmpty()) { + mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); + } + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); List sortList = new ArrayList<>(0); @@ -509,13 +676,33 @@ private List getAtlasPolicies(String serviceName, int batchSi List headers = discoveryService.directIndexSearch(indexSearchParams).getEntities(); if (headers != null) { ret.addAll(headers); + LOG.info("ES_SYNC_FIX: {}: ======= Found result with {} policies", serviceName, headers.size()); } else { found = false; + LOG.info("ES_SYNC_FIX: {}: ======= Found result with null policies", serviceName); } from += size; } while (found && ret.size() % size == 0); + if (Objects.equals(serviceName, "atlas")) { + boolean latestEditFound = false; + Date latestEditTimeAvailable = null; + for (AtlasEntityHeader entity : ret) { + // LOG.info("ES_SYNC_FIX: {}: Looping on returned policies: {}, size: {}", serviceName, entity.getDisplayText(), ret.size()); + if (latestEditTime == null || entity.getUpdateTime().compareTo(latestEditTime) >= 0) { + LOG.info("ES_SYNC_FIX: {}: Found latest policy: {}, latestEditTime: {}, found policy time: {}", serviceName, entity.getDisplayText(), latestEditTime, entity.getUpdateTime()); + latestEditFound = true; + break; + } + latestEditTimeAvailable = entity.getUpdateTime(); + // LOG.info("ES_SYNC_FIX: {}: Checked for latest edit, entity: {}, latestEditTimeAvailable: {}", serviceName, entity.getDisplayText(), latestEditTimeAvailable); + } + if (latestEditTime != null && !latestEditFound) { + LOG.info("ES_SYNC_FIX: {}: Latest edit not found yet, policies: {}, latestEditTime: {}, latestEditTimeAvailable: {}", serviceName, ret.size(), latestEditTime, latestEditTimeAvailable); + throw new AtlasBaseException("Latest edit not found yet"); + } + } } finally { RequestContext.get().endMetricRecord(recorder); diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java index 78b487a8dd..dc198ba2fa 100644 --- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java +++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -70,6 +70,7 @@ public final class ApplicationProperties extends PropertiesConfiguration { public static final boolean DEFAULT_INDEX_RECOVERY = true; public static final AtlasRunMode DEFAULT_ATLAS_RUN_MODE = AtlasRunMode.PROD; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; + public static final String DELTA_BASED_REFRESH = "atlas.authorizer.enable.delta_based_refresh"; public static final SimpleEntry DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true"); public static final SimpleEntry DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20"); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index c5868cbfa3..a8a535cc7c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -21,6 +21,7 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.plugin.util.KeycloakUserStore; @@ -51,10 +52,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; @@ -152,11 +150,12 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); } - if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { + Long latestEditTime = getLastEditTime(serviceName, lastUpdatedTime); + if (latestEditTime == null) { return null; } - ServicePolicies ret = policyTransformer.getPolicies(serviceName, pluginId, lastUpdatedTime); + ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime, new Date(latestEditTime)); updateLastSync(serviceName); @@ -184,7 +183,7 @@ private void updateLastSync(String serviceName) { } } - private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { + private Long getLastEditTime(String serviceName, long lastUpdatedTime) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.isPolicyUpdated." + serviceName); List entityUpdateToWatch = new ArrayList<>(); @@ -199,26 +198,41 @@ private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + List> sortList = new ArrayList<>(); + sortList.add(getMap("created", "desc")); + dsl.put("sort", sortList); + parameters.setDsl(dsl); + Long lastEditTime = 0L; // this timestamp is used to verify if the found policies are synced with any policy create or update op on cassandra try { EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); - - if (result == null || CollectionUtils.isEmpty(result.getEntityAudits())) { - return false; + if (result != null) { + if (!CollectionUtils.isEmpty(result.getEntityAudits())) { + EntityAuditEventV2 lastAuditLog = result.getEntityAudits().get(0); + if (!EntityAuditEventV2.EntityAuditActionV2.getDeleteActions().contains(lastAuditLog.getAction()) && + lastAuditLog.getTypeName().equals(POLICY_ENTITY_TYPE) + ) { + lastEditTime = lastAuditLog.getTimestamp(); + } else { + LOG.info("ES_SYNC_FIX: {}: found delete action, so ignoring the last edit time: {}", serviceName, lastAuditLog.getTimestamp()); + } + } else { + lastEditTime = null; // no edits found + } } } catch (AtlasBaseException e) { LOG.error("ERROR in getPoliciesIfUpdated while fetching entity audits {}: ", e.getMessage()); - return true; } finally { RequestContext.get().endMetricRecord(recorder); + LOG.info("Last edit time for service {} is {}, dsl: {}", serviceName, lastEditTime, dsl); } - return true; + return lastEditTime; } private Map getMap(String key, Object value) { From b97cd8e461a46029daf9072bd62dd76566354f23 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 18 Nov 2024 17:02:35 +0530 Subject: [PATCH 161/241] Remove time based policy fetch retry during ES sync --- .../atlas/plugin/util/PolicyRefresher.java | 6 +-- .../CachePolicyTransformerImpl.java | 54 +++---------------- .../org/apache/atlas/web/rest/AuthREST.java | 40 +++++--------- 3 files changed, 23 insertions(+), 77 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b18906dde4..617dc10e7f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -340,12 +340,10 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); if (lastUpdatedTiemInMillis == -1) { svcPolicies = transformer.getPoliciesAll(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis, null); + restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); } else if (this.enableDeltaBasedRefresh) { svcPolicies = transformer.getPoliciesDelta(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis); + restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); } else { svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index f092a488a6..4b04b6856d 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -18,7 +18,6 @@ package org.apache.atlas.policytransformer; -import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; @@ -60,7 +59,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.*; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.*; @@ -172,7 +170,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon AtlasEntityHeader tagService = getServiceEntity(tagServiceName); if (tagService != null) { - List tagRangerPolicies = getServicePolicies(tagService, 0, null); + List tagRangerPolicies = getServicePolicies(tagService, 0); TagPolicies tagPolicies = new TagPolicies(); @@ -206,7 +204,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon } - public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime, Date latestEditTime) { + public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime) { //TODO: return only if updated AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPolicies." + serviceName); @@ -220,7 +218,7 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long servicePolicies.setPolicyUpdateTime(new Date()); if (service != null) { - List allPolicies = getServicePolicies(service, 250, latestEditTime); + List allPolicies = getServicePolicies(service, 250); servicePolicies.setServiceName(serviceName); servicePolicies.setServiceId(service.getGuid()); @@ -234,7 +232,7 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long AtlasEntityHeader tagService = getServiceEntity(tagServiceName); if (tagService != null) { - allPolicies.addAll(getServicePolicies(tagService, 0, latestEditTime)); + allPolicies.addAll(getServicePolicies(tagService, 0)); TagPolicies tagPolicies = new TagPolicies(); @@ -272,30 +270,14 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long return servicePolicies; } - private List getServicePolicies(AtlasEntityHeader service, int batchSize, Date latestEditTime) throws AtlasBaseException, IOException, InterruptedException { + private List getServicePolicies(AtlasEntityHeader service, int batchSize) throws AtlasBaseException, IOException { List servicePolicies = new ArrayList<>(); - List atlasPolicies = new ArrayList<>(); String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, new ArrayList<>()); - int maxAttempts = 5; - int sleepFor = 500; - for (int attempt = 0; attempt <= maxAttempts; attempt++) { - try { - atlasPolicies = getAtlasPolicies(serviceName, batchSize, latestEditTime, new ArrayList<>()); - break; - } catch (AtlasBaseException e) { - LOG.error("ES_SYNC_FIX: {}: ERROR in getServicePolicies: {}", serviceName, e.getMessage()); - TimeUnit.MILLISECONDS.sleep(sleepFor); - if (attempt == maxAttempts) { - throw e; - } - sleepFor *= 2; - } - } - LOG.info("ES_SYNC_FIX: {}: Moving to transform policies, size: {}", serviceName, atlasPolicies.size()); if (CollectionUtils.isNotEmpty(atlasPolicies)) { //transform policies servicePolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -325,7 +307,7 @@ private List getServicePoliciesDelta(AtlasEntityHeader servic } ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); - List atlasPolicies = getAtlasPolicies(serviceName, batchSize, null, policyGuids); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, policyGuids); List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { @@ -614,7 +596,7 @@ private List getPolicyValiditySchedule(AtlasEntityHeader return ret; } - private List getAtlasPolicies(String serviceName, int batchSize, Date latestEditTime, List policyGuids) throws AtlasBaseException { + private List getAtlasPolicies(String serviceName, int batchSize, List policyGuids) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl."+service+".getAtlasPolicies"); List ret = new ArrayList<>(); @@ -676,33 +658,13 @@ private List getAtlasPolicies(String serviceName, int batchSi List headers = discoveryService.directIndexSearch(indexSearchParams).getEntities(); if (headers != null) { ret.addAll(headers); - LOG.info("ES_SYNC_FIX: {}: ======= Found result with {} policies", serviceName, headers.size()); } else { found = false; - LOG.info("ES_SYNC_FIX: {}: ======= Found result with null policies", serviceName); } from += size; } while (found && ret.size() % size == 0); - if (Objects.equals(serviceName, "atlas")) { - boolean latestEditFound = false; - Date latestEditTimeAvailable = null; - for (AtlasEntityHeader entity : ret) { - // LOG.info("ES_SYNC_FIX: {}: Looping on returned policies: {}, size: {}", serviceName, entity.getDisplayText(), ret.size()); - if (latestEditTime == null || entity.getUpdateTime().compareTo(latestEditTime) >= 0) { - LOG.info("ES_SYNC_FIX: {}: Found latest policy: {}, latestEditTime: {}, found policy time: {}", serviceName, entity.getDisplayText(), latestEditTime, entity.getUpdateTime()); - latestEditFound = true; - break; - } - latestEditTimeAvailable = entity.getUpdateTime(); - // LOG.info("ES_SYNC_FIX: {}: Checked for latest edit, entity: {}, latestEditTimeAvailable: {}", serviceName, entity.getDisplayText(), latestEditTimeAvailable); - } - if (latestEditTime != null && !latestEditFound) { - LOG.info("ES_SYNC_FIX: {}: Latest edit not found yet, policies: {}, latestEditTime: {}, latestEditTimeAvailable: {}", serviceName, ret.size(), latestEditTime, latestEditTimeAvailable); - throw new AtlasBaseException("Latest edit not found yet"); - } - } } finally { RequestContext.get().endMetricRecord(recorder); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index a8a535cc7c..bb945ffa97 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -21,7 +21,6 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AuditSearchParams; -import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.plugin.util.KeycloakUserStore; @@ -52,7 +51,10 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; @@ -150,12 +152,11 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); } - Long latestEditTime = getLastEditTime(serviceName, lastUpdatedTime); - if (latestEditTime == null) { + if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; } - ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime, new Date(latestEditTime)); + ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); updateLastSync(serviceName); @@ -183,7 +184,7 @@ private void updateLastSync(String serviceName) { } } - private Long getLastEditTime(String serviceName, long lastUpdatedTime) { + private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.isPolicyUpdated." + serviceName); List entityUpdateToWatch = new ArrayList<>(); @@ -198,41 +199,26 @@ private Long getLastEditTime(String serviceName, long lastUpdatedTime) { mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - List> sortList = new ArrayList<>(); - sortList.add(getMap("created", "desc")); - dsl.put("sort", sortList); - parameters.setDsl(dsl); - Long lastEditTime = 0L; // this timestamp is used to verify if the found policies are synced with any policy create or update op on cassandra try { EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); - if (result != null) { - if (!CollectionUtils.isEmpty(result.getEntityAudits())) { - EntityAuditEventV2 lastAuditLog = result.getEntityAudits().get(0); - if (!EntityAuditEventV2.EntityAuditActionV2.getDeleteActions().contains(lastAuditLog.getAction()) && - lastAuditLog.getTypeName().equals(POLICY_ENTITY_TYPE) - ) { - lastEditTime = lastAuditLog.getTimestamp(); - } else { - LOG.info("ES_SYNC_FIX: {}: found delete action, so ignoring the last edit time: {}", serviceName, lastAuditLog.getTimestamp()); - } - } else { - lastEditTime = null; // no edits found - } + + if (result == null || CollectionUtils.isEmpty(result.getEntityAudits())) { + return false; } } catch (AtlasBaseException e) { LOG.error("ERROR in getPoliciesIfUpdated while fetching entity audits {}: ", e.getMessage()); + return true; } finally { RequestContext.get().endMetricRecord(recorder); - LOG.info("Last edit time for service {} is {}, dsl: {}", serviceName, lastEditTime, dsl); } - return lastEditTime; + return true; } private Map getMap(String key, Object value) { From 8fba21861f838915e56bc03f940664f13b9e583a Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 18 Nov 2024 17:05:28 +0530 Subject: [PATCH 162/241] Add missing changes for policy conditons --- .../policytransformer/CachePolicyTransformerImpl.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 4b04b6856d..3a97d0144e 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -29,6 +29,7 @@ import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.util.ServicePolicies; import org.apache.atlas.plugin.model.RangerPolicy; @@ -561,13 +562,13 @@ private List getPolicyConditions(AtlasEntityHeader at return null; } - List> conditions = (List>) atlasPolicy.getAttribute("policyConditions"); + List conditions = (List) atlasPolicy.getAttribute("policyConditions"); - for (HashMap condition : conditions) { + for (AtlasStruct condition : conditions) { RangerPolicyItemCondition rangerCondition = new RangerPolicyItemCondition(); - rangerCondition.setType((String) condition.get("policyConditionType")); - rangerCondition.setValues((List) condition.get("policyConditionValues")); + rangerCondition.setType((String) condition.getAttribute("policyConditionType")); + rangerCondition.setValues((List) condition.getAttribute("policyConditionValues")); ret.add(rangerCondition); } From a5394d29cf5ad90524a4af4a156cb4928a31fc77 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 25 Nov 2024 15:57:55 +0530 Subject: [PATCH 163/241] Cleanup ranger id of transformed policy to get atlas policy guid --- .../java/org/apache/atlas/plugin/model/RangerPolicy.java | 7 +++++++ .../apache/atlas/plugin/util/RangerPolicyDeltaUtil.java | 4 ++-- .../policytransformer/CachePolicyTransformerImpl.java | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java index f81f8e2854..e1a6bda610 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -529,6 +529,13 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) { this.isDenyAllElse = isDenyAllElse == null ? Boolean.FALSE : isDenyAllElse; } + public String getAtlasGuid() { + if (getGuid().length() > 36) { + return getGuid().substring(0, 36); + } + return getGuid(); + } + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index b92ae004d4..e63345c10a 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -222,11 +222,11 @@ public static Boolean hasPolicyDeltas(final ServicePolicies servicePolicies) { if (isPoliciesExist && isPolicyDeltasExist) { LOG.warn("ServicePolicies contain both policies and policy-deltas!! Cannot build policy-engine from these servicePolicies. Please check server-side code!"); - LOG.warn("Downloaded ServicePolicies are [" + servicePolicies + "]"); + LOG.warn("Downloaded ServicePolicies for [" + servicePolicies.getServiceName() + "]"); ret = null; } else if (!isPoliciesExist && !isPolicyDeltasExist) { LOG.warn("ServicePolicies do not contain any policies or policy-deltas!! There are no material changes in the policies."); - LOG.warn("Downloaded ServicePolicies are [" + servicePolicies + "]"); + LOG.warn("Downloaded ServicePolicies for [" + servicePolicies.getServiceName() + "]"); ret = null; } else { ret = isPolicyDeltasExist; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 3a97d0144e..ebe8812db1 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -182,9 +182,9 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); + tagPolicies.setPolicies(tagRangerPolicies); servicePolicies.setTagPolicies(tagPolicies); - servicePolicies.getTagPolicies().setPolicies(tagRangerPolicies); LOG.info("PolicyDelta: {}: Found tag policies - {}", serviceName, tagRangerPolicies.size()); } } @@ -308,6 +308,7 @@ private List getServicePoliciesDelta(AtlasEntityHeader servic } ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); + // this will have less policies as deleted won't be found List atlasPolicies = getAtlasPolicies(serviceName, batchSize, policyGuids); List rangerPolicies = new ArrayList<>(); @@ -316,7 +317,7 @@ private List getServicePoliciesDelta(AtlasEntityHeader servic } for (RangerPolicy policy : rangerPolicies) { - Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getGuid())); + Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getAtlasGuid())); RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); policyDeltas.add(delta); } From d803b86bfdc81bb4ef3c0ffec004ad3f0f82d57a Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 25 Nov 2024 16:50:02 +0530 Subject: [PATCH 164/241] Use App Configuration instead of reading the config again --- .../java/org/apache/atlas/plugin/util/PolicyRefresher.java | 4 ++-- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 617dc10e7f..4a5dc552bd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -22,6 +22,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; @@ -114,10 +115,9 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); try { - this.atlasConfig = ApplicationProperties.get(); + this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); this.auditRepository = new ESBasedAuditRepository(atlasConfig); this.auditRepository.start(); - this.enableDeltaBasedRefresh = this.atlasConfig.getBoolean(DELTA_BASED_REFRESH, false); } catch (AtlasException e) { LOG.error("PolicyDelta: Error while reading atlas configuration", e); this.enableDeltaBasedRefresh = false; diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index df2bca7860..f8c8e82063 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -120,6 +120,8 @@ public enum AtlasConfiguration { ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), + DELTA_BASED_REFRESH_ENABLED("atlas.authorizer.enable.delta_based_refresh", false), + ATLAS_UD_RELATIONSHIPS_MAX_COUNT("atlas.ud.relationship.max.count", 100); From 04304e03e2435ce9fe4dd756bba06ee52e3e49a2 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 26 Nov 2024 13:13:30 +0530 Subject: [PATCH 165/241] Enable build on branch to test before merging --- .github/workflows/maven.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..38fc6c2898 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - ns/fix/delta-refresh jobs: build: @@ -48,7 +49,9 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + run: | + echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + echo "##[set-output name=branch_lower;]$(echo ${GITHUB_REF#refs/heads/} | awk '{gsub("/", "-"); print tolower($0)}')" id: get_branch - name: Create Maven Settings @@ -102,8 +105,8 @@ jobs: provenance: true push: true tags: | - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:latest + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:${{ steps.get_version.outputs.version }} - name: Scan Image uses: aquasecurity/trivy-action@master From 70bed029914e00268214591e9c3a7b429f325645 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 28 Nov 2024 00:20:55 +0530 Subject: [PATCH 166/241] Move audit log checks to authrest itself --- .../admin/client/AtlasAuthAdminClient.java | 2 +- .../admin/client/AtlasAuthRESTClient.java | 12 ++-- .../atlas/plugin/util/PolicyRefresher.java | 20 ++---- .../CachePolicyTransformerImpl.java | 49 ++++++++------- .../org/apache/atlas/web/rest/AuthREST.java | 61 ++++++++++++++++--- 5 files changed, 92 insertions(+), 52 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java index a79407ba9c..7d2f675d9e 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java @@ -8,7 +8,7 @@ public interface AtlasAuthAdminClient { void init(RangerPluginConfig config); - ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception; + ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws Exception; RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java index 10c97008c3..c2a7dafd83 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java @@ -32,6 +32,7 @@ public class AtlasAuthRESTClient implements AtlasAuthAdminClient { private static final String PARAM_LAST_UPDATED_TIME = "lastUpdatedTime"; private static final String PARAM_PLUGIN_ID = "pluginId"; + private static final String PARAM_USE_POLICY_DELTA = "usePolicyDelta"; @Override public void init(RangerPluginConfig config) { @@ -66,20 +67,20 @@ public String getPluginId(String serviceName, String appId) { } @Override - public ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception { - URI uri = buildURI("/download/policies/" + serviceName, lastUpdatedTimeInMillis); + public ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws Exception { + URI uri = buildURI("/download/policies/" + serviceName, lastUpdatedTimeInMillis, usePolicyDelta); return sendRequestAndGetResponse(uri, ServicePolicies.class); } @Override public RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception { - URI uri = buildURI("/download/roles/" + serviceName, lastUpdatedTimeInMillis); + URI uri = buildURI("/download/roles/" + serviceName, lastUpdatedTimeInMillis, false); return sendRequestAndGetResponse(uri, RangerRoles.class); } @Override public RangerUserStore getUserStoreIfUpdated(long lastUpdatedTimeInMillis) throws Exception { - URI uri = buildURI("/download/users/" + serviceName, lastUpdatedTimeInMillis); + URI uri = buildURI("/download/users/" + serviceName, lastUpdatedTimeInMillis, false); return sendRequestAndGetResponse(uri, RangerUserStore.class); } @@ -114,13 +115,14 @@ private T sendRequestAndGetResponse(URI uri, Class responseClass) throws return null; } - private URI buildURI(String path, long lastUpdatedTimeInMillis) throws URISyntaxException { + private URI buildURI(String path, long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws URISyntaxException { return new URIBuilder() .setScheme(SCHEME) .setHost(adminUrl) .setPath(path) .setParameter(PARAM_LAST_UPDATED_TIME, String.valueOf(lastUpdatedTimeInMillis)) .setParameter(PARAM_PLUGIN_ID, pluginId) + .setParameter(PARAM_USE_POLICY_DELTA, String.valueOf(usePolicyDelta)) .build(); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 4a5dc552bd..5946dc7d79 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -21,7 +21,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; @@ -45,8 +44,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import static org.apache.atlas.ApplicationProperties.DELTA_BASED_REFRESH; - public class PolicyRefresher extends Thread { private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); @@ -335,20 +332,15 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { - if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null) { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { RangerRESTUtils restUtils = new RangerRESTUtils(); CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); - if (lastUpdatedTiemInMillis == -1) { - svcPolicies = transformer.getPoliciesAll(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); - } else if (this.enableDeltaBasedRefresh) { - svcPolicies = transformer.getPoliciesDelta(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); - } else { - svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); - } + + svcPolicies = transformer.getPoliciesAll(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis); } else { - svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh); } boolean isUpdated = svcPolicies != null; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index ebe8812db1..6715103ce0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -142,7 +142,7 @@ public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Long lastUpdatedTime) { + public ServicePolicies createPoliciesWithDelta(String serviceName, Map policyChanges, List atlasPolicies) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); ServicePolicies servicePolicies = new ServicePolicies(); @@ -161,7 +161,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon String serviceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, serviceName); servicePolicies.setServiceDef(getResourceAsObject(serviceDefName, RangerServiceDef.class)); - List policiesDelta = getServicePoliciesDelta(service, 250, lastUpdatedTime); + List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasPolicies); servicePolicies.setPolicyDeltas(policiesDelta); @@ -189,8 +189,6 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon } } - - LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); LOG.info("PolicyDelta: Found and set {} policies as delta and {} tag policies", servicePolicies.getPolicyDeltas().size(), servicePolicies.getTagPolicies().getPolicies().size()); } @@ -286,38 +284,23 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private List getServicePoliciesDelta(AtlasEntityHeader service, int batchSize, Long lastUpdatedTime) throws AtlasBaseException, IOException { - + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getServicePoliciesWithDelta." + serviceName); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getRangerPolicyDelta." + serviceName); List policyDeltas = new ArrayList<>(); - - // TODO: when getServicePolicies (without delta) is removed, merge the pagination for audit logs and policy fetch into one - List auditEvents = queryPoliciesAuditLogs(serviceName, lastUpdatedTime, batchSize); - Map policiesWithChangeType = new HashMap<>(); - for (EntityAuditEventV2 event : auditEvents) { - if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policiesWithChangeType.containsKey(event.getEntityId())) { - policiesWithChangeType.put(event.getEntityId(), event.getAction()); - } - } - LOG.info("PolicyDelta: {}: Total audit logs found = {}, events for {} ({}) = {}", serviceName, auditEvents.size(), POLICY_ENTITY_TYPE, policiesWithChangeType.size(), policiesWithChangeType); - if (policiesWithChangeType.isEmpty()) { + if (policyChanges.isEmpty()) { return policyDeltas; } - ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); - // this will have less policies as deleted won't be found - List atlasPolicies = getAtlasPolicies(serviceName, batchSize, policyGuids); - List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); } for (RangerPolicy policy : rangerPolicies) { - Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getAtlasGuid())); + Integer changeType = auditEventToDeltaChangeType.get(policyChanges.get(policy.getAtlasGuid())); RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); policyDeltas.add(delta); } @@ -364,6 +347,26 @@ private List queryPoliciesAuditLogs(String serviceName, Long return events; } + public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List events) { + Map policyChanges = new HashMap<>(); + for (EntityAuditEventV2 event : events) { + if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policyChanges.containsKey(event.getEntityId())) { + policyChanges.put(event.getEntityId(), event.getAction()); + } + } + + List atlasPolicies = new ArrayList<>(); + for (EntityAuditEventV2 event : events) { + AtlasEntityHeader policy = event.getEntityHeader(); + if (policy != null) { + atlasPolicies.add(policy); + } + } + LOG.info("PolicyDelta: {}: Found {} policy changes with {} policies", serviceName, policyChanges.size(), atlasPolicies.size()); + + return createPoliciesWithDelta(serviceName, policyChanges, atlasPolicies); + } + private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, String serviceType, String serviceName) throws IOException, AtlasBaseException { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index bb945ffa97..14ed3b1101 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -21,6 +21,7 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.plugin.util.KeycloakUserStore; @@ -51,10 +52,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; @@ -144,6 +142,7 @@ public RangerUserStore downloadUserStore(@PathParam("serviceName") final String @Timed public ServicePolicies downloadPolicies(@PathParam("serviceName") final String serviceName, @QueryParam("pluginId") String pluginId, + @DefaultValue("false") @QueryParam("usePolicyDelta") boolean usePolicyDelta, @DefaultValue("0") @QueryParam("lastUpdatedTime") Long lastUpdatedTime) throws AtlasBaseException { AtlasPerfTracer perf = null; @@ -152,12 +151,16 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); } - if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { - return null; + ServicePolicies ret; + if (usePolicyDelta) { + List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); + ret = policyTransformer.extractAndTransformPolicyDelta(serviceName, auditEvents); + } else { + if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { + return null; + } + ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); } - - ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); - updateLastSync(serviceName); return ret; @@ -166,6 +169,46 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s } } + private List getPolicyAuditLogs(String serviceName, long lastUpdatedTime) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthREST.getPolicyAuditLogs." + serviceName); + + List entityUpdateToWatch = new ArrayList<>(); + entityUpdateToWatch.add(POLICY_ENTITY_TYPE); + entityUpdateToWatch.add(PERSONA_ENTITY_TYPE); + entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); + + AuditSearchParams parameters = new AuditSearchParams(); + Map dsl = getMap("size", 100); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); + + lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; + mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + parameters.setDsl(dsl); + parameters.setAttributes(new HashSet<>(Arrays.asList( + "qualifiedName", + "name" + ))); + + List events = new ArrayList<>(); + try { + EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + } catch (AtlasBaseException e) { + LOG.error("ERROR in getPolicyAuditLogs while fetching entity audits {}: ", e.getMessage()); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + + return events; + } + private void updateLastSync(String serviceName) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.updateLastSync." + serviceName); From 49e0deb477d94ec8600b3072bc4eff0af52e3c82 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 28 Nov 2024 12:01:55 +0530 Subject: [PATCH 167/241] Remove auditRepo from policy refresher and transformer --- .../atlas/plugin/util/PolicyRefresher.java | 14 ++----- .../CachePolicyTransformerImpl.java | 41 +------------------ 2 files changed, 4 insertions(+), 51 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 5946dc7d79..b9331a547c 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -68,9 +68,7 @@ public class PolicyRefresher extends Thread { private long lastActivationTimeInMillis; private boolean policiesSetInPlugin; private boolean serviceDefSetInPlugin; - private Configuration atlasConfig; private boolean enableDeltaBasedRefresh; - private ESBasedAuditRepository auditRepository; public PolicyRefresher(RangerBasePlugin plugIn) { @@ -111,14 +109,8 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.userStoreProvider = new RangerUserStoreProvider(getServiceType(), appId, getServiceName(), atlasAuthAdminClient, cacheDir, pluginConfig); this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); - try { - this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); - this.auditRepository = new ESBasedAuditRepository(atlasConfig); - this.auditRepository.start(); - } catch (AtlasException e) { - LOG.error("PolicyDelta: Error while reading atlas configuration", e); - this.enableDeltaBasedRefresh = false; - } + this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); + LOG.info("PolicyRefresher(serviceName=" + serviceName + ") - delta based policy refresh is enabled"); setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); @@ -334,7 +326,7 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { RangerRESTUtils restUtils = new RangerRESTUtils(); - CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); + CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); svcPolicies = transformer.getPoliciesAll(serviceName, restUtils.getPluginId(serviceName, plugIn.getAppId()), diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 6715103ce0..3edd214e3b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -112,15 +112,13 @@ public class CachePolicyTransformerImpl { private PurposeCachePolicyTransformer purposeTransformer; private AtlasEntityHeader service; - private final ESBasedAuditRepository auditRepository; private final Map auditEventToDeltaChangeType; @Inject - public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry, ESBasedAuditRepository auditRepository) throws AtlasBaseException { + public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { this.graph = new AtlasJanusGraph(); this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); - this.auditRepository = auditRepository; personaTransformer = new PersonaCachePolicyTransformer(entityRetriever); purposeTransformer = new PurposeCachePolicyTransformer(entityRetriever); @@ -310,43 +308,6 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } - private List queryPoliciesAuditLogs(String serviceName, Long afterTime, int batchSize) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.queryPoliciesAuditLogs." + serviceName); - - List entityUpdateToWatch = new ArrayList<>(); - entityUpdateToWatch.add(POLICY_ENTITY_TYPE); - entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); - - AuditSearchParams parameters = new AuditSearchParams(); - Map dsl = getMap("size", batchSize); - - List> mustClauseList = new ArrayList<>(); - mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); - afterTime = afterTime == -1 ? 0 : afterTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gte", afterTime)))); - - List> sortList = new ArrayList<>(0); - sortList.add(getMap("created", getMap("order", "desc"))); - dsl.put("sort", sortList); - - dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - - parameters.setDsl(dsl); - - List events = new ArrayList<>(); - try { - EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); - if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { - events = result.getEntityAudits(); - } - } catch (AtlasBaseException e) { - LOG.error("ERROR in queryPoliciesAuditLogs while fetching entity audits {}: ", e.getMessage(), e); - } finally { - RequestContext.get().endMetricRecord(recorder); - } - return events; - } - public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List events) { Map policyChanges = new HashMap<>(); for (EntityAuditEventV2 event : events) { From 2aad6a5c429fb6352d7decd9ca3da20db4d4f7b8 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 29 Nov 2024 15:11:06 +0530 Subject: [PATCH 168/241] Revert to ealier way to fetching changed policies Policy object in audit log was querying all policies one by one --- .../atlas/plugin/model/RangerPolicyDelta.java | 4 +-- .../plugin/service/RangerBasePlugin.java | 2 +- .../atlas/plugin/util/PolicyRefresher.java | 3 ++ .../CachePolicyTransformerImpl.java | 34 ++++++++----------- .../org/apache/atlas/web/rest/AuthREST.java | 13 +++---- 5 files changed, 24 insertions(+), 32 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index def6c6c0dd..b5f9b58233 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -19,8 +19,8 @@ package org.apache.atlas.plugin.model; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonIgnore; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index fb44584e44..76ee3c3e1f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -314,7 +314,7 @@ public void setPolicies(ServicePolicies policies) { Boolean hasPolicyDeltas = RangerPolicyDeltaUtil.hasPolicyDeltas(policies); if (hasPolicyDeltas == null) { - LOG.info("Downloaded policies do not require policy change !! [" + policies + "]"); + LOG.info("Downloaded policies do not require policy change !! [" + (policies.getPolicies() != null ? policies.getPolicies().size() : 0) + "]"); if (this.policyEngine == null) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b9331a547c..5b806fd08e 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -324,7 +324,9 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading all policies for first time"); RangerRESTUtils restUtils = new RangerRESTUtils(); CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); @@ -332,6 +334,7 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); } else { + LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTiemInMillis); svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 3edd214e3b..c0a8894cd5 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -22,10 +22,8 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.audit.AuditSearchParams; import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; -import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -42,7 +40,6 @@ import org.apache.atlas.plugin.model.RangerServiceDef; import org.apache.atlas.plugin.model.RangerValiditySchedule; import org.apache.atlas.plugin.util.ServicePolicies.TagPolicies; -import org.apache.atlas.repository.audit.ESBasedAuditRepository; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -97,12 +94,13 @@ public class CachePolicyTransformerImpl { private static final String ATTR_POLICY_GROUPS = "policyGroups"; private static final String ATTR_POLICY_USERS = "policyUsers"; private static final String ATTR_POLICY_ROLES = "policyRoles"; - private static final String ATTR_POLICY_VALIDITY = "policyValiditySchedule"; - private static final String ATTR_POLICY_CONDITIONS = "policyConditions"; - private static final String ATTR_POLICY_MASK_TYPE = "policyMaskType"; + public static final String ATTR_POLICY_VALIDITY = "policyValiditySchedule"; + public static final String ATTR_POLICY_CONDITIONS = "policyConditions"; + public static final String ATTR_POLICY_MASK_TYPE = "policyMaskType"; private static final String RESOURCE_SERVICE_DEF_PATH = "/service-defs/"; private static final String RESOURCE_SERVICE_DEF_PATTERN = RESOURCE_SERVICE_DEF_PATH + "atlas-servicedef-%s.json"; + public static final int POLICY_BATCH_SIZE = 250; private EntityDiscoveryService discoveryService; private AtlasGraph graph; @@ -140,7 +138,7 @@ public AtlasEntityHeader getService() { return service; } - public ServicePolicies createPoliciesWithDelta(String serviceName, Map policyChanges, List atlasPolicies) { + public ServicePolicies getPoliciesDelta(String serviceName, Map policyChanges) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); ServicePolicies servicePolicies = new ServicePolicies(); @@ -159,7 +157,7 @@ public ServicePolicies createPoliciesWithDelta(String serviceName, Map policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasPolicies); + List policiesDelta = getRangerPolicyDelta(service, policyChanges); servicePolicies.setPolicyDeltas(policiesDelta); @@ -215,7 +213,7 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long servicePolicies.setPolicyUpdateTime(new Date()); if (service != null) { - List allPolicies = getServicePolicies(service, 250); + List allPolicies = getServicePolicies(service, POLICY_BATCH_SIZE); servicePolicies.setServiceName(serviceName); servicePolicies.setServiceId(service.getGuid()); @@ -282,7 +280,7 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getRangerPolicyDelta." + serviceName); @@ -292,6 +290,9 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } + ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); + List atlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); + List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -308,7 +309,7 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } - public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List events) { + public Map createPolicyChangeMap(String serviceName, List events) { Map policyChanges = new HashMap<>(); for (EntityAuditEventV2 event : events) { if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policyChanges.containsKey(event.getEntityId())) { @@ -316,16 +317,9 @@ public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List atlasPolicies = new ArrayList<>(); - for (EntityAuditEventV2 event : events) { - AtlasEntityHeader policy = event.getEntityHeader(); - if (policy != null) { - atlasPolicies.add(policy); - } - } - LOG.info("PolicyDelta: {}: Found {} policy changes with {} policies", serviceName, policyChanges.size(), atlasPolicies.size()); + LOG.info("PolicyDelta: {}: Found {} policy changes in {} policies", serviceName, events.size(), policyChanges.size()); - return createPoliciesWithDelta(serviceName, policyChanges, atlasPolicies); + return policyChanges; } private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 14ed3b1101..e4ac2552bd 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -154,7 +154,8 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); - ret = policyTransformer.extractAndTransformPolicyDelta(serviceName, auditEvents); + Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); + ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges); } else { if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; @@ -174,8 +175,6 @@ private List getPolicyAuditLogs(String serviceName, long las List entityUpdateToWatch = new ArrayList<>(); entityUpdateToWatch.add(POLICY_ENTITY_TYPE); - entityUpdateToWatch.add(PERSONA_ENTITY_TYPE); - entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); AuditSearchParams parameters = new AuditSearchParams(); Map dsl = getMap("size", 100); @@ -187,16 +186,12 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - parameters.setDsl(dsl); - parameters.setAttributes(new HashSet<>(Arrays.asList( - "qualifiedName", - "name" - ))); List events = new ArrayList<>(); try { - EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + String query = parameters.getQueryString(); + EntityAuditSearchResult result = auditRepository.searchEvents(query); // attributes are not getting passed in query if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { events = result.getEntityAudits(); } From dbf2854f6fbdc3e9689e22ae2cdddf1a0388ed8e Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 2 Dec 2024 14:51:33 +0530 Subject: [PATCH 169/241] Use last policy updated at time as last update time --- .../atlas/plugin/model/RangerPolicy.java | 4 +- .../atlas/plugin/util/PolicyRefresher.java | 7 +-- .../atlas/plugin/util/ServicePolicies.java | 44 ++++++++++++++++--- .../CachePolicyTransformerImpl.java | 11 +++-- .../org/apache/atlas/web/rest/AuthREST.java | 3 +- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java index e1a6bda610..3a4e52fd3b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -19,8 +19,9 @@ package org.apache.atlas.plugin.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.collections.CollectionUtils; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -529,6 +530,7 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) { this.isDenyAllElse = isDenyAllElse == null ? Boolean.FALSE : isDenyAllElse; } + @JsonIgnore public String getAtlasGuid() { if (getGuid().length() > 36) { return getGuid().substring(0, 36); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 5b806fd08e..b74741cfad 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -22,11 +22,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; -import org.apache.atlas.repository.audit.ESBasedAuditRepository; -import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -281,7 +278,7 @@ private void loadPolicy() { serviceDefSetInPlugin = false; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; - lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; + lastUpdatedTiemInMillis = svcPolicies.getLatestUpdateTime() != null ? svcPolicies.getLatestUpdateTime().getTime() : -1L; } else { if (!policiesSetInPlugin && !serviceDefSetInPlugin) { plugIn.setPolicies(null); @@ -402,7 +399,7 @@ private ServicePolicies loadFromCache() { } lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue(); - lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); + lastUpdatedTiemInMillis = policies.getLatestUpdateTime() == null ? -1 : policies.getLatestUpdateTime().getTime(); } } catch (Exception excp) { LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 1a4dd81f29..2fb01d9e5f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -20,10 +20,11 @@ package org.apache.atlas.plugin.util; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.collections.MapUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import org.apache.atlas.plugin.model.RangerPolicy; import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.model.RangerServiceDef; @@ -33,12 +34,7 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @JsonInclude(JsonInclude.Include.NON_NULL) @XmlRootElement @@ -180,6 +176,40 @@ public String toString() { + "securityZones=" + securityZones ; } + + @JsonIgnore + public Date getLatestUpdateTime() { + Date lastestUpdateTime = new Date(0L); + + if (policies != null && !policies.isEmpty()) { + for (RangerPolicy policy : policies) { + if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { + lastestUpdateTime = policy.getUpdateTime(); + } + } + } + + if (tagPolicies != null && tagPolicies.getPolicies() != null) { + for (RangerPolicy policy : tagPolicies.getPolicies()) { + if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { + lastestUpdateTime = policy.getUpdateTime(); + } + } + } + + if (policyDeltas != null && !policyDeltas.isEmpty()) { + for (RangerPolicyDelta delta : policyDeltas) { + if (delta.getPolicy() != null && delta.getPolicy().getUpdateTime() != null && delta.getPolicy().getUpdateTime().after(lastestUpdateTime)) { + lastestUpdateTime = delta.getPolicy().getUpdateTime(); + } + } + } + if (Objects.equals(lastestUpdateTime, new Date(0L))) { + lastestUpdateTime = null; + } + return lastestUpdateTime; + } + public List getPolicyDeltas() { return this.policyDeltas; } public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index c0a8894cd5..8d4edad4e5 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -138,17 +138,18 @@ public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPoliciesDelta(String serviceName, Map policyChanges) { + public ServicePolicies getPoliciesDelta(String serviceName, Map policyChanges, long lastAuditEventTime) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); ServicePolicies servicePolicies = new ServicePolicies(); try { - servicePolicies.setServiceName(serviceName); - service = getServiceEntity(serviceName); + servicePolicies.setServiceName(serviceName); servicePolicies.setPolicyVersion(-1L); - servicePolicies.setPolicyUpdateTime(new Date()); + + Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); + servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { servicePolicies.setServiceName(serviceName); @@ -678,6 +679,8 @@ private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String servi policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); policy.setCreateTime(atlasPolicy.getCreateTime()); + policy.setUpdatedBy(atlasPolicy.getUpdatedBy()); + policy.setUpdateTime(atlasPolicy.getUpdateTime()); policy.setIsEnabled(getIsPolicyEnabled(atlasPolicy)); policy.setConditions(getPolicyConditions(atlasPolicy)); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index e4ac2552bd..3f552331a9 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -154,8 +154,9 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); + long lastEventTime = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); - ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges); + ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); } else { if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; From 0e678ed855b3b50c7005267b9620df672628edf2 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 2 Dec 2024 15:50:07 +0530 Subject: [PATCH 170/241] Use current time as last update time for policy refresher In local it was creating problem as all policies are updated on startup and query fetches policies which has greater or equal to that timestamp which include all of them. --- .../java/org/apache/atlas/plugin/util/PolicyRefresher.java | 3 ++- .../atlas/policytransformer/CachePolicyTransformerImpl.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b74741cfad..b8d7daf4df 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -37,6 +37,7 @@ import java.io.FileWriter; import java.io.Reader; import java.io.Writer; +import java.util.Date; import java.util.Timer; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -278,7 +279,7 @@ private void loadPolicy() { serviceDefSetInPlugin = false; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; - lastUpdatedTiemInMillis = svcPolicies.getLatestUpdateTime() != null ? svcPolicies.getLatestUpdateTime().getTime() : -1L; + lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; } else { if (!policiesSetInPlugin && !serviceDefSetInPlugin) { plugIn.setPolicies(null); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 8d4edad4e5..f8eef3c8e0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -148,7 +148,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map 0 ? new Date(lastAuditEventTime) : new Date(); + // Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); + Date policyUpdateTime = new Date(); servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { From d8e7ef60366a86144adb7a5cc1a1e770e52bdfc2 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 3 Dec 2024 17:11:55 +0530 Subject: [PATCH 171/241] Use pagination while fetching audit logs --- .../CachePolicyTransformerImpl.java | 12 +++++---- .../org/apache/atlas/web/rest/AuthREST.java | 26 ++++++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index f8eef3c8e0..7439fb9aec 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -169,7 +169,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map tagRangerPolicies = getServicePolicies(tagService, 0); +// List tagRangerPolicies = getServicePolicies(tagService, 0); TagPolicies tagPolicies = new TagPolicies(); @@ -180,15 +180,14 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map getAtlasPolicies(String serviceName, int batchSi Map dsl = getMap("size", 0); List> mustClauseList = new ArrayList<>(); - mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); if (!policyGuids.isEmpty()) { mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); + } else { + // no service filter required if guids are provided + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); } dsl.put("query", getMap("bool", getMap("must", mustClauseList))); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 3f552331a9..25e144344c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -163,6 +163,10 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s } ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); } + LOG.info("downloadPolicies: serviceName={}, lastUpdatedTime={}, policies fetched={} delta fetched={}", serviceName, + lastUpdatedTime > 0 ? new Date(lastUpdatedTime) : lastUpdatedTime, + ret != null && ret.getPolicies() != null ? ret.getPolicies().size() : 0, + ret != null && ret.getPolicyDeltas() != null ? ret.getPolicyDeltas().size() : 0); updateLastSync(serviceName); return ret; @@ -184,18 +188,26 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - parameters.setDsl(dsl); + + int from = 0; + int size = 100; List events = new ArrayList<>(); try { - String query = parameters.getQueryString(); - EntityAuditSearchResult result = auditRepository.searchEvents(query); // attributes are not getting passed in query - if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { - events = result.getEntityAudits(); - } + do { + dsl.put("from", from); + dsl.put("size", size); + parameters.setDsl(dsl); + String query = parameters.getQueryString(); + EntityAuditSearchResult result = auditRepository.searchEvents(query); // attributes are not getting passed in query + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + from += size; + } while (events.size() == size); } catch (AtlasBaseException e) { LOG.error("ERROR in getPolicyAuditLogs while fetching entity audits {}: ", e.getMessage()); } finally { From 94e53f182e9bf29b93e181f4490ba8a3ca618277 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:22:31 +0530 Subject: [PATCH 172/241] fix: emmit audits in case of policy update by accesscontrol resource update --- .../graph/v2/preprocessor/AuthPolicyPreProcessor.java | 7 +++++++ .../v2/preprocessor/accesscontrol/PersonaPreProcessor.java | 1 + .../v2/preprocessor/accesscontrol/PurposePreProcessor.java | 2 ++ 3 files changed, 10 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index 58fb516564..c87d1bc6e1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -139,6 +139,9 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { policy.setAttribute(ATTR_POLICY_USERS, new ArrayList<>()); policy.setAttribute(ATTR_POLICY_GROUPS, new ArrayList<>()); + if(parentEntity != null) { + policy.setAttribute(ATTR_POLICY_IS_ENABLED, getIsAccessControlEnabled(parentEntity)); + } //create ES alias aliasStore.updateAlias(parent, policy); @@ -155,6 +158,10 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { policy.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + if(parentEntity != null) { + policy.setAttribute(ATTR_POLICY_IS_ENABLED, getIsAccessControlEnabled(parentEntity)); + } + //create ES alias aliasStore.updateAlias(parent, policy); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index 8a2c8b1945..e6f815e36a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -205,6 +205,7 @@ private void updatePoliciesIsEnabledAttr(EntityMutationContext context, AtlasEnt policyToBeUpdated.setAttribute(ATTR_POLICY_IS_ENABLED, enable); context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + RequestContext.get().cacheDifferentialEntity(policyToBeUpdated); } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java index ca8fcb803f..3dbf342feb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java @@ -167,6 +167,7 @@ private void processUpdatePurpose(EntityMutationContext context, policyToBeUpdated.setAttribute(ATTR_POLICY_RESOURCES, newTagsResources); context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + RequestContext.get().cacheDifferentialEntity(policyToBeUpdated); existingPurposeExtInfo.addReferredEntity(policyToBeUpdated); } @@ -199,6 +200,7 @@ private void updatePoliciesIsEnabledAttr(EntityMutationContext context, AtlasEnt policyToBeUpdated.setAttribute(ATTR_POLICY_IS_ENABLED, enable); context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + RequestContext.get().cacheDifferentialEntity(policyToBeUpdated); } } } From cb35dd8181a9f4c030f05d04eeaf78e37b36eb4a Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 3 Dec 2024 17:59:38 +0530 Subject: [PATCH 173/241] Use policy udpate time as last refresh time --- .../policytransformer/CachePolicyTransformerImpl.java | 4 ++-- .../src/main/java/org/apache/atlas/web/rest/AuthREST.java | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 7439fb9aec..5623120dc0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -148,8 +148,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map 0 ? new Date(lastAuditEventTime) : new Date(); - Date policyUpdateTime = new Date(); + Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); +// Date policyUpdateTime = new Date(); servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 25e144344c..8d8ebaf036 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -154,7 +154,8 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); - long lastEventTime = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); + long lastEventTime = auditEvents.isEmpty() ? 0 : auditEvents.get(auditEvents.size() - 1).getCreated(); + long lastEventTime2 = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); } else { @@ -192,6 +193,10 @@ private List getPolicyAuditLogs(String serviceName, long las dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + List> sortClause = new ArrayList<>(); + sortClause.add(getMap("created", getMap("order", "asc"))); + dsl.put("sort", sortClause); + int from = 0; int size = 100; From 32e075fa28c9621d1467f665892bfd699a29ee3e Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Wed, 4 Dec 2024 17:09:55 +0530 Subject: [PATCH 174/241] Handle delete policy event --- .../atlas/plugin/util/PolicyRefresher.java | 2 +- .../plugin/util/RangerPolicyDeltaUtil.java | 9 +++-- .../CachePolicyTransformerImpl.java | 40 ++++++++++++------- .../org/apache/atlas/web/rest/AuthREST.java | 7 +++- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b8d7daf4df..5f9351be26 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -108,7 +108,7 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); - LOG.info("PolicyRefresher(serviceName=" + serviceName + ") - delta based policy refresh is enabled"); + LOG.info("PolicyRefresher(serviceName=" + serviceName + ") - delta based policy refresh enabled="+this.enableDeltaBasedRefresh); setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index e63345c10a..0c8267b5c7 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -91,7 +91,10 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List 1) { - LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray())); + if (CollectionUtils.isEmpty(deletedPolicies)) { + LOG.warn("Unexpected: found no policy for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray())); } break; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 5623120dc0..1edf210162 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -60,17 +60,7 @@ import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_PRIORITY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SUB_CATEGORY; -import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_DATAMESH; -import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PERSONA; -import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PURPOSE; -import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; -import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyCategory; +import static org.apache.atlas.repository.util.AccessControlUtils.*; @Component public class CachePolicyTransformerImpl { @@ -304,7 +294,29 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); policyDeltas.add(delta); } - LOG.info("PolicyDelta: {}: atlas policies found = {}, delta created = {}", serviceName, atlasPolicies.size(), policyDeltas.size()); + + // handle delete changes separately as they won't be present in atlas policies + List deletedPolicyDeltas = new ArrayList<>(); + for (String policyGuid : policyGuids) { + if (policyChanges.get(policyGuid) == EntityAuditActionV2.ENTITY_DELETE) { + RangerPolicy deletedPolicy = new RangerPolicy(); + deletedPolicy.setGuid(policyGuid); + deletedPolicy.setService(serviceName); + deletedPolicy.setServiceType(serviceType); + RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta( + deletedPolicy.getId(), + auditEventToDeltaChangeType.get(EntityAuditActionV2.ENTITY_DELETE), + deletedPolicy.getVersion(), + deletedPolicy + ); + deletedPolicyDeltas.add(deletedPolicyDelta); + } + } + + policyDeltas.addAll(deletedPolicyDeltas); + + LOG.info("PolicyDelta: {}: atlas policies found={}, delta created={}, including deleted policies={}", + serviceName, atlasPolicies.size(), policyDeltas.size(), deletedPolicyDeltas.size()); RequestContext.get().endMetricRecord(recorder); return policyDeltas; @@ -585,15 +597,13 @@ private List getAtlasPolicies(String serviceName, int batchSi Map dsl = getMap("size", 0); List> mustClauseList = new ArrayList<>(); - - mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); if (!policyGuids.isEmpty()) { mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); } else { - // no service filter required if guids are provided mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); } dsl.put("query", getMap("bool", getMap("must", mustClauseList))); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 8d8ebaf036..c668aa5c90 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -155,7 +155,10 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); long lastEventTime = auditEvents.isEmpty() ? 0 : auditEvents.get(auditEvents.size() - 1).getCreated(); - long lastEventTime2 = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); + LOG.info("PolicyDelta: serviceName={}, lastUpdatedTime={}, audit events found={}", serviceName, lastEventTime, auditEvents.size()); + if (auditEvents.isEmpty()) { + return null; + } Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); } else { @@ -189,7 +192,7 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("created", getMap("gt", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); From 48d3a9d0db615cc9932ff6b5de2ac634244925d1 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Thu, 5 Dec 2024 12:55:47 +0530 Subject: [PATCH 175/241] fix: optimizing lineage traversal --- .../org/apache/atlas/discovery/EntityLineageService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index a8841eb1aa..38fb2535dc 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -40,6 +40,7 @@ import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation; import org.apache.atlas.model.lineage.AtlasLineageOnDemandInfo.LineageInfoOnDemand; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; @@ -375,7 +376,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i visitedVertices.add(getId(datasetVertex)); AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesIn = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesIn"); - Iterator incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); + Iterator incomingEdges = GraphHelper.getActiveEdges(datasetVertex, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE, IN); RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesIn); while (incomingEdges.hasNext()) { @@ -403,7 +404,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i } AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); - Iterator outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); + Iterator outgoingEdges = GraphHelper.getActiveEdges(processVertex, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE, OUT); RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdgesOut); while (outgoingEdges.hasNext()) { From eca292554f79f0e05e854ca8162675ade211695b Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 5 Dec 2024 12:54:40 +0530 Subject: [PATCH 176/241] Use individual imports and remove dead code --- .github/workflows/maven.yml | 9 ++--- .../atlas/plugin/util/PolicyRefresher.java | 2 +- .../atlas/plugin/util/ServicePolicies.java | 35 +------------------ .../CachePolicyTransformerImpl.java | 33 ++++++++++++----- .../apache/atlas/ApplicationProperties.java | 1 - .../org/apache/atlas/web/rest/AuthREST.java | 6 +++- 6 files changed, 34 insertions(+), 52 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 38fc6c2898..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - ns/fix/delta-refresh jobs: build: @@ -49,9 +48,7 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: | - echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - echo "##[set-output name=branch_lower;]$(echo ${GITHUB_REF#refs/heads/} | awk '{gsub("/", "-"); print tolower($0)}')" + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" id: get_branch - name: Create Maven Settings @@ -105,8 +102,8 @@ jobs: provenance: true push: true tags: | - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:latest - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:${{ steps.get_version.outputs.version }} + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} - name: Scan Image uses: aquasecurity/trivy-action@master diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 5f9351be26..ed5b1a3687 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -400,7 +400,7 @@ private ServicePolicies loadFromCache() { } lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue(); - lastUpdatedTiemInMillis = policies.getLatestUpdateTime() == null ? -1 : policies.getLatestUpdateTime().getTime(); + lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); } } catch (Exception excp) { LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 2fb01d9e5f..77717e1e08 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -177,39 +177,6 @@ public String toString() { ; } - @JsonIgnore - public Date getLatestUpdateTime() { - Date lastestUpdateTime = new Date(0L); - - if (policies != null && !policies.isEmpty()) { - for (RangerPolicy policy : policies) { - if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { - lastestUpdateTime = policy.getUpdateTime(); - } - } - } - - if (tagPolicies != null && tagPolicies.getPolicies() != null) { - for (RangerPolicy policy : tagPolicies.getPolicies()) { - if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { - lastestUpdateTime = policy.getUpdateTime(); - } - } - } - - if (policyDeltas != null && !policyDeltas.isEmpty()) { - for (RangerPolicyDelta delta : policyDeltas) { - if (delta.getPolicy() != null && delta.getPolicy().getUpdateTime() != null && delta.getPolicy().getUpdateTime().after(lastestUpdateTime)) { - lastestUpdateTime = delta.getPolicy().getUpdateTime(); - } - } - } - if (Objects.equals(lastestUpdateTime, new Date(0L))) { - lastestUpdateTime = null; - } - return lastestUpdateTime; - } - public List getPolicyDeltas() { return this.policyDeltas; } public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } @@ -420,7 +387,7 @@ static public TagPolicies copyHeader(TagPolicies source, String componentService return ret; } - public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { + public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { ServicePolicies ret = copyHeader(servicePolicies); List oldResourcePolicies = policyEngine.getResourcePolicies(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 1edf210162..17a234f534 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -56,11 +56,32 @@ import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; -import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.util.AccessControlUtils.*; +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.SERVICE_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_PRIORITY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SUB_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_DATAMESH; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PERSONA; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PURPOSE; +import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyCategory; @Component public class CachePolicyTransformerImpl { @@ -159,24 +180,18 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map tagRangerPolicies = getServicePolicies(tagService, 0); - TagPolicies tagPolicies = new TagPolicies(); tagPolicies.setServiceName(tagServiceName); tagPolicies.setPolicyUpdateTime(new Date()); tagPolicies.setServiceId(tagService.getGuid()); tagPolicies.setPolicyVersion(-1L); - String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); -// tagPolicies.setPolicies(tagRangerPolicies); servicePolicies.setTagPolicies(tagPolicies); -// LOG.info("PolicyDelta: {}: Found tag policies - {}", serviceName, tagRangerPolicies.size()); } } - LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); } diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java index dc198ba2fa..78b487a8dd 100644 --- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java +++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -70,7 +70,6 @@ public final class ApplicationProperties extends PropertiesConfiguration { public static final boolean DEFAULT_INDEX_RECOVERY = true; public static final AtlasRunMode DEFAULT_ATLAS_RUN_MODE = AtlasRunMode.PROD; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; - public static final String DELTA_BASED_REFRESH = "atlas.authorizer.enable.delta_based_refresh"; public static final SimpleEntry DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true"); public static final SimpleEntry DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20"); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index c668aa5c90..d50ac8d0ea 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -52,7 +52,11 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Date; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; From 05d3ef3f19fed106d08d570f3f569a164be65646 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 5 Dec 2024 13:05:36 +0530 Subject: [PATCH 177/241] Map entity purge event to delete change as well --- .../atlas/policytransformer/CachePolicyTransformerImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 17a234f534..db33577349 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -143,6 +143,7 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_PURGE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } public AtlasEntityHeader getService() { @@ -313,14 +314,15 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, // handle delete changes separately as they won't be present in atlas policies List deletedPolicyDeltas = new ArrayList<>(); for (String policyGuid : policyGuids) { - if (policyChanges.get(policyGuid) == EntityAuditActionV2.ENTITY_DELETE) { + int deltaChangeType = auditEventToDeltaChangeType.get(policyChanges.get(policyGuid)); + if (deltaChangeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { RangerPolicy deletedPolicy = new RangerPolicy(); deletedPolicy.setGuid(policyGuid); deletedPolicy.setService(serviceName); deletedPolicy.setServiceType(serviceType); RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta( deletedPolicy.getId(), - auditEventToDeltaChangeType.get(EntityAuditActionV2.ENTITY_DELETE), + deltaChangeType, deletedPolicy.getVersion(), deletedPolicy ); From 5126f15bf96ebce3eb96abd66d2bc0a989b2a15e Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 5 Dec 2024 22:39:07 +0530 Subject: [PATCH 178/241] mesh-292: update unique qn attribute --- .../v2/preprocessor/datamesh/DataDomainPreProcessor.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index b4668b0868..88b0da188d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -53,6 +53,7 @@ public class DataDomainPreProcessor extends AbstractDomainPreProcessor { private Map updatedPolicyResources; private EntityGraphRetriever retrieverNoRelation = null; private Map updatedDomainQualifiedNames; + public static final String UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX = "__u_"; public DataDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { @@ -290,10 +291,15 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, String currentDomainQualifiedName = childDomainVertex.getProperty(QUALIFIED_NAME, String.class); String updatedDomainQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentDomainQualifiedName); + String uniqueQualifiedNameAttribute = UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + QUALIFIED_NAME; + // Change domain qualifiedName childDomainVertex.setProperty(QUALIFIED_NAME, updatedDomainQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedDomainQualifiedName); + // Change unique qualifiedName attribute + childDomainVertex.setProperty(uniqueQualifiedNameAttribute, updatedDomainQualifiedName); + //change superDomainQN, parentDomainQN childDomainVertex.setProperty(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); childDomainVertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); @@ -347,7 +353,10 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, String currentQualifiedName = productVertex.getProperty(QUALIFIED_NAME, String.class); String updatedQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentQualifiedName); + String uniqueQualifiedNameAttribute = UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + QUALIFIED_NAME; + productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); + productVertex.setProperty(uniqueQualifiedNameAttribute, updatedQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); productVertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); From 4b00c081920c6f4b65cd96605cef091b0ab59874 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 01:20:11 +0530 Subject: [PATCH 179/241] janus0.6 changes --- .github/workflows/maven.yml | 1 + .../org/apache/atlas/AtlasConfiguration.java | 1 + .../store/graph/v2/EntityGraphRetriever.java | 247 +++++++++++++++++- 3 files changed, 245 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..e290252038 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - janus0.6 jobs: build: diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index df2bca7860..60d933d6ae 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -117,6 +117,7 @@ public enum AtlasConfiguration { ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), + ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 34d3036042..d3762b04c2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -49,6 +49,8 @@ import org.apache.atlas.repository.graphdb.AtlasElement; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.janus.AtlasJanusEdge; +import org.apache.atlas.repository.util.AccessControlUtils; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; import org.apache.atlas.type.AtlasEntityType; @@ -65,8 +67,12 @@ import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.IteratorUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -88,10 +94,7 @@ import java.util.Objects; import java.util.Queue; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; +import java.util.concurrent.*; import java.util.stream.Collectors; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; @@ -1009,7 +1012,114 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } + private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes) { + Map propertiesMap = new HashMap<>(); + if (entityType == null) { + return propertiesMap; + } + // Execute the traversal to fetch properties + GraphTraversal> traversal = graph.V(entityVertex.getId()).properties(); + + // Check if any attribute is a struct or object type for edge lookup + boolean isAnyAttributeAStructOrObject = attributes.stream().anyMatch(a -> { + + AtlasAttribute attribute = entityType.getAttribute(a); + if (attribute == null || attribute.getAttributeType() == null) { + return false; + } + + if (!(attribute.getAttributeType() instanceof AtlasArrayType)) { + return false; + + } + AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); + + if (arrayType.getElementType() == null) { + return false; + } + + AtlasType arrayElementType = arrayType.getElementType(); + + return arrayElementType.getTypeCategory() == TypeCategory.STRUCT; + }); + + if (isAnyAttributeAStructOrObject) { + List edgeProperties = IteratorUtils.toList(entityVertex.getEdges(AtlasEdgeDirection.OUT).iterator()); + List edgeLabels = + edgeProperties.stream() + .map(e -> { + AtlasJanusEdge edge = (AtlasJanusEdge) e; + return edge.getLabel().toString().substring(2); + }) + .collect(Collectors.toList()); + + edgeLabels.stream().forEach(e -> propertiesMap.put(e, StringUtils.SPACE)); + } + + // Iterate through the resulting VertexProperty objects + while (traversal.hasNext()) { + try { + VertexProperty property = traversal.next(); + + AtlasAttribute attribute = entityType.getAttribute(property.key()); + TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; + TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + + if (property.isPresent()) { + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { + Object value = propertiesMap.get(property.key()); + if (value instanceof List) { + ((List) value).add(property.value()); + } else { + List values = new ArrayList<>(); + values.add(property.value()); + propertiesMap.put(property.key(), values); + } + } else { + if (propertiesMap.get(property.key()) == null) { + propertiesMap.put(property.key(), property.value()); + } + } + } + } catch (RuntimeException e) { + LOG.error("Error preloading properties for entity vertex: {}", entityVertex, e); + throw e; // Re-throw the exception after logging it + } + } + return propertiesMap; + } + + + private boolean isPolicyAttribute(Set attributes) { + Set exclusionSet = new HashSet<>(Arrays.asList(AccessControlUtils.ATTR_POLICY_TYPE, + AccessControlUtils.ATTR_POLICY_USERS, + AccessControlUtils.ATTR_POLICY_GROUPS, + AccessControlUtils.ATTR_POLICY_ROLES, + AccessControlUtils.ATTR_POLICY_ACTIONS, + AccessControlUtils.ATTR_POLICY_CATEGORY, + AccessControlUtils.ATTR_POLICY_SUB_CATEGORY, + AccessControlUtils.ATTR_POLICY_RESOURCES, + AccessControlUtils.ATTR_POLICY_IS_ENABLED, + AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY, + AccessControlUtils.ATTR_POLICY_SERVICE_NAME, + AccessControlUtils.ATTR_POLICY_PRIORITY, + AccessControlUtils.REL_ATTR_POLICIES)); + + return exclusionSet.stream().anyMatch(attributes::contains); + } + private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { + boolean shouldPrefetch = !isPolicyAttribute(attributes) + && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); + + if (shouldPrefetch) { + return mapVertexToAtlasEntityHeaderWithPrefetch(entityVertex, attributes); + } else { + return mapVertexToAtlasEntityHeaderWithoutPrefetch(entityVertex, attributes); + } + } + + private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithoutPrefetch(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); AtlasEntityHeader ret = new AtlasEntityHeader(); try { @@ -1078,6 +1188,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, } } + Object attrValue = getVertexAttribute(entityVertex, attribute); if (attrValue != null) { @@ -1093,6 +1204,104 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, return ret; } + private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeaderWithPrefetch"); + AtlasEntityHeader ret = new AtlasEntityHeader(); + try { + //pre-fetching the properties + String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); //properties.get returns null + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); // this is not costly + Map properties = preloadProperties(entityVertex, entityType, attributes); + + String guid = (String) properties.get(Constants.GUID_PROPERTY_KEY); + + Integer value = (Integer)properties.get(Constants.IS_INCOMPLETE_PROPERTY_KEY); + Boolean isIncomplete = value != null && value.equals(INCOMPLETE_ENTITY_VALUE) ? Boolean.TRUE : Boolean.FALSE; + + ret.setTypeName(typeName); + ret.setGuid(guid); + + String state = (String)properties.get(Constants.STATE_PROPERTY_KEY); + Id.EntityState entityState = state == null ? null : Id.EntityState.valueOf(state); + ret.setStatus((entityState == Id.EntityState.DELETED) ? AtlasEntity.Status.DELETED : AtlasEntity.Status.ACTIVE); + + RequestContext context = RequestContext.get(); + boolean includeClassifications = context.includeClassifications(); + boolean includeClassificationNames = context.isIncludeClassificationNames(); + if(includeClassifications){ + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } else if (!includeClassifications && includeClassificationNames) { + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } + ret.setIsIncomplete(isIncomplete); + ret.setLabels(getLabels(entityVertex)); + + ret.setCreatedBy(properties.get(CREATED_BY_KEY) != null ? (String) properties.get(CREATED_BY_KEY) : null); + ret.setUpdatedBy(properties.get(MODIFIED_BY_KEY) != null ? (String) properties.get(MODIFIED_BY_KEY) : null); + ret.setCreateTime(properties.get(TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)properties.get(TIMESTAMP_PROPERTY_KEY)) : null); + ret.setUpdateTime(properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY)) : null); + + if(RequestContext.get().includeMeanings()) { + List termAssignmentHeaders = mapAssignedTerms(entityVertex); + ret.setMeanings(termAssignmentHeaders); + ret.setMeaningNames( + termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) + .collect(Collectors.toList())); + } + + if (entityType != null) { + for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { + Object attrValue = getVertexAttributePreFetchCache(entityVertex, headerAttribute, properties); + + if (attrValue != null) { + ret.setAttribute(headerAttribute.getName(), attrValue); + } + } + + if(properties.get(NAME) != null){ + ret.setDisplayText(properties.get(NAME).toString()); + } else if(properties.get(DISPLAY_NAME) != null) { + ret.setDisplayText(properties.get(DISPLAY_NAME).toString()); + } else if(properties.get(QUALIFIED_NAME) != null) { + ret.setDisplayText(properties.get(QUALIFIED_NAME).toString()); + } + + + + //attributes = only the attributes of entityType + if (CollectionUtils.isNotEmpty(attributes)) { + for (String attrName : attributes) { + AtlasAttribute attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attrName = toNonQualifiedName(attrName); + + if (ret.hasAttribute(attrName)) { + continue; + } + + attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attribute = entityType.getRelationshipAttribute(attrName, null); + } + } + + //this is a call to cassandra + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, properties); //use prefetch cache + + if (attrValue != null) { + ret.setAttribute(attrName, attrValue); + } + } + } + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + return ret; + } + private String toNonQualifiedName(String attrName) { String ret; if (attrName.contains(".")) { @@ -1709,6 +1918,36 @@ public Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) t return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null; } + public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute attribute, Map properties) throws AtlasBaseException { + if (vertex == null || attribute == null) { + return null; + } + + TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); + TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + + // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives + if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && + (typeCategory.equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { + return properties.get(attribute.getName()); + } + + // if value is empty && element is array of primitives, return empty list + if (properties.get(attribute.getName()) == null && (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE))) { + return new ArrayList<>(); + } + + // value is present as marker, fetch the value from the vertex + if (properties.get(attribute.getName()) != null) { + AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); + Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); + RequestContext.get().endMetricRecord(nonPrimitiveAttributes); + return mappedVertex; + } + + return null; + } + private Object getVertexAttributeIgnoreInactive(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException { return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false, true, true) : null; } From af722cba4e49c49178ec8f0046b4dc0d036e1a4f Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 01:51:55 +0530 Subject: [PATCH 180/241] fix object type and simplify conditional logic --- .../store/graph/v2/EntityGraphRetriever.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index d3762b04c2..cc22257f13 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1014,23 +1014,26 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes) { Map propertiesMap = new HashMap<>(); - if (entityType == null) { - return propertiesMap; - } + // Execute the traversal to fetch properties GraphTraversal> traversal = graph.V(entityVertex.getId()).properties(); // Check if any attribute is a struct or object type for edge lookup boolean isAnyAttributeAStructOrObject = attributes.stream().anyMatch(a -> { - + if (entityType == null) { + return false; + } AtlasAttribute attribute = entityType.getAttribute(a); if (attribute == null || attribute.getAttributeType() == null) { return false; } + if ((attribute.getAttributeType() instanceof AtlasStructType)) { + return true; + } + if (!(attribute.getAttributeType() instanceof AtlasArrayType)) { return false; - } AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); @@ -1925,15 +1928,16 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && - (typeCategory.equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { + (typeCategory.equals(TypeCategory.PRIMITIVE) || isArrayOfPrimitives)) { return properties.get(attribute.getName()); } // if value is empty && element is array of primitives, return empty list - if (properties.get(attribute.getName()) == null && (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE))) { + if (properties.get(attribute.getName()) == null && isArrayOfPrimitives) { return new ArrayList<>(); } From 4f1289b53e3b4c4c762d5f0bdd447a6de0c58818 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 02:23:09 +0530 Subject: [PATCH 181/241] better approach --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index cc22257f13..40cdac6a2f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1052,7 +1052,8 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt edgeProperties.stream() .map(e -> { AtlasJanusEdge edge = (AtlasJanusEdge) e; - return edge.getLabel().toString().substring(2); + String edgeLabel = edge.getLabel(); + return edgeLabel.replaceFirst(EDGE_LABEL_PREFIX , StringUtils.EMPTY); }) .collect(Collectors.toList()); From 74858aa62f3d573d0533560172e51a97c40c364d Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 03:13:06 +0530 Subject: [PATCH 182/241] fallback always --- .../main/java/org/apache/atlas/AtlasConfiguration.java | 4 ++++ .../repository/store/graph/v2/EntityGraphRetriever.java | 8 +++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 60d933d6ae..b9c3c338c2 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -118,6 +118,10 @@ public enum AtlasConfiguration { ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), + /*** + * This configuration is used to enable fetching non primitive attributes in index search + */ + ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", true), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 40cdac6a2f..bc0f7f5291 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -97,6 +97,7 @@ import java.util.concurrent.*; import java.util.stream.Collectors; +import static org.apache.atlas.AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CREATED_BY; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_DESCRIPTION; @@ -1943,11 +1944,8 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } // value is present as marker, fetch the value from the vertex - if (properties.get(attribute.getName()) != null) { - AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); - Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); - RequestContext.get().endMetricRecord(nonPrimitiveAttributes); - return mappedVertex; + if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { + return mapVertexToAttribute(vertex, attribute, null, false); } return null; From d237e9aa0481855c45016ce1250d6b2daa631d40 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 09:42:21 +0530 Subject: [PATCH 183/241] add log to track non null attributes --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index bc0f7f5291..81e7044ce4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1945,7 +1945,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute // value is present as marker, fetch the value from the vertex if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { - return mapVertexToAttribute(vertex, attribute, null, false); + AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); + Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); + LOG.debug("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); + RequestContext.get().endMetricRecord(nonPrimitiveAttributes); + return mappedVertex; } return null; From d1e72e0e2df9bd2d2700812541fecd2c91a8df8b Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 10:26:05 +0530 Subject: [PATCH 184/241] add log to understand relationship labels --- .../repository/store/graph/v2/EntityGraphRetriever.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 81e7044ce4..2aa13381f7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1054,6 +1054,10 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt .map(e -> { AtlasJanusEdge edge = (AtlasJanusEdge) e; String edgeLabel = edge.getLabel(); + // check if edgeLabel contains __ as prefix + if(!edgeLabel.contains(EDGE_LABEL_PREFIX)) { + LOG.debug("Edge label {} does not contain prefix", edgeLabel); + } return edgeLabel.replaceFirst(EDGE_LABEL_PREFIX , StringUtils.EMPTY); }) .collect(Collectors.toList()); @@ -1938,6 +1942,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return properties.get(attribute.getName()); } + //when value is not present and type is primitive, return null + if(properties.get(attribute.getName()) == null && typeCategory.equals(TypeCategory.PRIMITIVE)) { + return null; + } + // if value is empty && element is array of primitives, return empty list if (properties.get(attribute.getName()) == null && isArrayOfPrimitives) { return new ArrayList<>(); From 837d50785ffd61affb30b31f45fafc4c658567bc Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 10:40:45 +0530 Subject: [PATCH 185/241] use prefetch final for map,struct and array of primitives --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 2aa13381f7..407493ab95 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1935,15 +1935,15 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); + boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives - if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && - (typeCategory.equals(TypeCategory.PRIMITIVE) || isArrayOfPrimitives)) { + if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { return properties.get(attribute.getName()); } //when value is not present and type is primitive, return null - if(properties.get(attribute.getName()) == null && typeCategory.equals(TypeCategory.PRIMITIVE)) { + if(properties.get(attribute.getName()) == null && isPrefetchValueFinal) { return null; } From 5c5bb2d3d293d03cc11bf18fd7e0a32bfa70e1b0 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 11:20:12 +0530 Subject: [PATCH 186/241] fallback on relationship links only --- .../store/graph/v2/EntityGraphRetriever.java | 57 ++----------------- 1 file changed, 6 insertions(+), 51 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 407493ab95..88bc1348b0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1019,52 +1019,6 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt // Execute the traversal to fetch properties GraphTraversal> traversal = graph.V(entityVertex.getId()).properties(); - // Check if any attribute is a struct or object type for edge lookup - boolean isAnyAttributeAStructOrObject = attributes.stream().anyMatch(a -> { - if (entityType == null) { - return false; - } - AtlasAttribute attribute = entityType.getAttribute(a); - if (attribute == null || attribute.getAttributeType() == null) { - return false; - } - - if ((attribute.getAttributeType() instanceof AtlasStructType)) { - return true; - } - - if (!(attribute.getAttributeType() instanceof AtlasArrayType)) { - return false; - } - AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); - - if (arrayType.getElementType() == null) { - return false; - } - - AtlasType arrayElementType = arrayType.getElementType(); - - return arrayElementType.getTypeCategory() == TypeCategory.STRUCT; - }); - - if (isAnyAttributeAStructOrObject) { - List edgeProperties = IteratorUtils.toList(entityVertex.getEdges(AtlasEdgeDirection.OUT).iterator()); - List edgeLabels = - edgeProperties.stream() - .map(e -> { - AtlasJanusEdge edge = (AtlasJanusEdge) e; - String edgeLabel = edge.getLabel(); - // check if edgeLabel contains __ as prefix - if(!edgeLabel.contains(EDGE_LABEL_PREFIX)) { - LOG.debug("Edge label {} does not contain prefix", edgeLabel); - } - return edgeLabel.replaceFirst(EDGE_LABEL_PREFIX , StringUtils.EMPTY); - }) - .collect(Collectors.toList()); - - edgeLabels.stream().forEach(e -> propertiesMap.put(e, StringUtils.SPACE)); - } - // Iterate through the resulting VertexProperty objects while (traversal.hasNext()) { try { @@ -1292,6 +1246,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e attribute = entityType.getAttribute(attrName); if (attribute == null) { + // dataContractLatest, meanings, links attribute = entityType.getRelationshipAttribute(attrName, null); } } @@ -1954,11 +1909,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute // value is present as marker, fetch the value from the vertex if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { - AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); - Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); - LOG.debug("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); - RequestContext.get().endMetricRecord(nonPrimitiveAttributes); - return mappedVertex; + //AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); + return mapVertexToAttribute(vertex, attribute, null, false); + //LOG.debug("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); + //RequestContext.get().endMetricRecord(nonPrimitiveAttributes); + //return mappedVertex; } return null; From f0163ebdb418cabdf3f54c4c53962be90442a1fc Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 6 Dec 2024 12:31:24 +0530 Subject: [PATCH 187/241] Handler service type for tag policies --- .../CachePolicyTransformerImpl.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index db33577349..47c9ea21c2 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -160,8 +160,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map 0 ? new Date(lastAuditEventTime) : new Date(); -// Date policyUpdateTime = new Date(); + Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { @@ -171,29 +170,35 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map policiesDelta = getRangerPolicyDelta(service, policyChanges); - servicePolicies.setPolicyDeltas(policiesDelta); + ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); + List allAtlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); + List atlasServicePolicies = allAtlasPolicies.stream().filter(x -> serviceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList()); + List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasServicePolicies); - //Process tag based policies + // Process tag based policies String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); if (StringUtils.isNotEmpty(tagServiceName)) { AtlasEntityHeader tagService = getServiceEntity(tagServiceName); - if (tagService != null) { TagPolicies tagPolicies = new TagPolicies(); - tagPolicies.setServiceName(tagServiceName); tagPolicies.setPolicyUpdateTime(new Date()); tagPolicies.setServiceId(tagService.getGuid()); tagPolicies.setPolicyVersion(-1L); + String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); - servicePolicies.setTagPolicies(tagPolicies); + + List tagServicePolicies = allAtlasPolicies.stream().filter(x -> tagServiceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList()); + List tagPoliciesDelta = getRangerPolicyDelta(tagService, policyChanges, tagServicePolicies); + policiesDelta.addAll(tagPoliciesDelta); } } - LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); + servicePolicies.setPolicyDeltas(policiesDelta); + + LOG.info("PolicyDelta: {}: Found total delta={}", serviceName, policiesDelta.size()); } } catch (Exception e) { @@ -287,7 +292,7 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges) throws AtlasBaseException, IOException { + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getRangerPolicyDelta." + serviceName); @@ -298,7 +303,7 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, } ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); - List atlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); +// List atlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { @@ -340,7 +345,7 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, } public Map createPolicyChangeMap(String serviceName, List events) { - Map policyChanges = new HashMap<>(); + Map policyChanges = new HashMap<>(); for (EntityAuditEventV2 event : events) { if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policyChanges.containsKey(event.getEntityId())) { policyChanges.put(event.getEntityId(), event.getAction()); From f41b7c8a77c289da42f6b4e4d791ec5306c563a3 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 6 Dec 2024 13:57:11 +0530 Subject: [PATCH 188/241] Remove commented code --- .../atlas/policytransformer/CachePolicyTransformerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 47c9ea21c2..87df320336 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -303,7 +303,6 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, } ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); -// List atlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { From 62c4f9712397f5733abe666505da16fbcc52999b Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 6 Dec 2024 14:22:30 +0530 Subject: [PATCH 189/241] preload properties for relatedvertex --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 88bc1348b0..78666f4cdc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -310,12 +310,16 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE Map attributes = new HashMap<>(); Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); + + // preloadProperties here if (CollectionUtils.isNotEmpty(relationAttributes)) { + Map referenceVertexProperties = preloadProperties(entityVertex, entityType, Collections.emptySet()); + for (String attributeName : relationAttributes) { AtlasAttribute attribute = entityType.getAttribute(attributeName); if (attribute != null && !uniqueAttributes.containsKey(attributeName)) { - Object attrValue = getVertexAttribute(entityVertex, attribute); + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, referenceVertexProperties); if (attrValue != null) { attributes.put(attribute.getName(), attrValue); } From e403fb62ac3f29318adad62df51ba7c57c39423e Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 6 Dec 2024 14:38:49 +0530 Subject: [PATCH 190/241] push flag --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 8db3701806..49ef869cf2 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -117,8 +117,12 @@ public enum AtlasConfiguration { ENABLE_ASYNC_INDEXSEARCH("atlas.indexsearch.async.enable", false), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), - ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), + /** + * hits elastic search async API + */ + ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), + ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), /*** * This configuration is used to enable fetching non primitive attributes in index search */ From b92aa4b60f54795e488fb9fbca5e2c4cb9a2f3bb Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 6 Dec 2024 14:40:59 +0530 Subject: [PATCH 191/241] enable janus optimization --- .../main/java/org/apache/atlas/AtlasConfiguration.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 49ef869cf2..2bd7eefbef 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -114,14 +114,18 @@ public enum AtlasConfiguration { HERACLES_API_SERVER_URL("atlas.heracles.api.service.url", "http://heracles-service.heracles.svc.cluster.local"), INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), + + /** + * hits elastic search async API + */ ENABLE_ASYNC_INDEXSEARCH("atlas.indexsearch.async.enable", false), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), + ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), /** - * hits elastic search async API + * enables janus/cassandra optimization */ - ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), /*** * This configuration is used to enable fetching non primitive attributes in index search From ee18260f464ad1092ca6583295ef972bb67b5e4e Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 6 Dec 2024 15:22:34 +0530 Subject: [PATCH 192/241] fix build --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 1 + 1 file changed, 1 insertion(+) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 2bd7eefbef..dbcacbf4ab 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -121,6 +121,7 @@ public enum AtlasConfiguration { ENABLE_ASYNC_INDEXSEARCH("atlas.indexsearch.async.enable", false), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), + FETCH_COLLAPSED_RESULT("atlas.indexsearch.fetch.collapsed.result", true), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), /** From e48975a2f4899b338ca69b44e15d660111f85b90 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 6 Dec 2024 15:52:28 +0530 Subject: [PATCH 193/241] mesh-294: migration API to migrate unique qn attribute --- .../instance/UniqueQnMigrationRequest.java | 66 +++++++++++++++ .../v2/UniqueQNAttributeMigrationService.java | 84 +++++++++++++++++++ .../apache/atlas/web/rest/MigrationREST.java | 26 ++++++ 3 files changed, 176 insertions(+) create mode 100644 intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java diff --git a/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java new file mode 100644 index 0000000000..df263eb761 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +/** + * Request to link/unlink policies from asset. + */ +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class UniqueQnMigrationRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private Set assetGuids; + + + public Set getAssetGuids() { + return assetGuids; + } + + public void setAssetGuids(Set assetGuids) { + this.assetGuids = assetGuids; + } + + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("LinkBusinessPolicyRequest{"); + sb.append("assetGuids=").append(assetGuids); + sb.append('}'); + return sb.toString(); + } +} + diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java new file mode 100644 index 0000000000..47e61b2b38 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java @@ -0,0 +1,84 @@ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public class UniqueQNAttributeMigrationService { + + private static final Logger LOG = LoggerFactory.getLogger(UniqueQNAttributeMigrationService.class); + + private final EntityGraphRetriever entityRetriever; + + + private Set entityGuids; + private final TransactionInterceptHelper transactionInterceptHelper; + + private final String QUALIFIED_NAME_ATTR = "qualifiedName"; + private final String UNIQUE_QUALIFIED_NAME_ATTR = "__u_qualifiedName"; + + public UniqueQNAttributeMigrationService(EntityGraphRetriever entityRetriever, Set entityGuids, TransactionInterceptHelper transactionInterceptHelper) { + this.entityRetriever = entityRetriever; + this.transactionInterceptHelper = transactionInterceptHelper; + this.entityGuids = entityGuids; + } + + public void migrateQN() throws Exception { + try { + for (String entityGuid : entityGuids) { + AtlasVertex entityVertex = entityRetriever.getEntityVertex(entityGuid); + + if (entityVertex == null) { + LOG.error("Entity vertex not found for guid: {}", entityGuid); + continue; + } + + boolean isCommitRequired = migrateuniqueQnAttr(entityVertex); + if (isCommitRequired){ + LOG.info("Committing changes for entity: {}", entityGuid); + commitChanges(); + } + else { + LOG.info("No changes to commit for entity: {} as no migration needed", entityGuid); + } + } + + } catch (Exception e) { + LOG.error("Error while migration unique qualifiedName attribute for entities: {}", entityGuids, e); + throw e; + } + } + + private boolean migrateuniqueQnAttr(AtlasVertex vertex) throws AtlasBaseException { + try{ + boolean isCommitRequired = false; + + String qualifiedName = vertex.getProperty(QUALIFIED_NAME_ATTR, String.class); + String uniqueQualifiedName = vertex.getProperty(UNIQUE_QUALIFIED_NAME_ATTR, String.class); + + if(!qualifiedName.equals(uniqueQualifiedName)) { + vertex.setProperty(UNIQUE_QUALIFIED_NAME_ATTR, qualifiedName); + isCommitRequired = true; + } + return isCommitRequired; + }catch (Exception e) { + LOG.error("Failed to migrate unique qualifiedName attribute for entity: ", e); + throw e; + } + } + + public void commitChanges() throws AtlasBaseException { + try { + transactionInterceptHelper.intercept(); + LOG.info("Committed a entity to the graph"); + } catch (Exception e){ + LOG.error("Failed to commit asset: ", e); + throw e; + } + } +} \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index 551d0f4aa2..a480793cf1 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -8,6 +8,7 @@ import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.UniqueQnMigrationRequest; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.store.graph.AtlasEntityStore; @@ -325,6 +326,31 @@ public List searchForType(@PathParam("typeName") String typeName, @ } } + @POST + @Path("update-unique-qualified-name") + @Timed + public Boolean updateUniqueQualifiedName(final UniqueQnMigrationRequest request) throws Exception { + AtlasPerfTracer perf = null; + try { + if (CollectionUtils.isEmpty(request.getAssetGuids())) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Asset GUIDs are required for which updating unique qualified name is required"); + } + + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.updateUniqueQualifiedName(" + request.getAssetGuids() + ")"); + } + + UniqueQNAttributeMigrationService migrationService = new UniqueQNAttributeMigrationService(entityRetriever, request.getAssetGuids(), transactionInterceptHelper); + migrationService.migrateQN(); + } catch (Exception e) { + LOG.error("Error while updating unique qualified name for guids: {}", request.getAssetGuids(), e); + throw e; + } finally { + AtlasPerfTracer.log(perf); + } + return Boolean.TRUE; + } + private List getEntitiesByIndexSearch(IndexSearchParams indexSearchParams, Boolean minExtInfo, boolean ignoreRelationships) throws AtlasBaseException { List entities = new ArrayList<>(); String indexName = "janusgraph_vertex_index"; From 9ee56451b57e044af219aea2203b7543fae7bffa Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sat, 7 Dec 2024 12:17:49 +0530 Subject: [PATCH 194/241] use properties method with janusgraph than gremlin --- .../org/apache/atlas/AtlasConfiguration.java | 20 +- .../discovery/EntityDiscoveryService.java | 101 +--- .../atlas/repository/graph/GraphHelper.java | 18 + .../store/graph/v2/EntityGraphRetriever.java | 481 +++++++++--------- 4 files changed, 279 insertions(+), 341 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 6735cb4c8a..98642093e1 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -113,25 +113,15 @@ public enum AtlasConfiguration { HERACLES_API_SERVER_URL("atlas.heracles.api.service.url", "http://heracles-service.heracles.svc.cluster.local"), INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), - - /** - * hits elastic search async API - */ ENABLE_ASYNC_INDEXSEARCH("atlas.indexsearch.async.enable", false), - - /*** - * enables parallel processing of janus graph vertices from cassandra - */ - ENABLE_JANUS_GRAPH_OPTIMISATION("atlas.janus.graph.optimisation.enable", true), - - /** - * No. of threads to be spawned for parallel processing - */ - THREADS_TO_BE_SPAWNED("atlas.janus.graph.optimisation.thread_count", (Runtime.getRuntime().availableProcessors())/2), - FETCH_COLLAPSED_RESULT("atlas.indexsearch.fetch.collapsed.result", true), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), + ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", true), + /*** + * This configuration is used to enable fetching non primitive attributes in index search + */ + ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", true), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 29a33d676b..a8f039e4fd 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -70,10 +70,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptException; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; -import java.util.function.Function; import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.*; @@ -1006,7 +1002,7 @@ public AtlasSearchResult directIndexSearch(SearchParams searchParams) throws Atl return null; } RequestContext.get().endMetricRecord(elasticSearchQueryMetric); - prepareSearchResult(ret, indexQueryResult, resultAttributes, AtlasConfiguration.FETCH_COLLAPSED_RESULT.getBoolean()); + prepareSearchResult(ret, indexQueryResult, resultAttributes, true); ret.setAggregations(indexQueryResult.getAggregationMap()); ret.setApproximateCount(indexQuery.vertexTotals()); @@ -1075,88 +1071,7 @@ public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) thro } } - @SuppressWarnings("rawtypes") - private void prepareSearchResultAsync(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { - SearchParams searchParams = ret.getSearchParameters(); - boolean showSearchScore = searchParams.getShowSearchScore(); - List results = new ArrayList<>(); - - // Collect results for batch processing - Iterator iterator = indexQueryResult.getIterator(); - while (iterator != null && iterator.hasNext()) { - results.add(iterator.next()); - } - - // Batch fetch vertices - List vertices = results.stream() - .map(Result::getVertex) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - // Use ConcurrentHashMap for thread-safe access - //ConcurrentHashMap headers = new ConcurrentHashMap<>(); - //ConcurrentHashMap entitiesSet = new ConcurrentHashMap<>(); - - List headers = entityRetriever.mapVerticesToAtlasEntityHeader(vertices, resultAttributes); - // Create a Set entitiesSet = headers.stream().collect(Collectors.toMap(AtlasEntityHeader::getGuid, Function.identity())); - - // Run vertex processing in limited parallel threads - /**CompletableFuture.runAsync(() -> vertices.parallelStream().forEach(vertex -> { - String guid = vertex.getProperty("__guid", String.class); - headers.computeIfAbsent(guid, k -> { - try { - //AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex, resultAttributes); - if (RequestContext.get().includeClassifications()) { - header.setClassifications(entityRetriever.getAllClassifications(vertex)); - } - return header; - } catch (AtlasBaseException e) { - throw new RuntimeException("Failed to process vertex with GUID: " + guid, e); - } - }); - }), CUSTOMTHREADPOOL).join();*/ - - // Process results and handle collapse in parallel - results.parallelStream().forEach(result -> { - AtlasVertex vertex = result.getVertex(); - if (vertex == null) return; - - String guid = vertex.getProperty("__guid", String.class); - AtlasEntityHeader header = entitiesSet.get(guid); - - if (showSearchScore) { - ret.addEntityScore(header.getGuid(), result.getScore()); - } - - if (fetchCollapsedResults) { - Map collapse; - try { - collapse = processCollapseResults(result, searchParams, resultAttributes); - } catch (AtlasBaseException e) { - throw new RuntimeException(e); - } - if (!collapse.isEmpty()) { - header.setCollapse(collapse); - } - } - - if (searchParams.getShowSearchMetadata()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); - ret.addSort(header.getGuid(), result.getSort()); - } else if (searchParams.getShowHighlights()) { - ret.addHighlights(header.getGuid(), result.getHighLights()); - } - - if (header != null) { - entitiesSet.put(header.getGuid(), header); - } - }); - ret.setEntities(new ArrayList<>(entitiesSet.values())); - scrubSearchResults(ret, searchParams.getSuppressLogs()); - } - - private void prepareSearchResultSync(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { + private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean collapseResults) throws AtlasBaseException { SearchParams searchParams = ret.getSearchParameters(); try { if(LOG.isDebugEnabled()){ @@ -1184,7 +1099,7 @@ private void prepareSearchResultSync(AtlasSearchResult ret, DirectIndexQueryResu if (showSearchScore) { ret.addEntityScore(header.getGuid(), result.getScore()); } - if (fetchCollapsedResults) { + if (collapseResults) { Map collapse = new HashMap<>(); Set collapseKeys = result.getCollapseKeys(); @@ -1206,7 +1121,7 @@ private void prepareSearchResultSync(AtlasSearchResult ret, DirectIndexQueryResu DirectIndexQueryResult indexQueryCollapsedResult = result.getCollapseVertices(collapseKey); collapseRet.setApproximateCount(indexQueryCollapsedResult.getApproximateCount()); - prepareSearchResultSync(collapseRet, indexQueryCollapsedResult, collapseResultAttributes, false); + prepareSearchResult(collapseRet, indexQueryCollapsedResult, collapseResultAttributes, false); collapseRet.setSearchParameters(null); collapse.put(collapseKey, collapseRet); @@ -1230,14 +1145,6 @@ private void prepareSearchResultSync(AtlasSearchResult ret, DirectIndexQueryResu scrubSearchResults(ret, searchParams.getSuppressLogs()); } - private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult indexQueryResult, Set resultAttributes, boolean fetchCollapsedResults) throws AtlasBaseException { - if (AtlasConfiguration.ENABLE_JANUS_GRAPH_OPTIMISATION.getBoolean()) { - LOG.debug("enabled janusGraphOptimisation"); - prepareSearchResultAsync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); - } else { - prepareSearchResultSync(ret, indexQueryResult, resultAttributes, fetchCollapsedResults); - } - } // Non-recursive collapse processing private Map processCollapseResults(Result result, SearchParams searchParams, Set resultAttributes) throws AtlasBaseException { Map collapse = new HashMap<>(); diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 6e3f5c90ed..5da3dca267 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -1080,6 +1080,10 @@ public static Boolean isEntityIncomplete(AtlasElement element) { return ret; } + public static Boolean isEntityIncomplete(Integer value) { + return value != null && value.equals(INCOMPLETE_ENTITY_VALUE) ? Boolean.TRUE : Boolean.FALSE; + } + public static Boolean getEntityHasLineage(AtlasElement element) { if (element.getPropertyKeys().contains(HAS_LINEAGE)) { return element.getProperty(HAS_LINEAGE, Boolean.class); @@ -1107,10 +1111,24 @@ public static Map getCustomAttributes(AtlasElement element) { return ret; } + public static Map getCustomAttributes(String customAttrsString) { + Map ret = null; + + if (customAttrsString != null) { + ret = AtlasType.fromJson(customAttrsString, Map.class); + } + + return ret; + } + public static Set getLabels(AtlasElement element) { return parseLabelsString(element.getProperty(LABELS_PROPERTY_KEY, String.class)); } + public static Set getLabels(String labels) { + return parseLabelsString(labels); + } + public static Integer getProvenanceType(AtlasElement element) { return element.getProperty(Constants.PROVENANCE_TYPE_KEY, Integer.class); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index d23401d570..80cc8b187d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -49,8 +49,9 @@ import org.apache.atlas.repository.graphdb.AtlasElement; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; +import org.apache.atlas.repository.graphdb.janus.AtlasJanusEdge; import org.apache.atlas.repository.graphdb.janus.AtlasJanusVertex; +import org.apache.atlas.repository.util.AccessControlUtils; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; import org.apache.atlas.type.AtlasEntityType; @@ -67,9 +68,12 @@ import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.IteratorUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; -import org.janusgraph.core.*; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -77,25 +81,11 @@ import javax.inject.Inject; import java.math.BigDecimal; import java.math.BigInteger; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Queue; -import java.util.Set; +import java.util.*; import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; +import static org.apache.atlas.AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CREATED_BY; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_DESCRIPTION; @@ -298,23 +288,25 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE if (entityType != null) { Map uniqueAttributes = new HashMap<>(); + Map attributes = new HashMap<>(); + Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); + // preloadProperties here + Map preloadProperties = preloadProperties(entityVertex, entityType, relationAttributes); for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) { - Object attrValue = getVertexAttribute(entityVertex, attribute); + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, preloadProperties); if (attrValue != null) { uniqueAttributes.put(attribute.getName(), attrValue); } } - Map attributes = new HashMap<>(); - Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); if (CollectionUtils.isNotEmpty(relationAttributes)) { for (String attributeName : relationAttributes) { AtlasAttribute attribute = entityType.getAttribute(attributeName); if (attribute != null && !uniqueAttributes.containsKey(attributeName)) { - Object attrValue = getVertexAttribute(entityVertex, attribute); + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, preloadProperties); if (attrValue != null) { attributes.put(attribute.getName(), attrValue); } @@ -1012,7 +1004,120 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } + private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes) { + Map propertiesMap = new HashMap<>(); + String guid = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); + // Execute the traversal to fetch properties + Iterator> traversal = ((AtlasJanusVertex)entityVertex).getWrappedElement().properties(); + + // Check if any attribute is a struct or object type for edge lookup + boolean isAnyAttributeAStructOrObject = attributes.stream().anyMatch(a -> { + + AtlasAttribute attribute = entityType.getAttribute(a); + if (attribute == null || attribute.getAttributeType() == null) { + return false; + } + + if ((attribute.getAttributeType() instanceof AtlasStructType)) { + return true; + } + + if (!(attribute.getAttributeType() instanceof AtlasArrayType)) { + return false; + } + AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); + + if (arrayType.getElementType() == null) { + return false; + } + + AtlasType arrayElementType = arrayType.getElementType(); + + return arrayElementType.getTypeCategory() == TypeCategory.STRUCT; + }); + + if (isAnyAttributeAStructOrObject) { + Set relationshipLabels = attributes.stream().map(attr -> entityType.getAttribute(attr)).filter(Objects::nonNull).filter(ele -> TypeCategory.ARRAY.equals(ele.getAttributeType().getTypeCategory())).map(ele -> AtlasGraphUtilsV2.getEdgeLabel(ele.getName())).collect(Collectors.toSet()); + relationshipLabels.addAll(attributes.stream().map(attr -> entityType.getRelationshipAttributes().getOrDefault(attr, Collections.emptyMap()).values().stream().filter(ele1 -> ele1.getName().equalsIgnoreCase(attr)).filter(Objects::nonNull).map(AtlasAttribute::getRelationshipEdgeLabel).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toSet())); + List edgeProperties = IteratorUtils.toList(entityVertex.getEdges(AtlasEdgeDirection.OUT, relationshipLabels.toArray(new String[0])).iterator()); + List edgeLabelsDebug = edgeProperties.stream().map(AtlasEdge::getLabel).collect(Collectors.toList()); + LOG.info("Edge labels for entityVertex: {}, is : {}", guid, edgeLabelsDebug); + Set edgeLabels = + edgeLabelsDebug.stream() + .map(edgeLabel -> { + Optional matchingAttrOpt = attributes.stream().filter(ele -> edgeLabel.contains(ele)).findFirst(); + return matchingAttrOpt.orElse(null); + }).filter(Objects::nonNull) + .collect(Collectors.toSet()); + + edgeLabels.stream().forEach(e -> propertiesMap.put(e, StringUtils.SPACE)); + } + + // Iterate through the resulting VertexProperty objects + while (traversal.hasNext()) { + try { + VertexProperty property = traversal.next(); + + AtlasAttribute attribute = entityType.getAttribute(property.key()); + TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; + TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + + if (property.isPresent()) { + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { + Object value = propertiesMap.get(property.key()); + if (value instanceof List) { + ((List) value).add(property.value()); + } else { + List values = new ArrayList<>(); + values.add(property.value()); + propertiesMap.put(property.key(), values); + } + } else { + if (propertiesMap.get(property.key()) == null) { + propertiesMap.put(property.key(), property.value()); + } + } + } + } catch (RuntimeException e) { + LOG.error("Error preloading properties for entity vertex: {}", entityVertex.getId(), e); + throw e; // Re-throw the exception after logging it + } + } + LOG.info("Preloaded properties for entity vertex: {}, {}", guid, propertiesMap); + return propertiesMap; + } + + + private boolean isPolicyAttribute(Set attributes) { + Set exclusionSet = new HashSet<>(Arrays.asList(AccessControlUtils.ATTR_POLICY_TYPE, + AccessControlUtils.ATTR_POLICY_USERS, + AccessControlUtils.ATTR_POLICY_GROUPS, + AccessControlUtils.ATTR_POLICY_ROLES, + AccessControlUtils.ATTR_POLICY_ACTIONS, + AccessControlUtils.ATTR_POLICY_CATEGORY, + AccessControlUtils.ATTR_POLICY_SUB_CATEGORY, + AccessControlUtils.ATTR_POLICY_RESOURCES, + AccessControlUtils.ATTR_POLICY_IS_ENABLED, + AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY, + AccessControlUtils.ATTR_POLICY_SERVICE_NAME, + AccessControlUtils.ATTR_POLICY_PRIORITY, + AccessControlUtils.REL_ATTR_POLICIES)); + + return exclusionSet.stream().anyMatch(attributes::contains); + } + private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { + boolean shouldPrefetch = !isPolicyAttribute(attributes) + && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); + + if (shouldPrefetch) { + return mapVertexToAtlasEntityHeaderWithPrefetch(entityVertex, attributes); + } else { + return mapVertexToAtlasEntityHeaderWithoutPrefetch(entityVertex, attributes); + } + } + + private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithoutPrefetch(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); AtlasEntityHeader ret = new AtlasEntityHeader(); try { @@ -1096,230 +1201,105 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, return ret; } - public List mapVerticesToAtlasEntityHeader(List entityVertices, Set attributes) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVerticesToAtlasEntityHeader"); - - List results = new ArrayList<>(); + private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeaderWithPrefetch"); + AtlasEntityHeader ret = new AtlasEntityHeader(); try { - // Convert AtlasVertex to JanusGraphVertex - - // Use multiQuery for optimized property fetching - AtlasJanusGraph janusGraph = ((AtlasJanusGraph)graph); - Map> multiQueryResults = getBatchPropertiesWithMultiQuery(entityVertices, attributes, janusGraph); - Map map = getJanusGraphVerticesMap(entityVertices); - multiQueryResults.forEach((janusGraphVertex, vertexProperties) -> { - AtlasEntityHeader ret = new AtlasEntityHeader(); + //pre-fetching the properties + String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); //properties.get returns null + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); // this is not costly + Map properties = preloadProperties(entityVertex, entityType, attributes); - // Populate AtlasEntityHeader with fetched properties - try { - populateEntityHeader(ret, map.get(janusGraphVertex), vertexProperties, attributes); - } catch (AtlasBaseException e) { - throw new RuntimeException(e); - } - results.add(ret); - }); - - } finally { - RequestContext.get().endMetricRecord(metricRecorder); - } + String guid = (String) properties.get(Constants.GUID_PROPERTY_KEY); - return results; - } + Integer value = (Integer)properties.get(Constants.IS_INCOMPLETE_PROPERTY_KEY); + Boolean isIncomplete = value != null && value.equals(INCOMPLETE_ENTITY_VALUE) ? Boolean.TRUE : Boolean.FALSE; - private Map getJanusGraphVerticesMap(List vertices) { - Map resultMap = new HashMap<>(); + ret.setTypeName(typeName); + ret.setGuid(guid); - for (AtlasVertex vertex : vertices) { - if (vertex instanceof AtlasJanusVertex) { - Object wrappedElement = ((AtlasJanusVertex) vertex).getWrappedElement(); + String state = (String)properties.get(Constants.STATE_PROPERTY_KEY); + Id.EntityState entityState = state == null ? null : Id.EntityState.valueOf(state); + ret.setStatus((entityState == Id.EntityState.DELETED) ? AtlasEntity.Status.DELETED : AtlasEntity.Status.ACTIVE); - if (wrappedElement instanceof JanusGraphVertex) { - resultMap.put((JanusGraphVertex) wrappedElement, vertex); - } else { - throw new IllegalArgumentException("Wrapped element is not an instance of JanusGraphVertex"); - } - } else { - throw new IllegalArgumentException("Provided vertex is not an instance of AtlasJanusVertex"); + RequestContext context = RequestContext.get(); + boolean includeClassifications = context.includeClassifications(); + boolean includeClassificationNames = context.isIncludeClassificationNames(); + if(includeClassifications){ + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } else if (!includeClassifications && includeClassificationNames) { + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); } - } + ret.setIsIncomplete(isIncomplete); + ret.setLabels(getLabels((String)properties.get(Constants.LABELS_PROPERTY_KEY))); - return resultMap; - } + ret.setCreatedBy(properties.get(CREATED_BY_KEY) != null ? (String) properties.get(CREATED_BY_KEY) : null); + ret.setUpdatedBy(properties.get(MODIFIED_BY_KEY) != null ? (String) properties.get(MODIFIED_BY_KEY) : null); + ret.setCreateTime(properties.get(TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)properties.get(TIMESTAMP_PROPERTY_KEY)) : null); + ret.setUpdateTime(properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY)) : null); - // Helper to convert AtlasVertex to JanusGraphVertex - private List getJanusGraphVertices(List vertices) { - List results = new ArrayList<>(); - for(AtlasVertex vertex : vertices) { - if (((AtlasJanusVertex) vertex).getWrappedElement() instanceof JanusGraphVertex) { - results.add(vertex.getWrappedElement()); - } else { - throw new IllegalArgumentException("Provided vertex is not an instance of JanusGraphVertex"); + if(RequestContext.get().includeMeanings()) { + List termAssignmentHeaders = mapAssignedTerms(entityVertex); + ret.setMeanings(termAssignmentHeaders); + ret.setMeaningNames( + termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) + .collect(Collectors.toList())); } - } - return results; - } - - public Map> getBatchPropertiesWithMultiQuery( - List entityVertices, Set attributes, AtlasJanusGraph graph) { - - Map> result = new ConcurrentHashMap<>(); - List vertices = getJanusGraphVertices(entityVertices); - // Split vertices into batches and process them in parallel - AtomicInteger counter = new AtomicInteger(0); - StreamSupport.stream(vertices.spliterator(), false) - .collect(Collectors.groupingBy(v -> counter.getAndIncrement() / BATCH_SIZE)) - .values() - .parallelStream() - .forEach(batch -> { - Map> batchResult = processBatch(batch, attributes, graph); - result.putAll(batchResult); - }); - - return result; - } + if (entityType != null) { + for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { + Object attrValue = getVertexAttributePreFetchCache(entityVertex, headerAttribute, properties); - private Map> processBatch( - List batch, Set attributes, AtlasJanusGraph graph) { + if (attrValue != null) { + ret.setAttribute(headerAttribute.getName(), attrValue); + } + } - JanusGraphTransaction transaction = graph.getTransaction(); - try { - JanusGraphMultiVertexQuery multiQuery = transaction.multiQuery(batch); - - // Add all required keys to fetch - Set keys = new HashSet<>(Arrays.asList( - Constants.TYPE_NAME_PROPERTY_KEY, Constants.GUID_PROPERTY_KEY, - Constants.CREATED_BY_KEY, Constants.MODIFIED_BY_KEY, - Constants.TIMESTAMP_PROPERTY_KEY, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY - )); - keys.addAll(attributes); - - // Set the keys in the multiQuery - multiQuery.keys(keys.toArray(new String[0])); - - // Fetch properties in bulk - Map> results = multiQuery.properties(); - - // Convert results to the desired map format - Map> vertexPropertiesMap = new HashMap<>(); - for (Map.Entry> entry : results.entrySet()) { - JanusGraphVertex vertex = entry.getKey(); - Iterable properties = entry.getValue(); - - Map propertiesMap = new HashMap<>(); - for (JanusGraphProperty property : properties) { - propertiesMap.put(property.key(), property.value()); + if(properties.get(NAME) != null){ + ret.setDisplayText(properties.get(NAME).toString()); + } else if(properties.get(DISPLAY_NAME) != null) { + ret.setDisplayText(properties.get(DISPLAY_NAME).toString()); + } else if(properties.get(QUALIFIED_NAME) != null) { + ret.setDisplayText(properties.get(QUALIFIED_NAME).toString()); } - vertexPropertiesMap.put(vertex, propertiesMap); - } - return vertexPropertiesMap; - } finally { - if (transaction != null) { - transaction.commit(); - transaction.close(); - } - } - } - // Populate AtlasEntityHeader - private void populateEntityHeader(AtlasEntityHeader ret, AtlasVertex entityVertex, Map vertexProperties, Set attributes) throws AtlasBaseException { - String typeName = (String) vertexProperties.get(Constants.TYPE_NAME_PROPERTY_KEY); - String guid = (String) vertexProperties.get(Constants.GUID_PROPERTY_KEY); - String createdBy = (String) vertexProperties.get(Constants.CREATED_BY_KEY); - String updatedBy = (String) vertexProperties.get(Constants.MODIFIED_BY_KEY); - Long createTime = (Long) vertexProperties.get(Constants.TIMESTAMP_PROPERTY_KEY); - Long updateTime = (Long) vertexProperties.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); - Boolean isIncomplete = isEntityIncomplete(entityVertex); - ret.setTypeName(typeName); - ret.setGuid(guid); - ret.setStatus(GraphHelper.getStatus(entityVertex)); - ret.setIsIncomplete(isIncomplete); - ret.setCreatedBy(createdBy); - ret.setUpdatedBy(updatedBy); - ret.setCreateTime(createTime != null ? new Date(createTime) : null); - ret.setUpdateTime(updateTime != null ? new Date(updateTime) : null); - ret.setLabels(getLabels(entityVertex)); + //attributes = only the attributes of entityType + if (CollectionUtils.isNotEmpty(attributes)) { + for (String attrName : attributes) { + AtlasAttribute attribute = entityType.getAttribute(attrName); - // Classifications - RequestContext context = RequestContext.get(); - if (context.includeClassifications() || context.isIncludeClassificationNames()) { - ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); - } + if (attribute == null) { + attrName = toNonQualifiedName(attrName); - // Meanings - if (context.includeMeanings()) { - List termAssignmentHeaders = mapAssignedTerms(entityVertex); - ret.setMeanings(termAssignmentHeaders); - ret.setMeaningNames(termAssignmentHeaders.stream() - .map(AtlasTermAssignmentHeader::getDisplayText) - .collect(Collectors.toList())); - } + if (ret.hasAttribute(attrName)) { + continue; + } - // Process entity type and attributes - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); - if (entityType != null) { - // Header attributes - for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { - Object attrValue = getVertexAttributeFromBatch(vertexProperties, headerAttribute); - if (attrValue != null) { - ret.setAttribute(headerAttribute.getName(), attrValue); - } - } + attribute = entityType.getAttribute(attrName); - // Display text - Object displayText = getDisplayText(entityVertex, entityType); - if (displayText != null) { - ret.setDisplayText(displayText.toString()); - } + if (attribute == null) { + // dataContractLatest, meanings, links + attribute = entityType.getRelationshipAttribute(attrName, null); + } + } + + //this is a call to cassandra + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, properties); //use prefetch cache - // Additional attributes - if (CollectionUtils.isNotEmpty(attributes)) { - for (String attrName : attributes) { - AtlasAttribute attribute = getEntityOrRelationshipAttribute(entityType, attrName); - if (attribute != null) { - Object attrValue = getVertexAttributeFromBatch(vertexProperties, attribute); if (attrValue != null) { ret.setAttribute(attrName, attrValue); } } } } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); } - - // Additional properties like classifications, meanings, and attributes... - } - - /** - * Retrieves a vertex attribute from the pre-fetched batch of properties. - */ - private Object getVertexAttributeFromBatch(Map properties, AtlasAttribute attribute) { - if (properties == null || attribute == null) { - return null; - } - - String propertyKey = attribute.getVertexPropertyName(); - return properties.get(propertyKey); + return ret; } - /** - * Retrieves an entity or relationship attribute from the entity type. - */ - private AtlasAttribute getEntityOrRelationshipAttribute(AtlasEntityType entityType, String attrName) { - AtlasAttribute attribute = entityType.getAttribute(attrName); - - if (attribute == null) { - attrName = toNonQualifiedName(attrName); - - attribute = entityType.getAttribute(attrName); - if (attribute == null) { - attribute = entityType.getRelationshipAttribute(attrName, null); - } - } - - return attribute; - } private String toNonQualifiedName(String attrName) { String ret; if (attrName.contains(".")) { @@ -1340,25 +1320,33 @@ public AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity ent try { if (entityVertex != null) { - entity.setGuid(getGuid(entityVertex)); - entity.setTypeName(getTypeName(entityVertex)); - entity.setStatus(GraphHelper.getStatus(entityVertex)); - entity.setVersion(GraphHelper.getVersion(entityVertex)); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + Map preloadProperties = preloadProperties(entityVertex, entityType, Collections.emptySet()); + String guid = (String) preloadProperties.get(Constants.GUID_PROPERTY_KEY); + entity.setGuid(guid); + entity.setTypeName(preloadProperties.get(Constants.TYPE_NAME_PROPERTY_KEY).toString()); + String state = (String)preloadProperties.get(Constants.STATE_PROPERTY_KEY); + Id.EntityState entityState = state == null ? null : Id.EntityState.valueOf(state); + entity.setStatus((entityState == Id.EntityState.DELETED) ? AtlasEntity.Status.DELETED : AtlasEntity.Status.ACTIVE); + entity.setVersion(preloadProperties.get(Constants.VERSION_PROPERTY_KEY) != null ? (Long)preloadProperties.get(Constants.VERSION_PROPERTY_KEY) : 0); + + entity.setCreatedBy(preloadProperties.get(CREATED_BY_KEY) != null ? (String) preloadProperties.get(CREATED_BY_KEY) : null); + entity.setUpdatedBy(preloadProperties.get(MODIFIED_BY_KEY) != null ? (String) preloadProperties.get(MODIFIED_BY_KEY) : null); + entity.setCreateTime(preloadProperties.get(TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)preloadProperties.get(TIMESTAMP_PROPERTY_KEY)) : null); + entity.setUpdateTime(preloadProperties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)preloadProperties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY)) : null); - entity.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); - entity.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); + entity.setHomeId(preloadProperties.get(HOME_ID_KEY).toString()); - entity.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); - entity.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); + entity.setIsProxy(preloadProperties.get(IS_PROXY_KEY) != null ? (Boolean)preloadProperties.get(IS_PROXY_KEY) : false); - entity.setHomeId(GraphHelper.getHomeId(entityVertex)); + Integer value = (Integer)preloadProperties.get(Constants.IS_INCOMPLETE_PROPERTY_KEY); + Boolean isIncomplete = value != null && value.equals(INCOMPLETE_ENTITY_VALUE) ? Boolean.TRUE : Boolean.FALSE; - entity.setIsProxy(GraphHelper.isProxy(entityVertex)); - entity.setIsIncomplete(isEntityIncomplete(entityVertex)); + entity.setIsIncomplete(isIncomplete); - entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex)); - entity.setCustomAttributes(getCustomAttributes(entityVertex)); - entity.setLabels(getLabels(entityVertex)); + entity.setProvenanceType(preloadProperties.get(PROVENANCE_TYPE_KEY) != null ? (Integer)preloadProperties.get(PROVENANCE_TYPE_KEY) : null); + entity.setCustomAttributes(getCustomAttributes(preloadProperties.get(CUSTOM_ATTRIBUTES_PROPERTY_KEY)!=null ? (String)preloadProperties.get(CUSTOM_ATTRIBUTES_PROPERTY_KEY) : null)); + entity.setLabels(getLabels(preloadProperties.get(LABELS_PROPERTY_KEY).toString())); entity.setPendingTasks(getPendingTasks(entityVertex)); } } catch (Throwable t) { @@ -1936,6 +1924,41 @@ public Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) t return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null; } + public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute attribute, Map properties) throws AtlasBaseException { + if (vertex == null || attribute == null) { + return null; + } + + TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); + TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); + boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); + + // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives + if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { + return properties.get(attribute.getName()); + } + + //when value is not present and type is primitive, return null + if(properties.get(attribute.getName()) == null && isPrefetchValueFinal) { + return null; + } + + // if value is empty && element is array of primitives, return empty list + if (properties.get(attribute.getName()) == null && typeCategory.equals(TypeCategory.ARRAY)) { + return new ArrayList<>(); + } + + // value is present as marker, fetch the value from the vertex + if (properties.get(attribute.getName()) == StringUtils.SPACE || TypeCategory.OBJECT_ID_TYPE.equals(typeCategory)) { + Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); + LOG.info("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); + return mappedVertex; + } + + return null; + } + private Object getVertexAttributeIgnoreInactive(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException { return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false, true, true) : null; } From c8578e43ff727cc5bf78823ead7592a231a73f33 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sat, 7 Dec 2024 12:41:25 +0530 Subject: [PATCH 195/241] use edge fetch always --- .../store/graph/v2/EntityGraphRetriever.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 80cc8b187d..e8a55fad97 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1036,22 +1036,20 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt return arrayElementType.getTypeCategory() == TypeCategory.STRUCT; }); - if (isAnyAttributeAStructOrObject) { - Set relationshipLabels = attributes.stream().map(attr -> entityType.getAttribute(attr)).filter(Objects::nonNull).filter(ele -> TypeCategory.ARRAY.equals(ele.getAttributeType().getTypeCategory())).map(ele -> AtlasGraphUtilsV2.getEdgeLabel(ele.getName())).collect(Collectors.toSet()); - relationshipLabels.addAll(attributes.stream().map(attr -> entityType.getRelationshipAttributes().getOrDefault(attr, Collections.emptyMap()).values().stream().filter(ele1 -> ele1.getName().equalsIgnoreCase(attr)).filter(Objects::nonNull).map(AtlasAttribute::getRelationshipEdgeLabel).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toSet())); - List edgeProperties = IteratorUtils.toList(entityVertex.getEdges(AtlasEdgeDirection.OUT, relationshipLabels.toArray(new String[0])).iterator()); - List edgeLabelsDebug = edgeProperties.stream().map(AtlasEdge::getLabel).collect(Collectors.toList()); - LOG.info("Edge labels for entityVertex: {}, is : {}", guid, edgeLabelsDebug); - Set edgeLabels = - edgeLabelsDebug.stream() - .map(edgeLabel -> { - Optional matchingAttrOpt = attributes.stream().filter(ele -> edgeLabel.contains(ele)).findFirst(); - return matchingAttrOpt.orElse(null); - }).filter(Objects::nonNull) - .collect(Collectors.toSet()); - - edgeLabels.stream().forEach(e -> propertiesMap.put(e, StringUtils.SPACE)); - } + Set relationshipLabels = attributes.stream().map(attr -> entityType.getAttribute(attr)).filter(Objects::nonNull).filter(ele -> TypeCategory.ARRAY.equals(ele.getAttributeType().getTypeCategory())).map(ele -> AtlasGraphUtilsV2.getEdgeLabel(ele.getName())).collect(Collectors.toSet()); + relationshipLabels.addAll(attributes.stream().map(attr -> entityType.getRelationshipAttributes().getOrDefault(attr, Collections.emptyMap()).values().stream().filter(ele1 -> ele1.getName().equalsIgnoreCase(attr)).filter(Objects::nonNull).map(AtlasAttribute::getRelationshipEdgeLabel).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toSet())); + List edgeProperties = IteratorUtils.toList(entityVertex.getEdges(AtlasEdgeDirection.OUT, relationshipLabels.toArray(new String[0])).iterator()); + List edgeLabelsDebug = edgeProperties.stream().map(AtlasEdge::getLabel).collect(Collectors.toList()); + LOG.info("Edge labels for entityVertex: {}, is : {}", guid, edgeLabelsDebug); + Set edgeLabels = + edgeLabelsDebug.stream() + .map(edgeLabel -> { + Optional matchingAttrOpt = attributes.stream().filter(ele -> edgeLabel.contains(ele)).findFirst(); + return matchingAttrOpt.orElse(null); + }).filter(Objects::nonNull) + .collect(Collectors.toSet()); + + edgeLabels.stream().forEach(e -> propertiesMap.put(e, StringUtils.SPACE)); // Iterate through the resulting VertexProperty objects while (traversal.hasNext()) { From a47bdeeb599b63dbded719971509bd5f823faa63 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sat, 7 Dec 2024 14:24:43 +0530 Subject: [PATCH 196/241] log edge and edge properties --- .../store/graph/v2/EntityGraphRetriever.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index e8a55fad97..6379167c03 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -72,6 +72,8 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Property; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.slf4j.Logger; @@ -1036,10 +1038,22 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt return arrayElementType.getTypeCategory() == TypeCategory.STRUCT; }); - Set relationshipLabels = attributes.stream().map(attr -> entityType.getAttribute(attr)).filter(Objects::nonNull).filter(ele -> TypeCategory.ARRAY.equals(ele.getAttributeType().getTypeCategory())).map(ele -> AtlasGraphUtilsV2.getEdgeLabel(ele.getName())).collect(Collectors.toSet()); + // Fetch edges in both directions + Iterator edges = entityVertex.getEdges(AtlasEdgeDirection.BOTH).iterator(); + List edgeLabelsDebug = new ArrayList<>(); + while (edges.hasNext()) { + AtlasJanusEdge edge = edges.next(); + LOG.info("Edge Label: {}", edge.getLabel()); + edgeLabelsDebug.add(edge.getLabel()); + // Fetch edge properties + Set edgeProperties = edge.getPropertyKeys(); + LOG.info("Edge Properties: {}", edgeProperties); + } + + /*Set relationshipLabels = attributes.stream().map(attr -> entityType.getAttribute(attr)).filter(Objects::nonNull).filter(ele -> TypeCategory.ARRAY.equals(ele.getAttributeType().getTypeCategory())).map(ele -> AtlasGraphUtilsV2.getEdgeLabel(ele.getName())).collect(Collectors.toSet()); relationshipLabels.addAll(attributes.stream().map(attr -> entityType.getRelationshipAttributes().getOrDefault(attr, Collections.emptyMap()).values().stream().filter(ele1 -> ele1.getName().equalsIgnoreCase(attr)).filter(Objects::nonNull).map(AtlasAttribute::getRelationshipEdgeLabel).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toSet())); List edgeProperties = IteratorUtils.toList(entityVertex.getEdges(AtlasEdgeDirection.OUT, relationshipLabels.toArray(new String[0])).iterator()); - List edgeLabelsDebug = edgeProperties.stream().map(AtlasEdge::getLabel).collect(Collectors.toList()); + List edgeLabelsDebug = edgeProperties.stream().map(AtlasEdge::getLabel).collect(Collectors.toList());*/ LOG.info("Edge labels for entityVertex: {}, is : {}", guid, edgeLabelsDebug); Set edgeLabels = edgeLabelsDebug.stream() @@ -1943,12 +1957,12 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } // if value is empty && element is array of primitives, return empty list - if (properties.get(attribute.getName()) == null && typeCategory.equals(TypeCategory.ARRAY)) { + if (properties.get(attribute.getName()) == null && isArrayOfPrimitives) { return new ArrayList<>(); } // value is present as marker, fetch the value from the vertex - if (properties.get(attribute.getName()) == StringUtils.SPACE || TypeCategory.OBJECT_ID_TYPE.equals(typeCategory)) { + if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); LOG.info("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); return mappedVertex; From 4dd2743f26af282f7fee8bce5481304d67210c3c Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sat, 7 Dec 2024 15:26:42 +0530 Subject: [PATCH 197/241] lookup only when marker exists or when attribute relationship is inwards --- .../store/graph/v2/EntityGraphRetriever.java | 38 +------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 6379167c03..770a1eaa7e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1012,48 +1012,14 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt // Execute the traversal to fetch properties Iterator> traversal = ((AtlasJanusVertex)entityVertex).getWrappedElement().properties(); - // Check if any attribute is a struct or object type for edge lookup - boolean isAnyAttributeAStructOrObject = attributes.stream().anyMatch(a -> { - - AtlasAttribute attribute = entityType.getAttribute(a); - if (attribute == null || attribute.getAttributeType() == null) { - return false; - } - - if ((attribute.getAttributeType() instanceof AtlasStructType)) { - return true; - } - - if (!(attribute.getAttributeType() instanceof AtlasArrayType)) { - return false; - } - AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); - - if (arrayType.getElementType() == null) { - return false; - } - - AtlasType arrayElementType = arrayType.getElementType(); - - return arrayElementType.getTypeCategory() == TypeCategory.STRUCT; - }); - // Fetch edges in both directions Iterator edges = entityVertex.getEdges(AtlasEdgeDirection.BOTH).iterator(); List edgeLabelsDebug = new ArrayList<>(); while (edges.hasNext()) { AtlasJanusEdge edge = edges.next(); - LOG.info("Edge Label: {}", edge.getLabel()); edgeLabelsDebug.add(edge.getLabel()); - // Fetch edge properties - Set edgeProperties = edge.getPropertyKeys(); - LOG.info("Edge Properties: {}", edgeProperties); } - /*Set relationshipLabels = attributes.stream().map(attr -> entityType.getAttribute(attr)).filter(Objects::nonNull).filter(ele -> TypeCategory.ARRAY.equals(ele.getAttributeType().getTypeCategory())).map(ele -> AtlasGraphUtilsV2.getEdgeLabel(ele.getName())).collect(Collectors.toSet()); - relationshipLabels.addAll(attributes.stream().map(attr -> entityType.getRelationshipAttributes().getOrDefault(attr, Collections.emptyMap()).values().stream().filter(ele1 -> ele1.getName().equalsIgnoreCase(attr)).filter(Objects::nonNull).map(AtlasAttribute::getRelationshipEdgeLabel).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toSet())); - List edgeProperties = IteratorUtils.toList(entityVertex.getEdges(AtlasEdgeDirection.OUT, relationshipLabels.toArray(new String[0])).iterator()); - List edgeLabelsDebug = edgeProperties.stream().map(AtlasEdge::getLabel).collect(Collectors.toList());*/ LOG.info("Edge labels for entityVertex: {}, is : {}", guid, edgeLabelsDebug); Set edgeLabels = edgeLabelsDebug.stream() @@ -1957,12 +1923,12 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } // if value is empty && element is array of primitives, return empty list - if (properties.get(attribute.getName()) == null && isArrayOfPrimitives) { + if (properties.get(attribute.getName()) == null && typeCategory.equals(TypeCategory.ARRAY) && !AtlasRelationshipEdgeDirection.IN.equals(attribute.getRelationshipEdgeDirection())) { return new ArrayList<>(); } // value is present as marker, fetch the value from the vertex - if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { + if (properties.get(attribute.getName()) == StringUtils.SPACE || AtlasRelationshipEdgeDirection.IN.equals(attribute.getRelationshipEdgeDirection())) { Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); LOG.info("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); return mappedVertex; From 76fe1c86c9902da3ab9c643d67880a69d9e13f17 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sat, 7 Dec 2024 17:31:58 +0530 Subject: [PATCH 198/241] include all policy attributes in prefetch exclude --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 +++++- .../apache/atlas/repository/util/AccessControlUtils.java | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 770a1eaa7e..036728b56a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1079,7 +1079,11 @@ private boolean isPolicyAttribute(Set attributes) { AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY, AccessControlUtils.ATTR_POLICY_SERVICE_NAME, AccessControlUtils.ATTR_POLICY_PRIORITY, - AccessControlUtils.REL_ATTR_POLICIES)); + AccessControlUtils.REL_ATTR_POLICIES, + AccessControlUtils.ATTR_SERVICE_SERVICE_TYPE, + AccessControlUtils.ATTR_SERVICE_TAG_SERVICE, + AccessControlUtils.ATTR_SERVICE_IS_ENABLED, + AccessControlUtils.ATTR_SERVICE_LAST_SYNC)); return exclusionSet.stream().anyMatch(attributes::contains); } diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index 68ee5318ed..7812af2cc4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -59,6 +59,11 @@ public final class AccessControlUtils { public static final String ATTR_PERSONA_USERS = "personaUsers"; public static final String ATTR_PERSONA_GROUPS = "personaGroups"; + public static final String ATTR_SERVICE_SERVICE_TYPE = "authServiceType"; + public static final String ATTR_SERVICE_TAG_SERVICE = "tagService"; + public static final String ATTR_SERVICE_IS_ENABLED = "authServiceIsEnabled"; + public static final String ATTR_SERVICE_LAST_SYNC = "authServicePolicyLastSync"; + public static final String ATTR_PURPOSE_CLASSIFICATIONS = "purposeClassifications"; public static final String ATTR_POLICY_TYPE = "policyType"; From 358a7bcc4088748c62501bce31bac6c54b559253 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sat, 7 Dec 2024 17:59:52 +0530 Subject: [PATCH 199/241] revert using prefetch in mapSystemAttributes and toAtlasToObjectId --- .../store/graph/v2/EntityGraphRetriever.java | 50 +++++++------------ 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 036728b56a..0a9394c971 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -68,13 +68,8 @@ import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.IteratorUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; -import org.apache.tinkerpop.gremlin.structure.Edge; -import org.apache.tinkerpop.gremlin.structure.Property; -import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,7 +82,6 @@ import java.util.concurrent.*; import java.util.stream.Collectors; -import static org.apache.atlas.AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CREATED_BY; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_DESCRIPTION; @@ -292,11 +286,9 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE Map uniqueAttributes = new HashMap<>(); Map attributes = new HashMap<>(); Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); - // preloadProperties here - Map preloadProperties = preloadProperties(entityVertex, entityType, relationAttributes); for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) { - Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, preloadProperties); + Object attrValue = getVertexAttribute(entityVertex, attribute); if (attrValue != null) { uniqueAttributes.put(attribute.getName(), attrValue); @@ -308,7 +300,7 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE AtlasAttribute attribute = entityType.getAttribute(attributeName); if (attribute != null && !uniqueAttributes.containsKey(attributeName)) { - Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, preloadProperties); + Object attrValue = getVertexAttribute(entityVertex, attribute); if (attrValue != null) { attributes.put(attribute.getName(), attrValue); } @@ -1036,7 +1028,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt try { VertexProperty property = traversal.next(); - AtlasAttribute attribute = entityType.getAttribute(property.key()); + AtlasAttribute attribute = entityType.getAttribute(property.key()) != null ? entityType.getAttribute(property.key()) : null; TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; @@ -1302,33 +1294,25 @@ public AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity ent try { if (entityVertex != null) { - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); - Map preloadProperties = preloadProperties(entityVertex, entityType, Collections.emptySet()); - String guid = (String) preloadProperties.get(Constants.GUID_PROPERTY_KEY); - entity.setGuid(guid); - entity.setTypeName(preloadProperties.get(Constants.TYPE_NAME_PROPERTY_KEY).toString()); - String state = (String)preloadProperties.get(Constants.STATE_PROPERTY_KEY); - Id.EntityState entityState = state == null ? null : Id.EntityState.valueOf(state); - entity.setStatus((entityState == Id.EntityState.DELETED) ? AtlasEntity.Status.DELETED : AtlasEntity.Status.ACTIVE); - entity.setVersion(preloadProperties.get(Constants.VERSION_PROPERTY_KEY) != null ? (Long)preloadProperties.get(Constants.VERSION_PROPERTY_KEY) : 0); - - entity.setCreatedBy(preloadProperties.get(CREATED_BY_KEY) != null ? (String) preloadProperties.get(CREATED_BY_KEY) : null); - entity.setUpdatedBy(preloadProperties.get(MODIFIED_BY_KEY) != null ? (String) preloadProperties.get(MODIFIED_BY_KEY) : null); - entity.setCreateTime(preloadProperties.get(TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)preloadProperties.get(TIMESTAMP_PROPERTY_KEY)) : null); - entity.setUpdateTime(preloadProperties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)preloadProperties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY)) : null); + entity.setGuid(getGuid(entityVertex)); + entity.setTypeName(getTypeName(entityVertex)); + entity.setStatus(GraphHelper.getStatus(entityVertex)); + entity.setVersion(GraphHelper.getVersion(entityVertex)); - entity.setHomeId(preloadProperties.get(HOME_ID_KEY).toString()); + entity.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); + entity.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); - entity.setIsProxy(preloadProperties.get(IS_PROXY_KEY) != null ? (Boolean)preloadProperties.get(IS_PROXY_KEY) : false); + entity.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); + entity.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); - Integer value = (Integer)preloadProperties.get(Constants.IS_INCOMPLETE_PROPERTY_KEY); - Boolean isIncomplete = value != null && value.equals(INCOMPLETE_ENTITY_VALUE) ? Boolean.TRUE : Boolean.FALSE; + entity.setHomeId(GraphHelper.getHomeId(entityVertex)); - entity.setIsIncomplete(isIncomplete); + entity.setIsProxy(GraphHelper.isProxy(entityVertex)); + entity.setIsIncomplete(isEntityIncomplete(entityVertex)); - entity.setProvenanceType(preloadProperties.get(PROVENANCE_TYPE_KEY) != null ? (Integer)preloadProperties.get(PROVENANCE_TYPE_KEY) : null); - entity.setCustomAttributes(getCustomAttributes(preloadProperties.get(CUSTOM_ATTRIBUTES_PROPERTY_KEY)!=null ? (String)preloadProperties.get(CUSTOM_ATTRIBUTES_PROPERTY_KEY) : null)); - entity.setLabels(getLabels(preloadProperties.get(LABELS_PROPERTY_KEY).toString())); + entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex)); + entity.setCustomAttributes(getCustomAttributes(entityVertex)); + entity.setLabels(getLabels(entityVertex)); entity.setPendingTasks(getPendingTasks(entityVertex)); } } catch (Throwable t) { From 5c6ad1bc790f85eeab16760fbc258c62c348b71a Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Sat, 7 Dec 2024 19:30:23 +0530 Subject: [PATCH 200/241] remove logs. update comments --- .../store/graph/v2/EntityGraphRetriever.java | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 0a9394c971..e3adc2c31b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -119,7 +119,6 @@ @Component public class EntityGraphRetriever { private static final Logger LOG = LoggerFactory.getLogger(EntityGraphRetriever.class); - private static final int BATCH_SIZE = 300; private static final String GLOSSARY_TERM_DISPLAY_NAME_ATTR = "name"; public static final String TERM_RELATION_NAME = "AtlasGlossarySemanticAssignment"; @@ -527,7 +526,7 @@ public Map> getClassificationPropagatedEntitiesMa List entitiesPropagatingTo = getImpactedVerticesV2(sourceEntityVertex, relationshipGuidToExclude, classificationId, CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode),toExclude); - LOG.info("Traversed {} vertices for Classification vertex id {} excluding RelationShip GUID {}", entitiesPropagatingTo.size(), classificationId, relationshipGuidToExclude); + LOG.debug("Traversed {} vertices for Classification vertex id {} excluding RelationShip GUID {}", entitiesPropagatingTo.size(), classificationId, relationshipGuidToExclude); ret.put(classificationVertex, entitiesPropagatingTo); } @@ -661,7 +660,7 @@ private void traverseImpactedVertices(final AtlasVertex entityVertexStart, final String entityVertexId = entityVertex.getIdForDisplay(); if (visitedVertices.contains(entityVertexId)) { - LOG.info("Already visited: {}", entityVertexId); + LOG.debug("Already visited: {}", entityVertexId); continue; } @@ -1012,7 +1011,6 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt edgeLabelsDebug.add(edge.getLabel()); } - LOG.info("Edge labels for entityVertex: {}, is : {}", guid, edgeLabelsDebug); Set edgeLabels = edgeLabelsDebug.stream() .map(edgeLabel -> { @@ -1053,11 +1051,9 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt throw e; // Re-throw the exception after logging it } } - LOG.info("Preloaded properties for entity vertex: {}, {}", guid, propertiesMap); return propertiesMap; } - private boolean isPolicyAttribute(Set attributes) { Set exclusionSet = new HashSet<>(Arrays.asList(AccessControlUtils.ATTR_POLICY_TYPE, AccessControlUtils.ATTR_POLICY_USERS, @@ -1910,16 +1906,14 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return null; } - // if value is empty && element is array of primitives, return empty list + // if value is empty && element is array and not inward relation, return empty list if (properties.get(attribute.getName()) == null && typeCategory.equals(TypeCategory.ARRAY) && !AtlasRelationshipEdgeDirection.IN.equals(attribute.getRelationshipEdgeDirection())) { return new ArrayList<>(); } - // value is present as marker, fetch the value from the vertex + // value is present as marker or is inward relation, fetch the value from the vertex if (properties.get(attribute.getName()) == StringUtils.SPACE || AtlasRelationshipEdgeDirection.IN.equals(attribute.getRelationshipEdgeDirection())) { - Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); - LOG.info("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); - return mappedVertex; + return mapVertexToAttribute(vertex, attribute, null, false); } return null; From bf345392949e582f7e5f6dc4e5c03aefa10844d0 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 9 Dec 2024 10:50:40 +0530 Subject: [PATCH 201/241] remove check for maps --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 78666f4cdc..4f6ec87933 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1894,7 +1894,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); - boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); + boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || isArrayOfPrimitives); // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { From f6c9bde646f2314b59b8c04138f46e884705811f Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 9 Dec 2024 14:22:55 +0530 Subject: [PATCH 202/241] fix business attributes --- .../store/graph/v2/EntityGraphRetriever.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 4f6ec87933..91628b6cd5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1031,9 +1031,10 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()); TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + boolean isBusinessAttribute = attribute == null; if (property.isPresent()) { - if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { + if ((typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE)) { Object value = propertiesMap.get(property.key()); if (value instanceof List) { ((List) value).add(property.value()); @@ -1045,6 +1046,12 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt } else { if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value()); + } else if (isBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute + LOG.info("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); + List values = new ArrayList<>(); + values.add(propertiesMap.get(property.key())); + values.add(property.value()); + propertiesMap.put(property.key(), values); } } } @@ -1891,10 +1898,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return null; } + TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); - boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || isArrayOfPrimitives); + boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { From 664501fda8d6d200d4e66b3321fec57d09e394f6 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 9 Dec 2024 14:25:58 +0530 Subject: [PATCH 203/241] fix log level --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 91628b6cd5..d1b2aef6be 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1034,7 +1034,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt boolean isBusinessAttribute = attribute == null; if (property.isPresent()) { - if ((typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE)) { + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { Object value = propertiesMap.get(property.key()); if (value instanceof List) { ((List) value).add(property.value()); @@ -1047,7 +1047,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value()); } else if (isBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute - LOG.info("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); + LOG.warn("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); List values = new ArrayList<>(); values.add(propertiesMap.get(property.key())); values.add(property.value()); From dce45ff80046eafa15788bd8bfbb499f5b652f3f Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:57:52 +0530 Subject: [PATCH 204/241] feat: add resources to tag service --- .../atlas-servicedef-atlas_tag.json | 450 +++++++++++++++++- 1 file changed, 448 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json index 98ff10877f..cba65d8f6b 100644 --- a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json +++ b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json @@ -29,11 +29,457 @@ "uiHint":"{ \"singleValue\":true }", "label": "TAG", "description": "TAG" + }, + { + "itemId": 2, + "name": "type-category", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Type Catagory", + "description": "Type Catagory" + }, + { + "itemId": 3, + "name": "type", + "type": "string", + "level": 20, + "mandatory": true, + "parent": "type-category", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Type Name", + "description": "Type Name", + "accessTypeRestrictions": ["type-read" ,"type-create", "type-update", "type-delete" ] + }, + { + "itemId": 4, + "name": "entity-type", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Entity Type", + "description": "Entity Type" + }, + { + "itemId": 5, + "name": "entity-classification", + "type": "string", + "level": 20, + "mandatory": true, + "parent": "entity-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Entity Classification", + "description": "Entity Classification" + }, + { + "itemId": 6, + "name": "entity", + "type": "string", + "level": 30, + "mandatory": true, + "parent": "entity-classification", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Entity ID", + "description": "Entity ID", + "accessTypeRestrictions": ["entity-read", "entity-create", "entity-update", "entity-delete"] + }, + { + "itemId": 7, + "name": "atlas-service", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Atlas Service", + "description": "Atlas Service", + "accessTypeRestrictions": ["admin-import", "admin-export", "admin-purge", "admin-audits", "admin-entity-audits", "admin-repair-index", "admin-task-cud"] + }, + { + "itemId": 8, + "name": "relationship-type", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Relationship Type", + "description": "Relationship Type" + }, + { + "itemId": 9, + "name": "end-one-entity-type", + "type": "string", + "level": 20, + "mandatory": true, + "parent": "relationship-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End1 Entity Type", + "description": "End1 Entity Type" + }, + { + "itemId": 10, + "name": "end-one-entity-classification", + "type": "string", + "level": 30, + "mandatory": true, + "parent": "end-one-entity-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End1 Entity Classification", + "description": "End1 Entity Classification" + }, + { + "itemId": 11, + "name": "end-one-entity", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "end-one-entity-classification", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "End1 Entity ID", + "description": "End1 Entity ID" + }, + { + "itemId": 12, + "name": "end-two-entity-type", + "type": "string", + "level": 50, + "mandatory": true, + "parent": "end-one-entity", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End2 Entity Type", + "description": "End2 Entity Type" + }, + { + "itemId": 13, + "name": "end-two-entity-classification", + "type": "string", + "level": 60, + "mandatory": true, + "parent": "end-two-entity-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End2 Entity Classification", + "description": "End2 Entity Classification" + }, + { + "itemId": 14, + "name": "end-two-entity", + "type": "string", + "level": 70, + "mandatory": true, + "parent": "end-two-entity-classification", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "End2 Entity ID", + "description": "End2 Entity ID", + "accessTypeRestrictions": [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + }, + { + "itemId": 15, + "name": "entity-label", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "entity", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Label", + "description": "Label", + "accessTypeRestrictions": [ + "entity-add-label", + "entity-remove-label" + ] + }, + { + "itemId": 16, + "name": "entity-business-metadata", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "entity", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Business Metadata", + "description": "Business Metadata", + "accessTypeRestrictions": [ + "entity-update-business-metadata" + ] + }, + { + "itemId": 17, + "name": "classification", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "entity", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Targetted classifications", + "description": "Targetted classifications", + "accessTypeRestrictions": [ + "entity-add-classification", + "entity-update-classification", + "entity-remove-classification" + ] } ], - "accessTypes": - [ + "accessTypes": [ + { + "itemId": 1, + "name": "type-create", + "label": "Create Type", + "impliedGrants": + [ + "type-read" + ] + }, + { + "itemId": 2, + "name": "type-update", + "label": "Update Type", + "impliedGrants": + [ + "type-read" + ] + }, + { + "itemId": 3, + "name": "type-delete", + "label": "Delete Type", + "impliedGrants": + [ + "type-read" + ] + }, + { + "itemId": 4, + "name": "entity-read", + "label": "Read Entity" + }, + { + "itemId": 5, + "name": "entity-create", + "label": "Create Entity" + }, + { + "itemId": 6, + "name": "entity-update", + "label": "Update Entity" + }, + { + "itemId": 7, + "name": "entity-delete", + "label": "Delete Entity" + }, + { + "itemId": 8, + "name": "entity-add-classification", + "label": "Add Classification" + }, + { + "itemId": 9, + "name": "entity-update-classification", + "label": "Update Classification" + }, + { + "itemId": 10, + "name": "entity-remove-classification", + "label": "Remove Classification" + }, + { + "itemId": 11, + "name": "admin-export", + "label": "Admin Export" + }, + { + "itemId": 12, + "name": "admin-import", + "label": "Admin Import" + }, + { + "itemId": 13, + "name": "add-relationship", + "label": "Add Relationship" + }, + { + "itemId": 14, + "name": "update-relationship", + "label": "Update Relationship" + }, + { + "itemId": 15, + "name": "remove-relationship", + "label": "Remove Relationship" + }, + { + "itemId": 16, + "name": "admin-purge", + "label": "Admin Purge" + }, + { + "itemId": 17, + "name": "entity-add-label", + "label": "Add Label" + }, + { + "itemId": 18, + "name": "entity-remove-label", + "label": "Remove Label" + }, + { + "itemId": 19, + "name": "entity-update-business-metadata", + "label": "Update Business Metadata" + }, + { + "itemId": 20, + "name": "type-read", + "label": "Read Type" + }, + { + "itemId": 21, + "name": "admin-audits", + "label": "Admin Audits" + }, + { + "itemId": 22, + "name": "admin-entity-audits", + "label": "Admin Entity Audits" + }, + { + "itemId": 23, + "name": "admin-repair-index", + "label": "Admin Repair Index" + }, + { + "itemId": 24, + "name": "admin-task-cud", + "label": "Admin task CUD API" + }, + { + "itemId": 25, + "name": "admin-featureFlag-cud", + "label": "Admin featureflag CUD API" + } ], From 87568edfee9559207a13038c790cccc66facfb85 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 9 Dec 2024 17:30:10 +0530 Subject: [PATCH 205/241] update branch --- .github/workflows/maven.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index e290252038..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - janus0.6 jobs: build: From 8da3eac719a3ed3e956a8b84319397ad9ed4c596 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Mon, 9 Dec 2024 18:00:37 +0530 Subject: [PATCH 206/241] fix: Filter deleted relations in List API traversal --- .../org/apache/atlas/discovery/EntityLineageService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 38fb2535dc..73446f49c3 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -534,15 +534,15 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line private void enqueueNeighbours(AtlasVertex currentVertex, boolean isDataset, AtlasLineageListContext lineageListContext, Queue traversalQueue, Set visitedVertices, Set skippedVertices, - Map> lineageParentsForEntityMap, Map> lineageChildrenForEntityMap) { + Map> lineageParentsForEntityMap, Map> lineageChildrenForEntityMap) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdges = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdges"); Iterator edges; if (isDataset) - edges = currentVertex.getEdges(IN, isInputDirection(lineageListContext) ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator(); + edges = GraphHelper.getActiveEdges(currentVertex, isInputDirection(lineageListContext) ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE, IN); else - edges = currentVertex.getEdges(OUT, isInputDirection(lineageListContext) ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator(); - RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdges); + edges = GraphHelper.getActiveEdges(currentVertex, isInputDirection(lineageListContext) ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE, OUT); + RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdges); while (edges.hasNext()) { AtlasEdge currentEdge = edges.next(); if (!lineageListContext.evaluateTraversalFilter(currentEdge)) From 9d71641cdf724087c63e4d18790cc734ae46fe0b Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:21:41 +0530 Subject: [PATCH 207/241] Revert "feat: add resources to tag service" This reverts commit dce45ff80046eafa15788bd8bfbb499f5b652f3f. --- .../atlas-servicedef-atlas_tag.json | 450 +----------------- 1 file changed, 2 insertions(+), 448 deletions(-) diff --git a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json index cba65d8f6b..98ff10877f 100644 --- a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json +++ b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json @@ -29,457 +29,11 @@ "uiHint":"{ \"singleValue\":true }", "label": "TAG", "description": "TAG" - }, - { - "itemId": 2, - "name": "type-category", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Type Catagory", - "description": "Type Catagory" - }, - { - "itemId": 3, - "name": "type", - "type": "string", - "level": 20, - "mandatory": true, - "parent": "type-category", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Type Name", - "description": "Type Name", - "accessTypeRestrictions": ["type-read" ,"type-create", "type-update", "type-delete" ] - }, - { - "itemId": 4, - "name": "entity-type", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Entity Type", - "description": "Entity Type" - }, - { - "itemId": 5, - "name": "entity-classification", - "type": "string", - "level": 20, - "mandatory": true, - "parent": "entity-type", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Entity Classification", - "description": "Entity Classification" - }, - { - "itemId": 6, - "name": "entity", - "type": "string", - "level": 30, - "mandatory": true, - "parent": "entity-classification", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Entity ID", - "description": "Entity ID", - "accessTypeRestrictions": ["entity-read", "entity-create", "entity-update", "entity-delete"] - }, - { - "itemId": 7, - "name": "atlas-service", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Atlas Service", - "description": "Atlas Service", - "accessTypeRestrictions": ["admin-import", "admin-export", "admin-purge", "admin-audits", "admin-entity-audits", "admin-repair-index", "admin-task-cud"] - }, - { - "itemId": 8, - "name": "relationship-type", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Relationship Type", - "description": "Relationship Type" - }, - { - "itemId": 9, - "name": "end-one-entity-type", - "type": "string", - "level": 20, - "mandatory": true, - "parent": "relationship-type", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "End1 Entity Type", - "description": "End1 Entity Type" - }, - { - "itemId": 10, - "name": "end-one-entity-classification", - "type": "string", - "level": 30, - "mandatory": true, - "parent": "end-one-entity-type", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "End1 Entity Classification", - "description": "End1 Entity Classification" - }, - { - "itemId": 11, - "name": "end-one-entity", - "type": "string", - "level": 40, - "mandatory": true, - "parent": "end-one-entity-classification", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "End1 Entity ID", - "description": "End1 Entity ID" - }, - { - "itemId": 12, - "name": "end-two-entity-type", - "type": "string", - "level": 50, - "mandatory": true, - "parent": "end-one-entity", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "End2 Entity Type", - "description": "End2 Entity Type" - }, - { - "itemId": 13, - "name": "end-two-entity-classification", - "type": "string", - "level": 60, - "mandatory": true, - "parent": "end-two-entity-type", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "End2 Entity Classification", - "description": "End2 Entity Classification" - }, - { - "itemId": 14, - "name": "end-two-entity", - "type": "string", - "level": 70, - "mandatory": true, - "parent": "end-two-entity-classification", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "End2 Entity ID", - "description": "End2 Entity ID", - "accessTypeRestrictions": [ - "add-relationship", - "update-relationship", - "remove-relationship" - ] - }, - { - "itemId": 15, - "name": "entity-label", - "type": "string", - "level": 40, - "mandatory": true, - "parent": "entity", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Label", - "description": "Label", - "accessTypeRestrictions": [ - "entity-add-label", - "entity-remove-label" - ] - }, - { - "itemId": 16, - "name": "entity-business-metadata", - "type": "string", - "level": 40, - "mandatory": true, - "parent": "entity", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Business Metadata", - "description": "Business Metadata", - "accessTypeRestrictions": [ - "entity-update-business-metadata" - ] - }, - { - "itemId": 17, - "name": "classification", - "type": "string", - "level": 40, - "mandatory": true, - "parent": "entity", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Targetted classifications", - "description": "Targetted classifications", - "accessTypeRestrictions": [ - "entity-add-classification", - "entity-update-classification", - "entity-remove-classification" - ] } ], - "accessTypes": [ - { - "itemId": 1, - "name": "type-create", - "label": "Create Type", - "impliedGrants": - [ - "type-read" - ] - }, - { - "itemId": 2, - "name": "type-update", - "label": "Update Type", - "impliedGrants": - [ - "type-read" - ] - }, - { - "itemId": 3, - "name": "type-delete", - "label": "Delete Type", - "impliedGrants": - [ - "type-read" - ] - }, - { - "itemId": 4, - "name": "entity-read", - "label": "Read Entity" - }, - { - "itemId": 5, - "name": "entity-create", - "label": "Create Entity" - }, - { - "itemId": 6, - "name": "entity-update", - "label": "Update Entity" - }, - { - "itemId": 7, - "name": "entity-delete", - "label": "Delete Entity" - }, - { - "itemId": 8, - "name": "entity-add-classification", - "label": "Add Classification" - }, - { - "itemId": 9, - "name": "entity-update-classification", - "label": "Update Classification" - }, - { - "itemId": 10, - "name": "entity-remove-classification", - "label": "Remove Classification" - }, - { - "itemId": 11, - "name": "admin-export", - "label": "Admin Export" - }, - { - "itemId": 12, - "name": "admin-import", - "label": "Admin Import" - }, - { - "itemId": 13, - "name": "add-relationship", - "label": "Add Relationship" - }, - { - "itemId": 14, - "name": "update-relationship", - "label": "Update Relationship" - }, - { - "itemId": 15, - "name": "remove-relationship", - "label": "Remove Relationship" - }, - { - "itemId": 16, - "name": "admin-purge", - "label": "Admin Purge" - }, - { - "itemId": 17, - "name": "entity-add-label", - "label": "Add Label" - }, - { - "itemId": 18, - "name": "entity-remove-label", - "label": "Remove Label" - }, - { - "itemId": 19, - "name": "entity-update-business-metadata", - "label": "Update Business Metadata" - }, - { - "itemId": 20, - "name": "type-read", - "label": "Read Type" - }, - { - "itemId": 21, - "name": "admin-audits", - "label": "Admin Audits" - }, - { - "itemId": 22, - "name": "admin-entity-audits", - "label": "Admin Entity Audits" - }, - { - "itemId": 23, - "name": "admin-repair-index", - "label": "Admin Repair Index" - }, - { - "itemId": 24, - "name": "admin-task-cud", - "label": "Admin task CUD API" - }, - { - "itemId": 25, - "name": "admin-featureFlag-cud", - "label": "Admin featureflag CUD API" - } + "accessTypes": + [ ], From 8b9f30b89aa13277629a7d874d30f712575941c9 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:22:13 +0530 Subject: [PATCH 208/241] feat: add atlas authZ in case of relationship evaluator --- .../policytransformer/CachePolicyTransformerImpl.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 87df320336..bfab5baa6c 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -82,6 +82,7 @@ import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PURPOSE; import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyCategory; +import static org.apache.atlas.services.tag.RangerServiceTag.TAG_RESOURCE_NAME; @Component public class CachePolicyTransformerImpl { @@ -709,7 +710,14 @@ private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String servi //policy.setId(atlasPolicy.getGuid()); policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME)); policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME)); - policy.setServiceType(serviceType); + + // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation + if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { + policy.setServiceType("atlas"); + } else { + policy.setServiceType(serviceType); + } + policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); policy.setCreateTime(atlasPolicy.getCreateTime()); From 7c281fce78fcfdaba92ed9967c5f593eecfb35f0 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Mon, 9 Dec 2024 19:40:40 +0530 Subject: [PATCH 209/241] fix beta build --- .../java/org/apache/atlas/discovery/EntityLineageService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index c28e2d56c2..8b682a0301 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -40,6 +40,7 @@ import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation; import org.apache.atlas.model.lineage.AtlasLineageOnDemandInfo.LineageInfoOnDemand; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; From b95c0e6f7720e7e74f2082d51e5fa18593330a26 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 9 Dec 2024 20:14:36 +0530 Subject: [PATCH 210/241] fix business attribute --- .../repository/store/graph/v2/EntityGraphRetriever.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index e3adc2c31b..4b1b0a105e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1029,6 +1029,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()) != null ? entityType.getAttribute(property.key()) : null; TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + boolean isBusinessAttribute = attribute == null; if (property.isPresent()) { if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { @@ -1043,6 +1044,12 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt } else { if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value()); + } else if (isBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute + LOG.warn("More than one value for property key {} for entity vertex: {}", property.key(), entityVertex); + List values = new ArrayList<>(); + values.add(propertiesMap.get(property.key())); + values.add(property.value()); + propertiesMap.put(property.key(), values); } } } From d5218aeeea92e7a78ccd2189b9aaafd1a8fe9026 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 10 Dec 2024 10:14:48 +0530 Subject: [PATCH 211/241] Remove all updated policies and take new as created --- .../org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 0c8267b5c7..08f5498c65 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -72,7 +72,7 @@ public static List applyDeltas(List policies, List(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getServiceType())) { + if (!serviceType.equals(delta.getPolicyServiceName())) { continue; } @@ -92,7 +92,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List 1) { + if (CollectionUtils.isEmpty(deletedPolicies)) { LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_UPDATE: " + Arrays.toString(deletedPolicies.toArray())); } break; From 6021ef38a0d8868f8ddf57c1e92f4f2480cfc0d8 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 10 Dec 2024 10:26:01 +0530 Subject: [PATCH 212/241] Fix missing method left over changes --- .../org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 08f5498c65..340dcbec15 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -72,7 +72,7 @@ public static List applyDeltas(List policies, List(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getPolicyServiceName())) { + if (!serviceType.equals(delta.getServiceType())) { continue; } From 4bf4f761dfa824493f897ea6a1151147f786ed50 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 10 Dec 2024 11:10:36 +0530 Subject: [PATCH 213/241] Use service name to separate tag policies during delta apply --- .../atlas/plugin/model/RangerPolicyDelta.java | 3 +++ .../plugin/store/EmbeddedServiceDefsUtil.java | 1 + .../plugin/util/RangerPolicyDeltaUtil.java | 24 +++++++++---------- .../atlas/plugin/util/ServicePolicies.java | 4 ++-- .../CachePolicyTransformerImpl.java | 14 +++++------ 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index b5f9b58233..f7dcba5466 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -89,6 +89,9 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol public void setPolicy(RangerPolicy policy) { this.policy = policy; } + @JsonIgnore + public String getPolicyServiceName() { return policy != null ? policy.getService() : null;} + @Override public String toString() { return "id:" + id diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java index 270141f22f..630a56819e 100755 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java @@ -53,6 +53,7 @@ public class EmbeddedServiceDefsUtil { private static final String PROPERTY_SUPPORTED_SERVICE_DEFS = "ranger.supportedcomponents"; private Set supportedServiceDefs; public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "tag"; + public static final String EMBEDDED_SERVICEDEF_TAG_SERVICE_NAME = "atlas_tag"; public static final String EMBEDDED_SERVICEDEF_HDFS_NAME = "hdfs"; public static final String EMBEDDED_SERVICEDEF_HBASE_NAME = "hbase"; public static final String EMBEDDED_SERVICEDEF_HIVE_NAME = "hive"; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 340dcbec15..2e41298a96 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -39,9 +39,9 @@ public class RangerPolicyDeltaUtil { private static final Log PERF_POLICY_DELTA_LOG = RangerPerfTracer.getPerfLogger("policy.delta"); - public static List applyDeltas(List policies, List deltas, String serviceType) { + public static List applyDeltas(List policies, List deltas, String serviceName) { if (LOG.isDebugEnabled()) { - LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); + LOG.debug("==> applyDeltas(serviceType=" + serviceName + ")"); } List ret; @@ -52,27 +52,27 @@ public static List applyDeltas(List policies, List(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getServiceType())) { + if (!serviceName.equals(delta.getPolicyServiceName())) { continue; } @@ -132,7 +132,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List oldResourcePolicies = policyEngine.getResourcePolicies(); List oldTagPolicies = policyEngine.getTagPolicies(); - List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceName()); ret.setPolicies(newResourcePolicies); @@ -402,7 +402,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, if (LOG.isDebugEnabled()) { LOG.debug("applyingDeltas for tag policies"); } - newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); + newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceName()); } else { if (LOG.isDebugEnabled()) { LOG.debug("No need to apply deltas for tag policies"); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index bfab5baa6c..b3b1225ca4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -710,13 +710,13 @@ private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String servi //policy.setId(atlasPolicy.getGuid()); policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME)); policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME)); - - // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation - if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { - policy.setServiceType("atlas"); - } else { - policy.setServiceType(serviceType); - } + policy.setServiceType(serviceType); +// // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation +// if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { +// policy.setServiceType("atlas"); +// } else { +// policy.setServiceType(serviceType); +// } policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); From ed4a51ec958ffe4883acf626d09a9f69c1bf0a52 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:19:41 +0530 Subject: [PATCH 214/241] Revert "Use service name to separate tag policies during delta apply" This reverts commit 4bf4f761dfa824493f897ea6a1151147f786ed50. --- .../atlas/plugin/model/RangerPolicyDelta.java | 3 --- .../plugin/store/EmbeddedServiceDefsUtil.java | 1 - .../plugin/util/RangerPolicyDeltaUtil.java | 24 +++++++++---------- .../atlas/plugin/util/ServicePolicies.java | 4 ++-- .../CachePolicyTransformerImpl.java | 14 +++++------ 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index f7dcba5466..b5f9b58233 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -89,9 +89,6 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol public void setPolicy(RangerPolicy policy) { this.policy = policy; } - @JsonIgnore - public String getPolicyServiceName() { return policy != null ? policy.getService() : null;} - @Override public String toString() { return "id:" + id diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java index 630a56819e..270141f22f 100755 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java @@ -53,7 +53,6 @@ public class EmbeddedServiceDefsUtil { private static final String PROPERTY_SUPPORTED_SERVICE_DEFS = "ranger.supportedcomponents"; private Set supportedServiceDefs; public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "tag"; - public static final String EMBEDDED_SERVICEDEF_TAG_SERVICE_NAME = "atlas_tag"; public static final String EMBEDDED_SERVICEDEF_HDFS_NAME = "hdfs"; public static final String EMBEDDED_SERVICEDEF_HBASE_NAME = "hbase"; public static final String EMBEDDED_SERVICEDEF_HIVE_NAME = "hive"; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 2e41298a96..340dcbec15 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -39,9 +39,9 @@ public class RangerPolicyDeltaUtil { private static final Log PERF_POLICY_DELTA_LOG = RangerPerfTracer.getPerfLogger("policy.delta"); - public static List applyDeltas(List policies, List deltas, String serviceName) { + public static List applyDeltas(List policies, List deltas, String serviceType) { if (LOG.isDebugEnabled()) { - LOG.debug("==> applyDeltas(serviceType=" + serviceName + ")"); + LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); } List ret; @@ -52,27 +52,27 @@ public static List applyDeltas(List policies, List(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceName.equals(delta.getPolicyServiceName())) { + if (!serviceType.equals(delta.getServiceType())) { continue; } @@ -132,7 +132,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List oldResourcePolicies = policyEngine.getResourcePolicies(); List oldTagPolicies = policyEngine.getTagPolicies(); - List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceName()); + List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); ret.setPolicies(newResourcePolicies); @@ -402,7 +402,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, if (LOG.isDebugEnabled()) { LOG.debug("applyingDeltas for tag policies"); } - newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceName()); + newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); } else { if (LOG.isDebugEnabled()) { LOG.debug("No need to apply deltas for tag policies"); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index b3b1225ca4..bfab5baa6c 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -710,13 +710,13 @@ private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String servi //policy.setId(atlasPolicy.getGuid()); policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME)); policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME)); - policy.setServiceType(serviceType); -// // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation -// if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { -// policy.setServiceType("atlas"); -// } else { -// policy.setServiceType(serviceType); -// } + + // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation + if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { + policy.setServiceType("atlas"); + } else { + policy.setServiceType(serviceType); + } policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); From 291cb2e09d47bffb0f57fd90430921e24368da16 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 13:52:08 +0530 Subject: [PATCH 215/241] add strict check for business attribute --- .../store/graph/v2/EntityGraphRetriever.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index d1b2aef6be..a4b0916258 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1031,7 +1031,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()); TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - boolean isBusinessAttribute = attribute == null; + boolean canBeBusinessAttribute = attribute == null; if (property.isPresent()) { if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { @@ -1046,7 +1046,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt } else { if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value()); - } else if (isBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute + } else if (canBeBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute LOG.warn("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); List values = new ArrayList<>(); values.add(propertiesMap.get(property.key())); @@ -1903,22 +1903,23 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); + boolean isMultiValueBusinessAttribute = attribute.getDefinedInType() != null && attribute.getDefinedInType().getTypeCategory() == TypeCategory.BUSINESS_METADATA && isArrayOfPrimitives; // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives - if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { + if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && (isMultiValueBusinessAttribute || isPrefetchValueFinal)) { return properties.get(attribute.getName()); } - //when value is not present and type is primitive, return null - if(properties.get(attribute.getName()) == null && isPrefetchValueFinal) { - return null; - } - // if value is empty && element is array of primitives, return empty list if (properties.get(attribute.getName()) == null && isArrayOfPrimitives) { return new ArrayList<>(); } + //when value is not present and type is primitive, return null + if(properties.get(attribute.getName()) == null && isPrefetchValueFinal) { + return null; + } + // value is present as marker, fetch the value from the vertex if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { //AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); From 4cb985ac8df547b72930203cf32a6599daf1b8a0 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 14:11:22 +0530 Subject: [PATCH 216/241] add check for business attributes --- .../store/graph/v2/EntityGraphRetriever.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 4b1b0a105e..55bfaa9540 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1029,7 +1029,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()) != null ? entityType.getAttribute(property.key()) : null; TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - boolean isBusinessAttribute = attribute == null; + boolean canBeBusinessAttribute = attribute == null; if (property.isPresent()) { if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { @@ -1044,7 +1044,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt } else { if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value()); - } else if (isBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute + } else if (canBeBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute LOG.warn("More than one value for property key {} for entity vertex: {}", property.key(), entityVertex); List values = new ArrayList<>(); values.add(propertiesMap.get(property.key())); @@ -1902,22 +1902,23 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); + boolean isMultiValueBusinessAttribute = attribute.getDefinedInType() != null && attribute.getDefinedInType().getTypeCategory() == TypeCategory.BUSINESS_METADATA && isArrayOfPrimitives; // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives - if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { + if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && (isMultiValueBusinessAttribute || isPrefetchValueFinal)) { return properties.get(attribute.getName()); } - //when value is not present and type is primitive, return null - if(properties.get(attribute.getName()) == null && isPrefetchValueFinal) { - return null; - } - // if value is empty && element is array and not inward relation, return empty list if (properties.get(attribute.getName()) == null && typeCategory.equals(TypeCategory.ARRAY) && !AtlasRelationshipEdgeDirection.IN.equals(attribute.getRelationshipEdgeDirection())) { return new ArrayList<>(); } + //when value is not present and type is primitive, return null + if(properties.get(attribute.getName()) == null && isPrefetchValueFinal) { + return null; + } + // value is present as marker or is inward relation, fetch the value from the vertex if (properties.get(attribute.getName()) == StringUtils.SPACE || AtlasRelationshipEdgeDirection.IN.equals(attribute.getRelationshipEdgeDirection())) { return mapVertexToAttribute(vertex, attribute, null, false); From 8df0b5e74788b01f076a043000ba1c3b64fb6d74 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:34:52 +0530 Subject: [PATCH 217/241] feat: update policy delta logic full revamp --- .../atlas/plugin/model/RangerPolicy.java | 5 +- .../policyengine/RangerPolicyRepository.java | 1 - .../plugin/util/RangerPolicyDeltaUtil.java | 129 +++++------------- .../atlas/plugin/util/ServicePolicies.java | 55 +++----- .../CachePolicyTransformerImpl.java | 66 +++++---- 5 files changed, 96 insertions(+), 160 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java index 3a4e52fd3b..e071531f80 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -532,10 +532,7 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) { @JsonIgnore public String getAtlasGuid() { - if (getGuid().length() > 36) { - return getGuid().substring(0, 36); - } - return getGuid(); + return getGuid().length() > 36 ? getGuid().substring(0, 36) : getGuid(); } @Override diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java index 4324ccb2cd..fc2112f8fc 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java @@ -1355,7 +1355,6 @@ private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) { } private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) { - LOG.info("PolicyDelta: RangerPolicyRepository.update is called, policyGuid: "+delta.getPolicyGuid()); if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")"); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 340dcbec15..a37187aee8 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -30,8 +30,8 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; import java.util.List; +import java.util.Map; public class RangerPolicyDeltaUtil { @@ -39,116 +39,59 @@ public class RangerPolicyDeltaUtil { private static final Log PERF_POLICY_DELTA_LOG = RangerPerfTracer.getPerfLogger("policy.delta"); + public static List deletePoliciesByDelta(List policies, Map deltas) { + if (MapUtils.isNotEmpty(deltas)) { + List ret = new ArrayList<>(policies); + ret.removeIf(policy -> deltas.containsKey(policy.getAtlasGuid())); + return ret; + } else { + return policies; + } + } + public static List applyDeltas(List policies, List deltas, String serviceType) { if (LOG.isDebugEnabled()) { LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); } - List ret; - - RangerPerfTracer perf = null; - - if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_DELTA_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_DELTA_LOG, "RangerPolicyDelta.applyDeltas()"); + if (CollectionUtils.isEmpty(deltas)) { + if (LOG.isDebugEnabled()) { + LOG.debug("applyDeltas called with empty deltas. Returning policies without changes."); + } + return policies; } - boolean hasExpectedServiceType = false; + boolean hasExpectedServiceType = deltas.stream().anyMatch(delta -> serviceType.equals(delta.getServiceType())); - if (CollectionUtils.isNotEmpty(deltas)) { + if (!hasExpectedServiceType) { if (LOG.isDebugEnabled()) { - LOG.debug("applyDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", serviceType=" + serviceType + ")"); - } - - for (RangerPolicyDelta delta : deltas) { - if (serviceType.equals(delta.getServiceType())) { - hasExpectedServiceType = true; - break; - } else if (!serviceType.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME) && !delta.getServiceType().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { - LOG.warn("Found unexpected serviceType in policyDelta:[" + delta + "]. Was expecting serviceType:[" + serviceType + "]. Should NOT have come here!! Ignoring delta and continuing"); - } + LOG.debug("No deltas match the expected serviceType: " + serviceType); } + return policies; + } - if (hasExpectedServiceType) { - ret = new ArrayList<>(policies); + List updatedPolicies = new ArrayList<>(policies); - for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getServiceType())) { - continue; - } - - int changeType = delta.getChangeType(); - - if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { - String policyId = delta.getPolicyGuid(); // change to getGuid() as id is not set in policy - - if (policyId == null) { - continue; - } - - List deletedPolicies = new ArrayList<>(); - - Iterator iter = ret.iterator(); - - while (iter.hasNext()) { - RangerPolicy policy = iter.next(); - if ( - (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE && policyId.equals(policy.getAtlasGuid())) || - (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE && policyId.equals(policy.getAtlasGuid())) - ){ - deletedPolicies.add(policy); - iter.remove(); - } - } - - switch(changeType) { - case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: { - if (CollectionUtils.isNotEmpty(deletedPolicies)) { - LOG.warn("Unexpected: found existing policy for CHANGE_TYPE_POLICY_CREATE: " + Arrays.toString(deletedPolicies.toArray())); - } - break; - } - case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: { - if (CollectionUtils.isEmpty(deletedPolicies)) { - LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_UPDATE: " + Arrays.toString(deletedPolicies.toArray())); - } - break; - } - case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: { - if (CollectionUtils.isEmpty(deletedPolicies)) { - LOG.warn("Unexpected: found no policy for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray())); - } - break; - } - default: - break; - } - - if (changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { - ret.add(delta.getPolicy()); - } - } else { - LOG.warn("Found unexpected changeType in policyDelta:[" + delta + "]. Ignoring delta"); - } - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("applyDeltas - none of the deltas is for " + serviceType + ")"); - } - ret = policies; + for (RangerPolicyDelta delta : deltas) { + if (!serviceType.equals(delta.getServiceType())) { + continue; } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("applyDeltas called with empty deltas. Will return policies without change"); + + switch (delta.getChangeType()) { + case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: + case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: + updatedPolicies.add(delta.getPolicy()); + break; + default: + LOG.warn("Unexpected changeType in policyDelta: [" + delta + "]. Ignoring delta."); } - ret = policies; } - RangerPerfTracer.log(perf); - if (LOG.isDebugEnabled()) { - LOG.debug("<== applyDeltas(serviceType=" + serviceType + "): " + ret); + LOG.debug("<== applyDeltas(serviceType=" + serviceType + "): " + updatedPolicies); } - return ret; + + return updatedPolicies; } public static boolean isValidDeltas(List deltas, String componentServiceType) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 77717e1e08..3431cfc1f9 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -53,6 +53,8 @@ public class ServicePolicies implements java.io.Serializable { private TagPolicies tagPolicies; private Map securityZones; private List policyDeltas; + + private Map deleteDeltas; private Map serviceConfig; /** @@ -181,6 +183,14 @@ public String toString() { public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } + public Map getDeleteDeltas() { + return deleteDeltas; + } + + public void setDeleteDeltas(Map deleteDeltas) { + this.deleteDeltas = deleteDeltas; + } + @JsonInclude(JsonInclude.Include.NON_NULL) @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @@ -393,16 +403,21 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, List oldResourcePolicies = policyEngine.getResourcePolicies(); List oldTagPolicies = policyEngine.getTagPolicies(); - List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + List resourcePoliciesAfterDelete = + RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, servicePolicies.getDeleteDeltas()); + List newResourcePolicies = + RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); ret.setPolicies(newResourcePolicies); - final List newTagPolicies; + List newTagPolicies; if (servicePolicies.getTagPolicies() != null) { if (LOG.isDebugEnabled()) { LOG.debug("applyingDeltas for tag policies"); } - newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); + List tagPoliciesAfterDelete = + RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, servicePolicies.getDeleteDeltas()); + newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); } else { if (LOG.isDebugEnabled()) { LOG.debug("No need to apply deltas for tag policies"); @@ -411,45 +426,13 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, } if (LOG.isDebugEnabled()) { - LOG.debug("New tag policies:[" + Arrays.toString(newTagPolicies.toArray()) + "]"); + LOG.debug("New tag policies: " + newTagPolicies); } if (ret.getTagPolicies() != null) { ret.getTagPolicies().setPolicies(newTagPolicies); } - if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { - Map newSecurityZones = new HashMap<>(); - - for (Map.Entry entry : servicePolicies.getSecurityZones().entrySet()) { - String zoneName = entry.getKey(); - SecurityZoneInfo zoneInfo = entry.getValue(); - - List zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName); - // There are no separate tag-policy-repositories for each zone - - if (LOG.isDebugEnabled()) { - LOG.debug("Applying deltas for security-zone:[" + zoneName + "]"); - } - - final List newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); - - if (LOG.isDebugEnabled()) { - LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]"); - } - - SecurityZoneInfo newZoneInfo = new SecurityZoneInfo(); - - newZoneInfo.setZoneName(zoneName); - newZoneInfo.setResources(zoneInfo.getResources()); - newZoneInfo.setPolicies(newZonePolicies); - - newSecurityZones.put(zoneName, newZoneInfo); - } - - ret.setSecurityZones(newSecurityZones); - } - return ret; } } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index bfab5baa6c..86a956704c 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -66,6 +66,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.Iterator; import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; @@ -142,7 +143,7 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa this.auditEventToDeltaChangeType = new HashMap<>(); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); - this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_PURGE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } @@ -176,6 +177,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map atlasServicePolicies = allAtlasPolicies.stream().filter(x -> serviceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList()); List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasServicePolicies); + Map deletedPolicyDeltas = getRangerPolicyDeleteDelta(service, policyChanges); + servicePolicies.setDeleteDeltas(deletedPolicyDeltas); // Process tag based policies String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); @@ -293,6 +296,42 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } + private Map getRangerPolicyDeleteDelta(AtlasEntityHeader service, Map policyChanges) { + String serviceName = (String) service.getAttribute("name"); + String serviceType = (String) service.getAttribute("authServiceType"); + Map policyDeltas = new HashMap<>(); + if (policyChanges.isEmpty()) { + return policyDeltas; + } + + Iterator> iterator = policyChanges.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + String policyGuid = entry.getKey(); + EntityAuditActionV2 policyChange = entry.getValue(); + + if (policyChange == EntityAuditActionV2.ENTITY_UPDATE || policyChange == EntityAuditActionV2.ENTITY_DELETE + || policyChange == EntityAuditActionV2.ENTITY_PURGE) { + RangerPolicy atlasDeletedPolicy = new RangerPolicy(); + atlasDeletedPolicy.setGuid(policyGuid); + atlasDeletedPolicy.setService(serviceName); + atlasDeletedPolicy.setServiceType(serviceType); + + policyDeltas.put(policyGuid, new RangerPolicyDelta(atlasDeletedPolicy.getId(), + RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE, + atlasDeletedPolicy.getVersion(), + atlasDeletedPolicy)); + } + + if (policyChange == EntityAuditActionV2.ENTITY_DELETE || policyChange == EntityAuditActionV2.ENTITY_PURGE) { + iterator.remove(); // Remove for ENTITY_DELETE and ENTITY_PURGE + } + } + + return policyDeltas; + } + + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); @@ -303,8 +342,6 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } - ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); - List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -316,29 +353,6 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, policyDeltas.add(delta); } - // handle delete changes separately as they won't be present in atlas policies - List deletedPolicyDeltas = new ArrayList<>(); - for (String policyGuid : policyGuids) { - int deltaChangeType = auditEventToDeltaChangeType.get(policyChanges.get(policyGuid)); - if (deltaChangeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { - RangerPolicy deletedPolicy = new RangerPolicy(); - deletedPolicy.setGuid(policyGuid); - deletedPolicy.setService(serviceName); - deletedPolicy.setServiceType(serviceType); - RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta( - deletedPolicy.getId(), - deltaChangeType, - deletedPolicy.getVersion(), - deletedPolicy - ); - deletedPolicyDeltas.add(deletedPolicyDelta); - } - } - - policyDeltas.addAll(deletedPolicyDeltas); - - LOG.info("PolicyDelta: {}: atlas policies found={}, delta created={}, including deleted policies={}", - serviceName, atlasPolicies.size(), policyDeltas.size(), deletedPolicyDeltas.size()); RequestContext.get().endMetricRecord(recorder); return policyDeltas; From 10781824bb0d7953bf554d3f3e3d8f5f63269326 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 16:27:22 +0530 Subject: [PATCH 218/241] code cleanup --- .../store/graph/v2/EntityGraphRetriever.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index a4b0916258..e8df762193 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -73,6 +73,8 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.janusgraph.core.Cardinality; +import org.janusgraph.graphdb.relations.CacheVertexProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -1031,28 +1033,16 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()); TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - boolean canBeBusinessAttribute = attribute == null; + boolean isMultiValuedProperty = (property instanceof CacheVertexProperty && ((CacheVertexProperty) property).propertyKey().cardinality().equals(Cardinality.SET)); if (property.isPresent()) { if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { - Object value = propertiesMap.get(property.key()); - if (value instanceof List) { - ((List) value).add(property.value()); - } else { - List values = new ArrayList<>(); - values.add(property.value()); - propertiesMap.put(property.key(), values); - } - } else { - if (propertiesMap.get(property.key()) == null) { - propertiesMap.put(property.key(), property.value()); - } else if (canBeBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute - LOG.warn("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); - List values = new ArrayList<>(); - values.add(propertiesMap.get(property.key())); - values.add(property.value()); - propertiesMap.put(property.key(), values); - } + updateAttrValue(propertiesMap, property); + } else if (attribute == null && isMultiValuedProperty) { + LOG.warn("Multiple value found for key {} for entity vertex: {}", property.key(), entityVertex); + updateAttrValue(propertiesMap, property); + } else if (propertiesMap.get(property.key()) == null) { + propertiesMap.put(property.key(), property.value()); } } } catch (RuntimeException e) { @@ -1063,6 +1053,16 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt return propertiesMap; } + private void updateAttrValue( Map propertiesMap, VertexProperty property){ + Object value = propertiesMap.get(property.key()); + if (value instanceof List) { + ((List) value).add(property.value()); + } else { + List values = new ArrayList<>(); + values.add(property.value()); + propertiesMap.put(property.key(), values); + } + } private boolean isPolicyAttribute(Set attributes) { Set exclusionSet = new HashSet<>(Arrays.asList(AccessControlUtils.ATTR_POLICY_TYPE, From dce4f05e90fced4d28fb53c248f7645f585b82af Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 17:14:17 +0530 Subject: [PATCH 219/241] compute flag only if property exists --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index e8df762193..f49d343e32 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1033,9 +1033,13 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()); TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - boolean isMultiValuedProperty = (property instanceof CacheVertexProperty && ((CacheVertexProperty) property).propertyKey().cardinality().equals(Cardinality.SET)); if (property.isPresent()) { + + // If the attribute is not known (null) + // validate if it is multi-valued + boolean isMultiValuedProperty = (property instanceof CacheVertexProperty && ((CacheVertexProperty) property).propertyKey().cardinality().equals(Cardinality.SET)); + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { updateAttrValue(propertiesMap, property); } else if (attribute == null && isMultiValuedProperty) { From b2a58a64dcbf23a0b927695a65ec8f2e881396cc Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 17:15:47 +0530 Subject: [PATCH 220/241] verbose comment --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index f49d343e32..09ec1c3eb2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1037,7 +1037,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt if (property.isPresent()) { // If the attribute is not known (null) - // validate if it is multi-valued + // validate if prefetched property is multi-valued boolean isMultiValuedProperty = (property instanceof CacheVertexProperty && ((CacheVertexProperty) property).propertyKey().cardinality().equals(Cardinality.SET)); if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { From c74e6d4bd5e7e3457de0c0701bb08933d022148b Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:18:30 +0530 Subject: [PATCH 221/241] fix: delete working as expected --- .../atlas/plugin/model/RangerPolicyDelta.java | 3 + .../atlas/plugin/util/ServicePolicies.java | 47 +++++++++++++- .../CachePolicyTransformerImpl.java | 65 ++++++++----------- 3 files changed, 74 insertions(+), 41 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index b5f9b58233..d1976c9193 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -76,6 +76,9 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol @JsonIgnore public String getPolicyGuid() { return policy != null ? policy.getGuid() : null; } + @JsonIgnore + public String getPolicyAtlasGuid() { return policy != null ? policy.getAtlasGuid() : null; } + @JsonIgnore public String getZoneName() { return policy != null ? policy.getZoneName() : null; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 3431cfc1f9..64146db782 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -397,14 +397,25 @@ static public TagPolicies copyHeader(TagPolicies source, String componentService return ret; } + private static Map fetchDeletedDeltaMap(List deltas) { + Map ret = new HashMap<>(); + for (RangerPolicyDelta delta : deltas) { + if (delta.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE || delta.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE) { + ret.put(delta.getPolicyAtlasGuid(), delta); + } + } + return ret; + } + public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { ServicePolicies ret = copyHeader(servicePolicies); List oldResourcePolicies = policyEngine.getResourcePolicies(); List oldTagPolicies = policyEngine.getTagPolicies(); + Map deletedDeltaMap = fetchDeletedDeltaMap(servicePolicies.getPolicyDeltas()); List resourcePoliciesAfterDelete = - RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, servicePolicies.getDeleteDeltas()); + RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, deletedDeltaMap); List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); @@ -416,7 +427,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, LOG.debug("applyingDeltas for tag policies"); } List tagPoliciesAfterDelete = - RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, servicePolicies.getDeleteDeltas()); + RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, deletedDeltaMap); newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); } else { if (LOG.isDebugEnabled()) { @@ -433,6 +444,38 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, ret.getTagPolicies().setPolicies(newTagPolicies); } + if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { + Map newSecurityZones = new HashMap<>(); + + for (Map.Entry entry : servicePolicies.getSecurityZones().entrySet()) { + String zoneName = entry.getKey(); + SecurityZoneInfo zoneInfo = entry.getValue(); + + List zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName); + // There are no separate tag-policy-repositories for each zone + + if (LOG.isDebugEnabled()) { + LOG.debug("Applying deltas for security-zone:[" + zoneName + "]"); + } + + final List newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + + if (LOG.isDebugEnabled()) { + LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]"); + } + + SecurityZoneInfo newZoneInfo = new SecurityZoneInfo(); + + newZoneInfo.setZoneName(zoneName); + newZoneInfo.setResources(zoneInfo.getResources()); + newZoneInfo.setPolicies(newZonePolicies); + + newSecurityZones.put(zoneName, newZoneInfo); + } + + ret.setSecurityZones(newSecurityZones); + } + return ret; } } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 86a956704c..b85569a780 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -143,7 +143,7 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa this.auditEventToDeltaChangeType = new HashMap<>(); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); - this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_PURGE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } @@ -177,8 +177,6 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map atlasServicePolicies = allAtlasPolicies.stream().filter(x -> serviceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList()); List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasServicePolicies); - Map deletedPolicyDeltas = getRangerPolicyDeleteDelta(service, policyChanges); - servicePolicies.setDeleteDeltas(deletedPolicyDeltas); // Process tag based policies String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); @@ -296,42 +294,6 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private Map getRangerPolicyDeleteDelta(AtlasEntityHeader service, Map policyChanges) { - String serviceName = (String) service.getAttribute("name"); - String serviceType = (String) service.getAttribute("authServiceType"); - Map policyDeltas = new HashMap<>(); - if (policyChanges.isEmpty()) { - return policyDeltas; - } - - Iterator> iterator = policyChanges.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - String policyGuid = entry.getKey(); - EntityAuditActionV2 policyChange = entry.getValue(); - - if (policyChange == EntityAuditActionV2.ENTITY_UPDATE || policyChange == EntityAuditActionV2.ENTITY_DELETE - || policyChange == EntityAuditActionV2.ENTITY_PURGE) { - RangerPolicy atlasDeletedPolicy = new RangerPolicy(); - atlasDeletedPolicy.setGuid(policyGuid); - atlasDeletedPolicy.setService(serviceName); - atlasDeletedPolicy.setServiceType(serviceType); - - policyDeltas.put(policyGuid, new RangerPolicyDelta(atlasDeletedPolicy.getId(), - RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE, - atlasDeletedPolicy.getVersion(), - atlasDeletedPolicy)); - } - - if (policyChange == EntityAuditActionV2.ENTITY_DELETE || policyChange == EntityAuditActionV2.ENTITY_PURGE) { - iterator.remove(); // Remove for ENTITY_DELETE and ENTITY_PURGE - } - } - - return policyDeltas; - } - - private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); @@ -342,6 +304,8 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } + ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); + List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -353,6 +317,29 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, policyDeltas.add(delta); } + // handle delete changes separately as they won't be present in atlas policies + List deletedPolicyDeltas = new ArrayList<>(); + for (String policyGuid : policyGuids) { + int deltaChangeType = auditEventToDeltaChangeType.get(policyChanges.get(policyGuid)); + if (deltaChangeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { + RangerPolicy deletedPolicy = new RangerPolicy(); + deletedPolicy.setGuid(policyGuid); + deletedPolicy.setService(serviceName); + deletedPolicy.setServiceType(serviceType); + RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta( + deletedPolicy.getId(), + deltaChangeType, + deletedPolicy.getVersion(), + deletedPolicy + ); + deletedPolicyDeltas.add(deletedPolicyDelta); + } + } + + policyDeltas.addAll(deletedPolicyDeltas); + + LOG.info("PolicyDelta: {}: atlas policies found={}, delta created={}, including deleted policies={}", + serviceName, atlasPolicies.size(), policyDeltas.size(), deletedPolicyDeltas.size()); RequestContext.get().endMetricRecord(recorder); return policyDeltas; From 37d92550f7ea24ad07b893b9a70c60189f670144 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:45:39 +0530 Subject: [PATCH 222/241] fix: create policy engine from scratch in case of updation --- .../org/apache/atlas/plugin/service/RangerBasePlugin.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index 76ee3c3e1f..4dd41bcd40 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -376,10 +376,8 @@ public void setPolicies(ServicePolicies policies) { } if (oldPolicyEngine != null) { - RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine; - - newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies); - //TODO: this looks like a mistake, second arg should be servicePolicies which has the applied delta + // Create new evaluator for the updated policies + newPolicyEngine = new RangerPolicyEngineImpl(servicePolicies, pluginContext, roles); } if (newPolicyEngine != null) { From 0da7195c976c664c3346a7404816ba062e96297b Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 19:23:52 +0530 Subject: [PATCH 223/241] add policy variables --- .../store/graph/v2/EntityGraphRetriever.java | 11 ++++++----- .../atlas/repository/util/AccessControlUtils.java | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 09ec1c3eb2..e9ee72fcfe 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1081,7 +1081,12 @@ private boolean isPolicyAttribute(Set attributes) { AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY, AccessControlUtils.ATTR_POLICY_SERVICE_NAME, AccessControlUtils.ATTR_POLICY_PRIORITY, - AccessControlUtils.REL_ATTR_POLICIES)); + AccessControlUtils.REL_ATTR_POLICIES, + AccessControlUtils.ATTR_SERVICE_SERVICE_TYPE, + AccessControlUtils.ATTR_SERVICE_TAG_SERVICE, + AccessControlUtils.ATTR_SERVICE_IS_ENABLED, + AccessControlUtils.ATTR_SERVICE_LAST_SYNC) + ); return exclusionSet.stream().anyMatch(attributes::contains); } @@ -1926,11 +1931,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute // value is present as marker, fetch the value from the vertex if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { - //AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); return mapVertexToAttribute(vertex, attribute, null, false); - //LOG.debug("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); - //RequestContext.get().endMetricRecord(nonPrimitiveAttributes); - //return mappedVertex; } return null; diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index 7101868e30..1f89e4d65c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -61,6 +61,10 @@ public final class AccessControlUtils { public static final String ATTR_PURPOSE_CLASSIFICATIONS = "purposeClassifications"; + public static final String ATTR_SERVICE_SERVICE_TYPE = "authServiceType"; + public static final String ATTR_SERVICE_TAG_SERVICE = "tagService"; + public static final String ATTR_SERVICE_IS_ENABLED = "authServiceIsEnabled"; + public static final String ATTR_SERVICE_LAST_SYNC = "authServicePolicyLastSync"; public static final String ATTR_POLICY_TYPE = "policyType"; public static final String ATTR_POLICY_USERS = "policyUsers"; public static final String ATTR_POLICY_GROUPS = "policyGroups"; From 759c2302f694ae1e3725cacd8743444d896b9225 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 20:03:55 +0530 Subject: [PATCH 224/241] fix pipeline --- .github/workflows/maven.yml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..b8481ac283 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,3 +1,4 @@ + # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -26,6 +27,7 @@ on: - development - master - lineageondemand + - janus0.6 jobs: build: @@ -34,12 +36,12 @@ jobs: steps: - uses: actions/checkout@v2 - + - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: java-version: 1.8 - + - name: Cache Maven packages uses: actions/cache@v2 with: @@ -48,8 +50,9 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: get_branch + run: | + echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + echo BRANCH_NAME=${GITHUB_REF#refs/heads/} - name: Create Maven Settings uses: s4u/maven-settings-action@v2.8.0 @@ -63,7 +66,7 @@ jobs: - name: Build with Maven run: | - branch_name=${{ steps.get_branch.outputs.branch }} + branch_name=${{ env.BRANCH_NAME }} if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]] then echo "build without dashboard" @@ -73,19 +76,26 @@ jobs: chmod +x ./build.sh && ./build.sh fi - - run: echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV + - name: Get Repository Name + run: echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" - id: get_version + # run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: | + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | head -n 1)abcd" + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" >> $GITHUB_ENV + + - name: Get commit ID + run: echo "COMMIT_ID=$(echo ${GITHUB_SHA} | cut -c1-7)abcd" >> $GITHUB_ENV - - name: Set up Buildx + - name: Set up Buildx id: buildx uses: docker/setup-buildx-action@v1 - name: Login to GitHub Registry - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: registry: ghcr.io username: $GITHUB_ACTOR @@ -102,8 +112,8 @@ jobs: provenance: true push: true tags: | - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:latest + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:${{ env.COMMIT_ID }} - name: Scan Image uses: aquasecurity/trivy-action@master @@ -116,4 +126,4 @@ jobs: - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v2.1.33 with: - sarif_file: 'trivy-image-results.sarif' + sarif_file: 'trivy-image-results.sarif' \ No newline at end of file From f2de42dafe54c7fd945445d347a03e6da4176103 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 21:00:43 +0530 Subject: [PATCH 225/241] remove log line --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index e9ee72fcfe..59125c492c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1043,7 +1043,6 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { updateAttrValue(propertiesMap, property); } else if (attribute == null && isMultiValuedProperty) { - LOG.warn("Multiple value found for key {} for entity vertex: {}", property.key(), entityVertex); updateAttrValue(propertiesMap, property); } else if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value()); From 2e5048921d56ce5fb81ce33cb6702b8807d2b68d Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 21:13:13 +0530 Subject: [PATCH 226/241] nit: few refactors and policy saving based name instead of types --- .../plugin/service/RangerBasePlugin.java | 3 +- .../plugin/util/RangerPolicyDeltaUtil.java | 10 ++-- .../atlas/plugin/util/ServicePolicies.java | 53 +++---------------- .../CachePolicyTransformerImpl.java | 10 +--- 4 files changed, 16 insertions(+), 60 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index 4dd41bcd40..af0aa716e1 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -428,7 +428,8 @@ public void setPolicies(ServicePolicies policies) { } } catch (Exception e) { - LOG.error("setPolicies: policy engine initialization failed! Leaving current policy engine as-is. Exception : ", e); + LOG.error("setPolicies: Failed to set policies, didn't set policies", e); + throw e; } if (LOG.isDebugEnabled()) { LOG.debug("<== setPolicies(" + policies + ")"); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index a37187aee8..d1e5ebf03a 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -49,7 +49,7 @@ public static List deletePoliciesByDelta(List polici } } - public static List applyDeltas(List policies, List deltas, String serviceType) { + public static List applyDeltas(List policies, List deltas, String serviceType, String serviceName) { if (LOG.isDebugEnabled()) { LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); } @@ -61,9 +61,9 @@ public static List applyDeltas(List policies, List serviceType.equals(delta.getServiceType())); + boolean hasExpectedServiceName = deltas.stream().anyMatch(delta -> serviceName.equals(delta.getPolicy().getService())); - if (!hasExpectedServiceType) { + if (!hasExpectedServiceName) { if (LOG.isDebugEnabled()) { LOG.debug("No deltas match the expected serviceType: " + serviceType); } @@ -73,7 +73,7 @@ public static List applyDeltas(List policies, List updatedPolicies = new ArrayList<>(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getServiceType())) { + if (!serviceName.equals(delta.getPolicy().getService())) { continue; } @@ -83,7 +83,7 @@ public static List applyDeltas(List policies, List securityZones; private List policyDeltas; - - private Map deleteDeltas; private Map serviceConfig; /** @@ -183,14 +186,6 @@ public String toString() { public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } - public Map getDeleteDeltas() { - return deleteDeltas; - } - - public void setDeleteDeltas(Map deleteDeltas) { - this.deleteDeltas = deleteDeltas; - } - @JsonInclude(JsonInclude.Include.NON_NULL) @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @@ -417,7 +412,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, List resourcePoliciesAfterDelete = RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, deletedDeltaMap); List newResourcePolicies = - RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName(), servicePolicies.getServiceName()); ret.setPolicies(newResourcePolicies); @@ -428,7 +423,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, } List tagPoliciesAfterDelete = RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, deletedDeltaMap); - newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); + newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName(), servicePolicies.getTagPolicies().getServiceName()); } else { if (LOG.isDebugEnabled()) { LOG.debug("No need to apply deltas for tag policies"); @@ -444,38 +439,6 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, ret.getTagPolicies().setPolicies(newTagPolicies); } - if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { - Map newSecurityZones = new HashMap<>(); - - for (Map.Entry entry : servicePolicies.getSecurityZones().entrySet()) { - String zoneName = entry.getKey(); - SecurityZoneInfo zoneInfo = entry.getValue(); - - List zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName); - // There are no separate tag-policy-repositories for each zone - - if (LOG.isDebugEnabled()) { - LOG.debug("Applying deltas for security-zone:[" + zoneName + "]"); - } - - final List newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); - - if (LOG.isDebugEnabled()) { - LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]"); - } - - SecurityZoneInfo newZoneInfo = new SecurityZoneInfo(); - - newZoneInfo.setZoneName(zoneName); - newZoneInfo.setResources(zoneInfo.getResources()); - newZoneInfo.setPolicies(newZonePolicies); - - newSecurityZones.put(zoneName, newZoneInfo); - } - - ret.setSecurityZones(newSecurityZones); - } - return ret; } } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index b85569a780..71e798091b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -708,17 +708,9 @@ private Map getMap(String key, Object value) { private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String serviceType) { RangerPolicy policy = new RangerPolicy(); - //policy.setId(atlasPolicy.getGuid()); policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME)); policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME)); - - // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation - if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { - policy.setServiceType("atlas"); - } else { - policy.setServiceType(serviceType); - } - + policy.setServiceType(serviceType); policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); policy.setCreateTime(atlasPolicy.getCreateTime()); From 0577ea18462f7f55c65158bfd83249c6ac97334d Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 21:13:48 +0530 Subject: [PATCH 227/241] nit: add exceptions in setpolicies incase of failure it will retry --- .../atlas/plugin/util/PolicyRefresher.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index ed5b1a3687..99deb21d2d 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -37,7 +37,6 @@ import java.io.FileWriter; import java.io.Reader; import java.io.Writer; -import java.util.Date; import java.util.Timer; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -62,7 +61,7 @@ public class PolicyRefresher extends Thread { private final BlockingQueue policyDownloadQueue = new LinkedBlockingQueue<>(); private Timer policyDownloadTimer; private long lastKnownVersion = -1L; - private long lastUpdatedTiemInMillis = -1L; + private long lastUpdatedTimeInMillis = -1L; private long lastActivationTimeInMillis; private boolean policiesSetInPlugin; private boolean serviceDefSetInPlugin; @@ -222,7 +221,7 @@ public void run() { loadPolicy(); loadUserStore(); } catch(InterruptedException excp) { - LOG.info("PolicyRefresher(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp); + LOG.info("PolicyRefreshxer(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp); break; } finally { if (trigger != null) { @@ -279,7 +278,7 @@ private void loadPolicy() { serviceDefSetInPlugin = false; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; - lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; + lastUpdatedTimeInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; } else { if (!policiesSetInPlugin && !serviceDefSetInPlugin) { plugIn.setPolicies(null); @@ -293,10 +292,10 @@ private void loadPolicy() { serviceDefSetInPlugin = true; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = -1; - lastUpdatedTiemInMillis = -1; + lastUpdatedTimeInMillis = -1; } } catch (Exception excp) { - LOG.error("Encountered unexpected exception, ignoring..", excp); + LOG.error("Encountered unexpected exception!!!!!!!!!!!", excp); } RangerPerfTracer.log(perf); @@ -323,17 +322,17 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { - if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTimeInMillis == -1) { LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading all policies for first time"); RangerRESTUtils restUtils = new RangerRESTUtils(); CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); svcPolicies = transformer.getPoliciesAll(serviceName, restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis); + lastUpdatedTimeInMillis); } else { - LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTiemInMillis); - svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh); + LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTimeInMillis); + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTimeInMillis, this.enableDeltaBasedRefresh); } boolean isUpdated = svcPolicies != null; @@ -400,7 +399,7 @@ private ServicePolicies loadFromCache() { } lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue(); - lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); + lastUpdatedTimeInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); } } catch (Exception excp) { LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp); From f4e02273f7c1ccbfb8fccb2fa61634535921e74f Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Dec 2024 08:18:03 +0530 Subject: [PATCH 228/241] mesh-294: resolved PR comments --- .../instance/UniqueQnMigrationRequest.java | 66 ------------------- .../v2/UniqueQNAttributeMigrationService.java | 11 +++- .../apache/atlas/web/rest/MigrationREST.java | 13 ++-- 3 files changed, 15 insertions(+), 75 deletions(-) delete mode 100644 intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java diff --git a/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java deleted file mode 100644 index df263eb761..0000000000 --- a/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.model.instance; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; -import java.util.Set; - -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; - -/** - * Request to link/unlink policies from asset. - */ -@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@XmlRootElement -@XmlAccessorType(XmlAccessType.PROPERTY) -public class UniqueQnMigrationRequest implements Serializable { - private static final long serialVersionUID = 1L; - - private Set assetGuids; - - - public Set getAssetGuids() { - return assetGuids; - } - - public void setAssetGuids(Set assetGuids) { - this.assetGuids = assetGuids; - } - - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("LinkBusinessPolicyRequest{"); - sb.append("assetGuids=").append(assetGuids); - sb.append('}'); - return sb.toString(); - } -} - diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java index 47e61b2b38..2c8f345791 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java @@ -30,6 +30,7 @@ public UniqueQNAttributeMigrationService(EntityGraphRetriever entityRetriever, S public void migrateQN() throws Exception { try { + int count = 0; for (String entityGuid : entityGuids) { AtlasVertex entityVertex = entityRetriever.getEntityVertex(entityGuid); @@ -40,14 +41,20 @@ public void migrateQN() throws Exception { boolean isCommitRequired = migrateuniqueQnAttr(entityVertex); if (isCommitRequired){ - LOG.info("Committing changes for entity: {}", entityGuid); - commitChanges(); + count++; } else { LOG.info("No changes to commit for entity: {} as no migration needed", entityGuid); } } + if (count > 0) { + commitChanges(); + } + else { + LOG.info("No changes to commit for entities as no migration needed"); + } + } catch (Exception e) { LOG.error("Error while migration unique qualifiedName attribute for entities: {}", entityGuids, e); throw e; diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index a480793cf1..c50e472d84 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -8,7 +8,6 @@ import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.EntityMutationResponse; -import org.apache.atlas.model.instance.UniqueQnMigrationRequest; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.store.graph.AtlasEntityStore; @@ -327,23 +326,23 @@ public List searchForType(@PathParam("typeName") String typeName, @ } @POST - @Path("update-unique-qualified-name") + @Path("repair-unique-qualified-name") @Timed - public Boolean updateUniqueQualifiedName(final UniqueQnMigrationRequest request) throws Exception { + public Boolean updateUniqueQualifiedName(final Set assetGuids) throws Exception { AtlasPerfTracer perf = null; try { - if (CollectionUtils.isEmpty(request.getAssetGuids())) { + if (CollectionUtils.isEmpty(assetGuids)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Asset GUIDs are required for which updating unique qualified name is required"); } if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.updateUniqueQualifiedName(" + request.getAssetGuids() + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.updateUniqueQualifiedName(" + assetGuids + ")"); } - UniqueQNAttributeMigrationService migrationService = new UniqueQNAttributeMigrationService(entityRetriever, request.getAssetGuids(), transactionInterceptHelper); + UniqueQNAttributeMigrationService migrationService = new UniqueQNAttributeMigrationService(entityRetriever, assetGuids, transactionInterceptHelper); migrationService.migrateQN(); } catch (Exception e) { - LOG.error("Error while updating unique qualified name for guids: {}", request.getAssetGuids(), e); + LOG.error("Error while updating unique qualified name for guids: {}", assetGuids, e); throw e; } finally { AtlasPerfTracer.log(perf); From 14234a1126e51319d57a862bfca49cedc4c4ef72 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Dec 2024 08:20:24 +0530 Subject: [PATCH 229/241] mesh-294: added info log --- .../store/graph/v2/UniqueQNAttributeMigrationService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java index 2c8f345791..52925970ef 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java @@ -49,6 +49,7 @@ public void migrateQN() throws Exception { } if (count > 0) { + LOG.info("Total Vertex updated: {}", count); commitChanges(); } else { From 8eef37795e33e54e91597623e5a7e3bbbd3c64eb Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 11 Dec 2024 11:53:37 +0530 Subject: [PATCH 230/241] code cleanup --- .../store/graph/v2/EntityGraphRetriever.java | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 55bfaa9540..eaebce115c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -81,6 +81,8 @@ import java.util.*; import java.util.concurrent.*; import java.util.stream.Collectors; +import org.janusgraph.core.Cardinality; +import org.janusgraph.graphdb.relations.CacheVertexProperty; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CREATED_BY; @@ -1029,28 +1031,19 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()) != null ? entityType.getAttribute(property.key()) : null; TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - boolean canBeBusinessAttribute = attribute == null; if (property.isPresent()) { + + // If the attribute is not known (null) + // validate if prefetched property is multi-valued + boolean isMultiValuedProperty = (property instanceof CacheVertexProperty && ((CacheVertexProperty) property).propertyKey().cardinality().equals(Cardinality.SET)); + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { - Object value = propertiesMap.get(property.key()); - if (value instanceof List) { - ((List) value).add(property.value()); - } else { - List values = new ArrayList<>(); - values.add(property.value()); - propertiesMap.put(property.key(), values); - } - } else { - if (propertiesMap.get(property.key()) == null) { - propertiesMap.put(property.key(), property.value()); - } else if (canBeBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute - LOG.warn("More than one value for property key {} for entity vertex: {}", property.key(), entityVertex); - List values = new ArrayList<>(); - values.add(propertiesMap.get(property.key())); - values.add(property.value()); - propertiesMap.put(property.key(), values); - } + updateAttrValue(propertiesMap, property); + } else if (attribute == null && isMultiValuedProperty) { + updateAttrValue(propertiesMap, property); + } else if (propertiesMap.get(property.key()) == null) { + propertiesMap.put(property.key(), property.value()); } } } catch (RuntimeException e) { @@ -1061,6 +1054,18 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt return propertiesMap; } + private void updateAttrValue( Map propertiesMap, VertexProperty property){ + Object value = propertiesMap.get(property.key()); + if (value instanceof List) { + ((List) value).add(property.value()); + } else { + List values = new ArrayList<>(); + values.add(property.value()); + propertiesMap.put(property.key(), values); + } + } + + private boolean isPolicyAttribute(Set attributes) { Set exclusionSet = new HashSet<>(Arrays.asList(AccessControlUtils.ATTR_POLICY_TYPE, AccessControlUtils.ATTR_POLICY_USERS, From 41b2fb863b86a1fffdfe82bb3fa09fc65831cadb Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Dec 2024 11:54:33 +0530 Subject: [PATCH 231/241] mesh-294: added batch in commiting changes --- .../v2/UniqueQNAttributeMigrationService.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java index 52925970ef..a7e0fc46e7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java @@ -31,6 +31,7 @@ public UniqueQNAttributeMigrationService(EntityGraphRetriever entityRetriever, S public void migrateQN() throws Exception { try { int count = 0; + int totalUpdatedCount = 0; for (String entityGuid : entityGuids) { AtlasVertex entityVertex = entityRetriever.getEntityVertex(entityGuid); @@ -42,19 +43,25 @@ public void migrateQN() throws Exception { boolean isCommitRequired = migrateuniqueQnAttr(entityVertex); if (isCommitRequired){ count++; + totalUpdatedCount++; } else { LOG.info("No changes to commit for entity: {} as no migration needed", entityGuid); } + + if (count == 20) { + LOG.info("Committing batch of 20 entities..."); + commitChanges(); + count = 0; + } } if (count > 0) { - LOG.info("Total Vertex updated: {}", count); + LOG.info("Committing remaining {} entities...", count); commitChanges(); } - else { - LOG.info("No changes to commit for entities as no migration needed"); - } + + LOG.info("Total Vertex updated: {}", totalUpdatedCount); } catch (Exception e) { LOG.error("Error while migration unique qualifiedName attribute for entities: {}", entityGuids, e); From 82e3448c06c8fac05932b42861fb252ecbb64e2c Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Dec 2024 13:12:08 +0530 Subject: [PATCH 232/241] mesh-292: changed constants --- .../main/java/org/apache/atlas/repository/Constants.java | 2 ++ .../v2/preprocessor/datamesh/DataDomainPreProcessor.java | 9 ++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 4fc62e0e6e..ff3741878b 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -39,6 +39,7 @@ import java.util.Set; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.encodePropertyKey; +import static org.apache.atlas.type.AtlasStructType.UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX; /** * Repository Constants. @@ -269,6 +270,7 @@ public final class Constants { public static final String NAME = "name"; public static final String QUALIFIED_NAME = "qualifiedName"; + public static final String UNIQUE_QUALIFIED_NAME = UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + QUALIFIED_NAME; public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName"; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index 88b0da188d..5a83f833b7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -53,7 +53,6 @@ public class DataDomainPreProcessor extends AbstractDomainPreProcessor { private Map updatedPolicyResources; private EntityGraphRetriever retrieverNoRelation = null; private Map updatedDomainQualifiedNames; - public static final String UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX = "__u_"; public DataDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { @@ -291,14 +290,12 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, String currentDomainQualifiedName = childDomainVertex.getProperty(QUALIFIED_NAME, String.class); String updatedDomainQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentDomainQualifiedName); - String uniqueQualifiedNameAttribute = UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + QUALIFIED_NAME; - // Change domain qualifiedName childDomainVertex.setProperty(QUALIFIED_NAME, updatedDomainQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedDomainQualifiedName); // Change unique qualifiedName attribute - childDomainVertex.setProperty(uniqueQualifiedNameAttribute, updatedDomainQualifiedName); + childDomainVertex.setProperty(UNIQUE_QUALIFIED_NAME, updatedDomainQualifiedName); //change superDomainQN, parentDomainQN childDomainVertex.setProperty(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); @@ -353,10 +350,8 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, String currentQualifiedName = productVertex.getProperty(QUALIFIED_NAME, String.class); String updatedQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentQualifiedName); - String uniqueQualifiedNameAttribute = UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + QUALIFIED_NAME; - productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); - productVertex.setProperty(uniqueQualifiedNameAttribute, updatedQualifiedName); + productVertex.setProperty(UNIQUE_QUALIFIED_NAME, updatedQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); productVertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); From 7f7f1b299f9cf2e98f3cee6f95f01f342e2f83fd Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 11 Dec 2024 14:40:34 +0530 Subject: [PATCH 233/241] fix duplicate --- .../main/java/org/apache/atlas/AtlasConfiguration.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index c9effd5f40..dbcacbf4ab 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -114,16 +114,15 @@ public enum AtlasConfiguration { HERACLES_API_SERVER_URL("atlas.heracles.api.service.url", "http://heracles-service.heracles.svc.cluster.local"), INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), + + /** + * hits elastic search async API + */ ENABLE_ASYNC_INDEXSEARCH("atlas.indexsearch.async.enable", false), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), FETCH_COLLAPSED_RESULT("atlas.indexsearch.fetch.collapsed.result", true), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), - ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", true), - /*** - * This configuration is used to enable fetching non primitive attributes in index search - */ - ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", true), /** * enables janus/cassandra optimization From 48938303ab87625bb89f61f579b9725525d394e0 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 11 Dec 2024 14:47:52 +0530 Subject: [PATCH 234/241] remove enable cache --- webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java | 1 - 1 file changed, 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index fc214fea3c..942ef5982e 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -899,7 +899,6 @@ public EntityMutationResponse createOrUpdate(AtlasEntitiesWithExtInfo entities, @QueryParam("replaceBusinessAttributes") @DefaultValue("false") boolean replaceBusinessAttributes, @QueryParam("overwriteBusinessAttributes") @DefaultValue("false") boolean isOverwriteBusinessAttributes) throws AtlasBaseException { AtlasPerfTracer perf = null; - RequestContext.get().setEnableCache(false); try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.createOrUpdate(entityCount=" + From 3e06228d08332f9cf7b3791ffe8ceea96bae06a5 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 11 Dec 2024 14:50:13 +0530 Subject: [PATCH 235/241] remove test tools --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index d60c540769..d3f77f5647 100755 --- a/build.sh +++ b/build.sh @@ -24,9 +24,9 @@ unzip -o keycloak-15.0.2.1.zip -d ~/.m2/repository/org echo "Maven Building" if [ "$1" == "build_without_dashboard" ]; then - mvn -pl '!test-tools,!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge,!dashboardv2,!dashboardv3' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipOverlay -DskipEnunciate=true package -Pdist + mvn -pl '!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge,!dashboardv2,!dashboardv3' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipOverlay -DskipEnunciate=true package -Pdist else - mvn -pl '!test-tools,!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipEnunciate=true package -Pdist + mvn -pl '!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipEnunciate=true package -Pdist fi echo "[DEBUG listing distro/target" From d8c00b0d18d22ba1e1c2e325912e4eb4eea3ff8f Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 11 Dec 2024 14:53:12 +0530 Subject: [PATCH 236/241] renove cache --- .../src/main/java/org/apache/atlas/RequestContext.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index c4a150a112..eb182c19f3 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -736,14 +736,6 @@ public String getRequestUri() { return this.requestUri; } - public void setEnableCache(boolean cacheEnabled) { - this.cacheEnabled = cacheEnabled; - } - - public boolean isCacheEnabled() { - return this.cacheEnabled; - } - public boolean isIncludeClassificationNames() { return includeClassificationNames; } From 7630c8ca13b3f86674601f74e4529a393f5d6a16 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 11 Dec 2024 14:56:22 +0530 Subject: [PATCH 237/241] remove scope --- webapp/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/webapp/pom.xml b/webapp/pom.xml index 05a97f200a..3dd213b5c5 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -552,7 +552,6 @@ org.apache.atlas atlas-testtools ${project.version} - test From 9c9a2f5af1f6cee060a2218d015df4090aa77d74 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 11 Dec 2024 17:30:18 +0530 Subject: [PATCH 238/241] fix access utils --- .../org/apache/atlas/repository/util/AccessControlUtils.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index f4b72f42f6..9ec45e7db7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -65,11 +65,6 @@ public final class AccessControlUtils { public static final String ATTR_SERVICE_LAST_SYNC = "authServicePolicyLastSync"; public static final String ATTR_PURPOSE_CLASSIFICATIONS = "purposeClassifications"; - - public static final String ATTR_SERVICE_SERVICE_TYPE = "authServiceType"; - public static final String ATTR_SERVICE_TAG_SERVICE = "tagService"; - public static final String ATTR_SERVICE_IS_ENABLED = "authServiceIsEnabled"; - public static final String ATTR_SERVICE_LAST_SYNC = "authServicePolicyLastSync"; public static final String ATTR_POLICY_TYPE = "policyType"; public static final String ATTR_POLICY_USERS = "policyUsers"; public static final String ATTR_POLICY_GROUPS = "policyGroups"; From 50c9804db3c4c2e58cee145151818ff068105dc3 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 11 Dec 2024 17:41:26 +0530 Subject: [PATCH 239/241] fix imports --- .../src/main/java/org/apache/atlas/tasks/TaskRegistry.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java index 24f698656b..73a7ee3147 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskRegistry.java @@ -17,8 +17,8 @@ */ package org.apache.atlas.tasks; -import com.datastax.oss.driver.shaded.fasterxml.jackson.core.JsonProcessingException; -import com.datastax.oss.driver.shaded.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.GraphTransaction; From 9b6fb716d5736263e90cde1b5a8dd6050ab680b2 Mon Sep 17 00:00:00 2001 From: abhijeet-atlan Date: Wed, 11 Dec 2024 18:12:45 +0530 Subject: [PATCH 240/241] Update maven.yml --- .github/workflows/maven.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 7d63cdff0b..da9e4b9d2b 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,10 +25,7 @@ on: - beta - development - master - - lineageondemand - taskdg1924 - - dg1908 - - ns/fix/delta-refresh jobs: build: @@ -69,7 +66,7 @@ jobs: - name: Build with Maven run: | branch_name=${{ steps.get_branch.outputs.branch }} - if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'taskdg1924' || $branch_name == 'dg1908' ]] + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'taskdg1924']] then echo "build without dashboard" chmod +x ./build.sh && ./build.sh build_without_dashboard From 7a71c1415a2df089bef72e440c16380cb4a50eca Mon Sep 17 00:00:00 2001 From: abhijeet-atlan Date: Wed, 11 Dec 2024 18:14:47 +0530 Subject: [PATCH 241/241] Update maven.yml --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index da9e4b9d2b..d4346667cf 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -66,7 +66,7 @@ jobs: - name: Build with Maven run: | branch_name=${{ steps.get_branch.outputs.branch }} - if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'taskdg1924']] + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'taskdg1924' ]] then echo "build without dashboard" chmod +x ./build.sh && ./build.sh build_without_dashboard