From 7c5d99e7e2d67bd632d2e40efa9372210bd88b40 Mon Sep 17 00:00:00 2001 From: Abhijeet Kumar Date: Thu, 23 Jan 2025 19:00:42 +0530 Subject: [PATCH] pushed fixes --- .github/workflows/maven.yml | 3 +- .../store/graph/v1/DeleteHandlerV1.java | 5 +- .../store/graph/v2/EntityGraphMapper.java | 152 +++++++++--------- 3 files changed, 82 insertions(+), 78 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 57788b53ad..8f601c64ef 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -29,6 +29,7 @@ on: - lineageondemand - makerlogic - taskdg1924deleteprop + - tagpropv1master jobs: build: @@ -68,7 +69,7 @@ jobs: - name: Build with Maven run: | branch_name=${{ steps.get_branch.outputs.branch }} - if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'taskdg1924deleteprop' ]] + if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'tagpropv1master' ]] then echo "build without dashboard" chmod +x ./build.sh && ./build.sh build_without_dashboard diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index e867d346f6..5754b10237 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -1220,8 +1220,11 @@ public void updateTagPropagations(AtlasEdge edge, AtlasRelationship relationship } } + // update the 'assetsCountToPropagate' on in memory java object. AtlasTask currentTask = RequestContext.get().getCurrentTask(); - currentTask.setAssetsCountToPropagate((long) (addPropagationsMap.size() + removePropagationsMap.size())); + currentTask.setAssetsCountToPropagate((long) addPropagationsMap.size() + removePropagationsMap.size() - 1); + + //update the 'assetsCountToPropagate' in the current task vertex. AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next(); currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, currentTask.getAssetsCountToPropagate()); graph.commit(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 3420b1516b..6eb2a73686 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -196,7 +196,7 @@ public class EntityGraphMapper { private final EntityGraphRetriever retrieverNoRelation; private static final Set excludedTypes = new HashSet<>(Arrays.asList(TYPE_GLOSSARY, TYPE_CATEGORY, TYPE_TERM, TYPE_PRODUCT, TYPE_DOMAIN)); - private static final Set edgeLabelsForHardDeletion = new HashSet<>(Arrays.asList(OUTPUT_PORT_PRODUCT_EDGE_LABEL, INPUT_PORT_PRODUCT_EDGE_LABEL)); + private static final Set edgeLabelsForHardDeletion = new HashSet<>(Arrays.asList(OUTPUT_PORT_PRODUCT_EDGE_LABEL, INPUT_PORT_PRODUCT_EDGE_LABEL, TERM_ASSIGNMENT_LABEL)); @Inject public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, AtlasGraph graph, @@ -1085,7 +1085,7 @@ private void mapAppendRemoveRelationshipAttributes(AtlasEntity entity, AtlasEnti MetricRecorder metric = RequestContext.get().startMetricRecord("mapAppendRemoveRelationshipAttributes"); if (isAppendOp && MapUtils.isNotEmpty(entity.getAppendRelationshipAttributes())) { - if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) { + if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) { // relationship attributes mapping for (String attrName : entityType.getRelationshipAttributes().keySet()) { if (entity.hasAppendRelationshipAttribute(attrName)) { @@ -1120,7 +1120,7 @@ private void mapAppendRemoveRelationshipAttributes(AtlasEntity entity, AtlasEnti } private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context) throws AtlasBaseException { - mapAttribute(attribute, attrValue, vertex, op, context, false, false); + mapAttribute(attribute, attrValue, vertex, op, context, false, false); } private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context, boolean isAppendOp, boolean isRemoveOp) throws AtlasBaseException { @@ -1268,7 +1268,7 @@ private Object mapToVertexByTypeCategory(AttributeMutationContext ctx, EntityMut } if (isRemoveOp){ - return removeArrayValue(ctx, context); + return removeArrayValue(ctx, context); } return mapArrayValue(ctx, context); @@ -1318,40 +1318,40 @@ private void addInverseReference(EntityMutationContext context, AtlasAttribute i boolean inverseUpdated = true; switch (inverseAttribute.getAttributeType().getTypeCategory()) { - case OBJECT_ID_TYPE: - if (inverseEdge != null) { - if (!inverseEdge.equals(newEdge)) { - // Disconnect old reference - deleteDelegate.getHandler().deleteEdgeReference(inverseEdge, inverseAttribute.getAttributeType().getTypeCategory(), - inverseAttribute.isOwnedRef(), true, inverseVertex); - } - else { - // Edge already exists for this attribute between these vertices. - inverseUpdated = false; + case OBJECT_ID_TYPE: + if (inverseEdge != null) { + if (!inverseEdge.equals(newEdge)) { + // Disconnect old reference + deleteDelegate.getHandler().deleteEdgeReference(inverseEdge, inverseAttribute.getAttributeType().getTypeCategory(), + inverseAttribute.isOwnedRef(), true, inverseVertex); + } + else { + // Edge already exists for this attribute between these vertices. + inverseUpdated = false; + } } - } - break; - case ARRAY: - // Add edge ID to property value - List elements = inverseVertex.getProperty(propertyName, List.class); - if (newEdge != null && elements == null) { - elements = new ArrayList<>(); - elements.add(newEdge.getId().toString()); - inverseVertex.setProperty(propertyName, elements); - } - else { - if (newEdge != null && !elements.contains(newEdge.getId().toString())) { + break; + case ARRAY: + // Add edge ID to property value + List elements = inverseVertex.getProperty(propertyName, List.class); + if (newEdge != null && elements == null) { + elements = new ArrayList<>(); elements.add(newEdge.getId().toString()); inverseVertex.setProperty(propertyName, elements); - } - else { - // Property value list already contains the edge ID. - inverseUpdated = false; - } - } - break; - default: - break; + } + else { + if (newEdge != null && !elements.contains(newEdge.getId().toString())) { + elements.add(newEdge.getId().toString()); + inverseVertex.setProperty(propertyName, elements); + } + else { + // Property value list already contains the edge ID. + inverseUpdated = false; + } + } + break; + default: + break; } if (inverseUpdated) { @@ -1387,7 +1387,7 @@ private AtlasEdge createInverseReferenceUsingRelationship(EntityMutationContext } else { if (LOG.isDebugEnabled()) { LOG.debug("No RelationshipDef defined between {} and {} on attribute: {}", inverseAttributeType, - AtlasGraphUtilsV2.getTypeName(vertex), inverseAttributeName); + AtlasGraphUtilsV2.getTypeName(vertex), inverseAttributeName); } // if no RelationshipDef found, use legacy way to create edges ret = createInverseReference(inverseAttribute, (AtlasStructType) inverseAttributeType, inverseVertex, vertex); @@ -1765,7 +1765,7 @@ private AtlasEdge getEdgeUsingRelationship(AttributeMutationContext ctx, EntityM AtlasRelationship relationship = new AtlasRelationship(relationshipName, relationshipAttributes); if (createEdge) { - edge = relationshipStore.getOrCreate(fromVertex, toVertex, relationship); + edge = relationshipStore.getOrCreate(fromVertex, toVertex, relationship, false); boolean isCreated = graphHelper.getCreatedTime(edge) == RequestContext.get().getRequestTime(); if (isCreated) { @@ -1832,7 +1832,7 @@ private Map mapMapValue(AttributeMutationContext ctx, EntityMuta AtlasEdge existingEdge = isSoftReference ? null : getEdgeIfExists(mapType, currentMap, key); AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(), - propertyName, mapType.getValueType(), existingEdge); + propertyName, mapType.getValueType(), existingEdge); // Add/Update/Remove property value Object newEntry = mapCollectionElementsToVertex(mapCtx, context); @@ -1896,7 +1896,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType(); AtlasType elementType = arrType.getElementType(); boolean isStructType = (TypeCategory.STRUCT == elementType.getTypeCategory()) || - (TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory()); + (TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory()); boolean isReference = isReference(elementType); boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); @@ -1938,7 +1938,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co for (int index = 0; index < newElements.size(); index++) { AtlasEdge existingEdge = (isSoftReference) ? null : getEdgeAt(currentElements, index, elementType); AttributeMutationContext arrCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), ctx.getAttribute(), newElements.get(index), - ctx.getVertexProperty(), elementType, existingEdge); + ctx.getVertexProperty(), elementType, existingEdge); if (deleteExistingRelations) { removeExistingRelationWithOtherVertex(arrCtx, ctx, context); } @@ -1972,11 +1972,11 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co } // add index to attributes of array type - for (int index = 0; allArrayElements != null && index < allArrayElements.size(); index++) { - Object element = allArrayElements.get(index); + for (int index = 0; allArrayElements != null && index < allArrayElements.size(); index++) { + Object element = allArrayElements.get(index); - if (element instanceof AtlasEdge) { - AtlasGraphUtilsV2.setEncodedProperty((AtlasEdge) element, ATTRIBUTE_INDEX_PROPERTY_KEY, index); + if (element instanceof AtlasEdge) { + AtlasGraphUtilsV2.setEncodedProperty((AtlasEdge) element, ATTRIBUTE_INDEX_PROPERTY_KEY, index); } } @@ -2323,7 +2323,7 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c addOrRemoveDaapInternalAttr(toVertex, attrName, createdElements, deletedElements, currentElements); }else{ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Can not update product relations while updating any asset"); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Can not update product relations while updating any asset"); } RequestContext.get().endMetricRecord(metricRecorder); } @@ -2369,18 +2369,18 @@ private boolean shouldDeleteExistingRelations(AttributeMutationContext ctx, Atla } /* - * Before creating new edges between referring vertex & new vertex coming from array, - * delete old relationship with same relationship type between new vertex coming from array & any other vertex. - * e.g - * table_a has columns as col_0 & col_1 - * create new table_b add columns col_0 & col_1 - * Now creating new relationships between table_b -> col_0 & col_1 - * This should also delete existing relationships between table_a -> col_0 & col_1 - * this behaviour is needed because endDef1 has SINGLE cardinality - * - * This method will delete existing edges. - * Skip if both ends are of SET cardinality, e.g. Catalog.inputs, Catalog.outputs - * */ + * Before creating new edges between referring vertex & new vertex coming from array, + * delete old relationship with same relationship type between new vertex coming from array & any other vertex. + * e.g + * table_a has columns as col_0 & col_1 + * create new table_b add columns col_0 & col_1 + * Now creating new relationships between table_b -> col_0 & col_1 + * This should also delete existing relationships between table_a -> col_0 & col_1 + * this behaviour is needed because endDef1 has SINGLE cardinality + * + * This method will delete existing edges. + * Skip if both ends are of SET cardinality, e.g. Catalog.inputs, Catalog.outputs + * */ private void removeExistingRelationWithOtherVertex(AttributeMutationContext arrCtx, AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException { MetricRecorder metric = RequestContext.get().startMetricRecord("removeExistingRelationWithOtherVertex"); @@ -2664,25 +2664,25 @@ private AtlasEntityType getEntityType(String typeName) throws AtlasBaseException private Object mapCollectionElementsToVertex(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException { switch(ctx.getAttrType().getTypeCategory()) { - case PRIMITIVE: - case ENUM: - case MAP: - case ARRAY: - return ctx.getValue(); + case PRIMITIVE: + case ENUM: + case MAP: + case ARRAY: + return ctx.getValue(); - case STRUCT: - return mapStructValue(ctx, context); + case STRUCT: + return mapStructValue(ctx, context); - case OBJECT_ID_TYPE: - AtlasEntityType instanceType = getInstanceType(ctx.getValue(), context); - ctx.setElementType(instanceType); - if (ctx.getAttributeDef().isSoftReferenced()) { - return mapSoftRefValue(ctx, context); - } + case OBJECT_ID_TYPE: + AtlasEntityType instanceType = getInstanceType(ctx.getValue(), context); + ctx.setElementType(instanceType); + if (ctx.getAttributeDef().isSoftReferenced()) { + return mapSoftRefValue(ctx, context); + } - return mapObjectIdValueUsingRelationship(ctx, context); + return mapObjectIdValueUsingRelationship(ctx, context); - default: + default: throw new AtlasBaseException(AtlasErrorCode.TYPE_CATEGORY_INVALID, ctx.getAttrType().getTypeCategory().name()); } } @@ -3500,7 +3500,7 @@ public List propagateClassification(String entityGuid, String classifica List edgeLabelsToCheck = CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode); Boolean toExclude = propagationMode == CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ? true:false; List impactedVertices = entityRetriever.getIncludedImpactedVerticesV2(entityVertex, relationshipGuid, classificationVertexId, edgeLabelsToCheck,toExclude); - + if (CollectionUtils.isEmpty(impactedVertices)) { LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", entityGuid, classificationVertexId); @@ -3571,7 +3571,7 @@ public List processClassificationPropagationAddition(List v RequestContext.get().endMetricRecord(classificationPropagationMetricRecorder); } - return propagatedEntitiesGuids; + return propagatedEntitiesGuids; } @@ -4043,7 +4043,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li private AtlasEdge mapClassification(EntityOperation operation, final EntityMutationContext context, AtlasClassification classification, AtlasEntityType entityType, AtlasVertex parentInstanceVertex, AtlasVertex traitInstanceVertex) - throws AtlasBaseException { + throws AtlasBaseException { if (classification.getValidityPeriods() != null) { String strValidityPeriods = AtlasJson.toJson(classification.getValidityPeriods()); @@ -4496,7 +4496,7 @@ private void validateClassificationExists(List existingClassifications, private AtlasEdge getOrCreateRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, String relationshipName, Map relationshipAttributes) throws AtlasBaseException { - return relationshipStore.getOrCreate(end1Vertex, end2Vertex, new AtlasRelationship(relationshipName, relationshipAttributes)); + return relationshipStore.getOrCreate(end1Vertex, end2Vertex, new AtlasRelationship(relationshipName, relationshipAttributes), false); } private void recordEntityUpdate(AtlasVertex vertex) throws AtlasBaseException {