Skip to content

Commit

Permalink
pushed fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@abhijeet-atlan
abhijeet-atlan committed Jan 23, 2025
1 parent f8b7eb9 commit 7c5d99e
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 78 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/maven.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ on:
- lineageondemand
- makerlogic
- taskdg1924deleteprop
- tagpropv1master

jobs:
build:
Expand Down Expand Up @@ -68,7 +69,7 @@ jobs:
- name: Build with Maven
run: |
branch_name=${{ steps.get_branch.outputs.branch }}
if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'taskdg1924deleteprop' ]]
if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'tagpropv1master' ]]
then
echo "build without dashboard"
chmod +x ./build.sh && ./build.sh build_without_dashboard
Expand Down
5 changes: 4 additions & 1 deletion repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1220,8 +1220,11 @@ public void updateTagPropagations(AtlasEdge edge, AtlasRelationship relationship
}
}

// update the 'assetsCountToPropagate' on in memory java object.
AtlasTask currentTask = RequestContext.get().getCurrentTask();
currentTask.setAssetsCountToPropagate((long) (addPropagationsMap.size() + removePropagationsMap.size()));
currentTask.setAssetsCountToPropagate((long) addPropagationsMap.size() + removePropagationsMap.size() - 1);

//update the 'assetsCountToPropagate' in the current task vertex.
AtlasVertex currentTaskVertex = (AtlasVertex) graph.query().has(TASK_GUID, currentTask.getGuid()).vertices().iterator().next();
currentTaskVertex.setProperty(TASK_ASSET_COUNT_TO_PROPAGATE, currentTask.getAssetsCountToPropagate());
graph.commit();
Expand Down
152 changes: 76 additions & 76 deletions repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public class EntityGraphMapper {
private final EntityGraphRetriever retrieverNoRelation;

private static final Set<String> excludedTypes = new HashSet<>(Arrays.asList(TYPE_GLOSSARY, TYPE_CATEGORY, TYPE_TERM, TYPE_PRODUCT, TYPE_DOMAIN));
private static final Set<String> edgeLabelsForHardDeletion = new HashSet<>(Arrays.asList(OUTPUT_PORT_PRODUCT_EDGE_LABEL, INPUT_PORT_PRODUCT_EDGE_LABEL));
private static final Set<String> edgeLabelsForHardDeletion = new HashSet<>(Arrays.asList(OUTPUT_PORT_PRODUCT_EDGE_LABEL, INPUT_PORT_PRODUCT_EDGE_LABEL, TERM_ASSIGNMENT_LABEL));

@Inject
public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, AtlasGraph graph,
Expand Down Expand Up @@ -1085,7 +1085,7 @@ private void mapAppendRemoveRelationshipAttributes(AtlasEntity entity, AtlasEnti
MetricRecorder metric = RequestContext.get().startMetricRecord("mapAppendRemoveRelationshipAttributes");

if (isAppendOp && MapUtils.isNotEmpty(entity.getAppendRelationshipAttributes())) {
if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) {
if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) {
// relationship attributes mapping
for (String attrName : entityType.getRelationshipAttributes().keySet()) {
if (entity.hasAppendRelationshipAttribute(attrName)) {
Expand Down Expand Up @@ -1120,7 +1120,7 @@ private void mapAppendRemoveRelationshipAttributes(AtlasEntity entity, AtlasEnti
}

private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context) throws AtlasBaseException {
mapAttribute(attribute, attrValue, vertex, op, context, false, false);
mapAttribute(attribute, attrValue, vertex, op, context, false, false);
}

private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context, boolean isAppendOp, boolean isRemoveOp) throws AtlasBaseException {
Expand Down Expand Up @@ -1268,7 +1268,7 @@ private Object mapToVertexByTypeCategory(AttributeMutationContext ctx, EntityMut
}

if (isRemoveOp){
return removeArrayValue(ctx, context);
return removeArrayValue(ctx, context);
}

return mapArrayValue(ctx, context);
Expand Down Expand Up @@ -1318,40 +1318,40 @@ private void addInverseReference(EntityMutationContext context, AtlasAttribute i

boolean inverseUpdated = true;
switch (inverseAttribute.getAttributeType().getTypeCategory()) {
case OBJECT_ID_TYPE:
if (inverseEdge != null) {
if (!inverseEdge.equals(newEdge)) {
// Disconnect old reference
deleteDelegate.getHandler().deleteEdgeReference(inverseEdge, inverseAttribute.getAttributeType().getTypeCategory(),
inverseAttribute.isOwnedRef(), true, inverseVertex);
}
else {
// Edge already exists for this attribute between these vertices.
inverseUpdated = false;
case OBJECT_ID_TYPE:
if (inverseEdge != null) {
if (!inverseEdge.equals(newEdge)) {
// Disconnect old reference
deleteDelegate.getHandler().deleteEdgeReference(inverseEdge, inverseAttribute.getAttributeType().getTypeCategory(),
inverseAttribute.isOwnedRef(), true, inverseVertex);
}
else {
// Edge already exists for this attribute between these vertices.
inverseUpdated = false;
}
}
}
break;
case ARRAY:
// Add edge ID to property value
List<String> elements = inverseVertex.getProperty(propertyName, List.class);
if (newEdge != null && elements == null) {
elements = new ArrayList<>();
elements.add(newEdge.getId().toString());
inverseVertex.setProperty(propertyName, elements);
}
else {
if (newEdge != null && !elements.contains(newEdge.getId().toString())) {
break;
case ARRAY:
// Add edge ID to property value
List<String> elements = inverseVertex.getProperty(propertyName, List.class);
if (newEdge != null && elements == null) {
elements = new ArrayList<>();
elements.add(newEdge.getId().toString());
inverseVertex.setProperty(propertyName, elements);
}
else {
// Property value list already contains the edge ID.
inverseUpdated = false;
}
}
break;
default:
break;
}
else {
if (newEdge != null && !elements.contains(newEdge.getId().toString())) {
elements.add(newEdge.getId().toString());
inverseVertex.setProperty(propertyName, elements);
}
else {
// Property value list already contains the edge ID.
inverseUpdated = false;
}
}
break;
default:
break;
}

if (inverseUpdated) {
Expand Down Expand Up @@ -1387,7 +1387,7 @@ private AtlasEdge createInverseReferenceUsingRelationship(EntityMutationContext
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("No RelationshipDef defined between {} and {} on attribute: {}", inverseAttributeType,
AtlasGraphUtilsV2.getTypeName(vertex), inverseAttributeName);
AtlasGraphUtilsV2.getTypeName(vertex), inverseAttributeName);
}
// if no RelationshipDef found, use legacy way to create edges
ret = createInverseReference(inverseAttribute, (AtlasStructType) inverseAttributeType, inverseVertex, vertex);
Expand Down Expand Up @@ -1765,7 +1765,7 @@ private AtlasEdge getEdgeUsingRelationship(AttributeMutationContext ctx, EntityM
AtlasRelationship relationship = new AtlasRelationship(relationshipName, relationshipAttributes);

if (createEdge) {
edge = relationshipStore.getOrCreate(fromVertex, toVertex, relationship);
edge = relationshipStore.getOrCreate(fromVertex, toVertex, relationship, false);
boolean isCreated = graphHelper.getCreatedTime(edge) == RequestContext.get().getRequestTime();

if (isCreated) {
Expand Down Expand Up @@ -1832,7 +1832,7 @@ private Map<String, Object> mapMapValue(AttributeMutationContext ctx, EntityMuta
AtlasEdge existingEdge = isSoftReference ? null : getEdgeIfExists(mapType, currentMap, key);

AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(),
propertyName, mapType.getValueType(), existingEdge);
propertyName, mapType.getValueType(), existingEdge);
// Add/Update/Remove property value
Object newEntry = mapCollectionElementsToVertex(mapCtx, context);

Expand Down Expand Up @@ -1896,7 +1896,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co
AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType();
AtlasType elementType = arrType.getElementType();
boolean isStructType = (TypeCategory.STRUCT == elementType.getTypeCategory()) ||
(TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory());
(TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory());
boolean isReference = isReference(elementType);
boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced();
AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
Expand Down Expand Up @@ -1938,7 +1938,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co
for (int index = 0; index < newElements.size(); index++) {
AtlasEdge existingEdge = (isSoftReference) ? null : getEdgeAt(currentElements, index, elementType);
AttributeMutationContext arrCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), ctx.getAttribute(), newElements.get(index),
ctx.getVertexProperty(), elementType, existingEdge);
ctx.getVertexProperty(), elementType, existingEdge);
if (deleteExistingRelations) {
removeExistingRelationWithOtherVertex(arrCtx, ctx, context);
}
Expand Down Expand Up @@ -1972,11 +1972,11 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co
}

// add index to attributes of array type
for (int index = 0; allArrayElements != null && index < allArrayElements.size(); index++) {
Object element = allArrayElements.get(index);
for (int index = 0; allArrayElements != null && index < allArrayElements.size(); index++) {
Object element = allArrayElements.get(index);

if (element instanceof AtlasEdge) {
AtlasGraphUtilsV2.setEncodedProperty((AtlasEdge) element, ATTRIBUTE_INDEX_PROPERTY_KEY, index);
if (element instanceof AtlasEdge) {
AtlasGraphUtilsV2.setEncodedProperty((AtlasEdge) element, ATTRIBUTE_INDEX_PROPERTY_KEY, index);
}
}

Expand Down Expand Up @@ -2323,7 +2323,7 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List<Object> c

addOrRemoveDaapInternalAttr(toVertex, attrName, createdElements, deletedElements, currentElements);
}else{
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Can not update product relations while updating any asset");
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Can not update product relations while updating any asset");
}
RequestContext.get().endMetricRecord(metricRecorder);
}
Expand Down Expand Up @@ -2369,18 +2369,18 @@ private boolean shouldDeleteExistingRelations(AttributeMutationContext ctx, Atla
}

/*
* Before creating new edges between referring vertex & new vertex coming from array,
* delete old relationship with same relationship type between new vertex coming from array & any other vertex.
* e.g
* table_a has columns as col_0 & col_1
* create new table_b add columns col_0 & col_1
* Now creating new relationships between table_b -> col_0 & col_1
* This should also delete existing relationships between table_a -> col_0 & col_1
* this behaviour is needed because endDef1 has SINGLE cardinality
*
* This method will delete existing edges.
* Skip if both ends are of SET cardinality, e.g. Catalog.inputs, Catalog.outputs
* */
* Before creating new edges between referring vertex & new vertex coming from array,
* delete old relationship with same relationship type between new vertex coming from array & any other vertex.
* e.g
* table_a has columns as col_0 & col_1
* create new table_b add columns col_0 & col_1
* Now creating new relationships between table_b -> col_0 & col_1
* This should also delete existing relationships between table_a -> col_0 & col_1
* this behaviour is needed because endDef1 has SINGLE cardinality
*
* This method will delete existing edges.
* Skip if both ends are of SET cardinality, e.g. Catalog.inputs, Catalog.outputs
* */
private void removeExistingRelationWithOtherVertex(AttributeMutationContext arrCtx, AttributeMutationContext ctx,
EntityMutationContext context) throws AtlasBaseException {
MetricRecorder metric = RequestContext.get().startMetricRecord("removeExistingRelationWithOtherVertex");
Expand Down Expand Up @@ -2664,25 +2664,25 @@ private AtlasEntityType getEntityType(String typeName) throws AtlasBaseException

private Object mapCollectionElementsToVertex(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException {
switch(ctx.getAttrType().getTypeCategory()) {
case PRIMITIVE:
case ENUM:
case MAP:
case ARRAY:
return ctx.getValue();
case PRIMITIVE:
case ENUM:
case MAP:
case ARRAY:
return ctx.getValue();

case STRUCT:
return mapStructValue(ctx, context);
case STRUCT:
return mapStructValue(ctx, context);

case OBJECT_ID_TYPE:
AtlasEntityType instanceType = getInstanceType(ctx.getValue(), context);
ctx.setElementType(instanceType);
if (ctx.getAttributeDef().isSoftReferenced()) {
return mapSoftRefValue(ctx, context);
}
case OBJECT_ID_TYPE:
AtlasEntityType instanceType = getInstanceType(ctx.getValue(), context);
ctx.setElementType(instanceType);
if (ctx.getAttributeDef().isSoftReferenced()) {
return mapSoftRefValue(ctx, context);
}

return mapObjectIdValueUsingRelationship(ctx, context);
return mapObjectIdValueUsingRelationship(ctx, context);

default:
default:
throw new AtlasBaseException(AtlasErrorCode.TYPE_CATEGORY_INVALID, ctx.getAttrType().getTypeCategory().name());
}
}
Expand Down Expand Up @@ -3500,7 +3500,7 @@ public List<String> propagateClassification(String entityGuid, String classifica
List<String> edgeLabelsToCheck = CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode);
Boolean toExclude = propagationMode == CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ? true:false;
List<AtlasVertex> impactedVertices = entityRetriever.getIncludedImpactedVerticesV2(entityVertex, relationshipGuid, classificationVertexId, edgeLabelsToCheck,toExclude);

if (CollectionUtils.isEmpty(impactedVertices)) {
LOG.debug("propagateClassification(entityGuid={}, classificationVertexId={}): found no entities to propagate the classification", entityGuid, classificationVertexId);

Expand Down Expand Up @@ -3571,7 +3571,7 @@ public List<String> processClassificationPropagationAddition(List<AtlasVertex> v
RequestContext.get().endMetricRecord(classificationPropagationMetricRecorder);
}

return propagatedEntitiesGuids;
return propagatedEntitiesGuids;

}

Expand Down Expand Up @@ -4043,7 +4043,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li

private AtlasEdge mapClassification(EntityOperation operation, final EntityMutationContext context, AtlasClassification classification,
AtlasEntityType entityType, AtlasVertex parentInstanceVertex, AtlasVertex traitInstanceVertex)
throws AtlasBaseException {
throws AtlasBaseException {
if (classification.getValidityPeriods() != null) {
String strValidityPeriods = AtlasJson.toJson(classification.getValidityPeriods());

Expand Down Expand Up @@ -4496,7 +4496,7 @@ private void validateClassificationExists(List<String> existingClassifications,

private AtlasEdge getOrCreateRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, String relationshipName,
Map<String, Object> relationshipAttributes) throws AtlasBaseException {
return relationshipStore.getOrCreate(end1Vertex, end2Vertex, new AtlasRelationship(relationshipName, relationshipAttributes));
return relationshipStore.getOrCreate(end1Vertex, end2Vertex, new AtlasRelationship(relationshipName, relationshipAttributes), false);
}

private void recordEntityUpdate(AtlasVertex vertex) throws AtlasBaseException {
Expand Down

0 comments on commit 7c5d99e

Please sign in to comment.