Merge branch 'taskdg1924' into taskdg1924deleteprop

.github/workflows/maven.yml

@@ -25,7 +25,6 @@ on:
       - beta
       - development
       - master
-      - lineageondemand
       - taskdg1924
       - taskdg1924deleteprop
 
@@ -36,12 +35,12 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      
+
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
-      
+
       - name: Cache Maven packages
         uses: actions/cache@v2
         with:
@@ -50,8 +49,9 @@ jobs:
           restore-keys: ${{ runner.os }}-m2
 
       - name: Get branch name
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: get_branch
+        run: |
+          echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
+          echo BRANCH_NAME=${GITHUB_REF#refs/heads/}
 
       - name: Create Maven Settings
         uses: s4u/[email protected]
@@ -65,8 +65,9 @@ jobs:
 
       - name: Build with Maven
         run: |
+
           branch_name=${{ steps.get_branch.outputs.branch }} 
-          if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'taskdg1924deleteprop' ]]
+          if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'taskdg1924' ]]
           then
               echo "build without dashboard"
               chmod +x ./build.sh && ./build.sh build_without_dashboard
@@ -75,19 +76,26 @@ jobs:
               chmod +x ./build.sh && ./build.sh
           fi
 
-      - run:   echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV
+      - name: Get Repository Name
+        run:   echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV
         shell: bash
 
       - name: Get version tag
-        run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd"
-        id: get_version
+        # run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7`)abcd"
+        run: |
+          echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | head -n 1)abcd"
+          echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd"
+          echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" >> $GITHUB_ENV
+
+      - name: Get commit ID
+        run: echo "COMMIT_ID=$(echo ${GITHUB_SHA} | cut -c1-7)abcd" >> $GITHUB_ENV
 
-      - name: Set up Buildx 
+      - name: Set up Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
 
       - name: Login to GitHub Registry
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: $GITHUB_ACTOR
@@ -104,8 +112,8 @@ jobs:
           provenance: true
           push: true
           tags: |
-            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest
-            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }}
+            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:latest
+            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:${{ env.COMMIT_ID }}
 
       - name: Scan Image
         uses: aquasecurity/trivy-action@master
@@ -118,4 +126,4 @@ jobs:
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/[email protected]
         with:
-          sarif_file: 'trivy-image-results.sarif'
+          sarif_file: 'trivy-image-results.sarif'

README.txt

@@ -72,3 +72,5 @@ Build Process
    distro/target/apache-atlas-<version>-falcon-hook.tar.gz
 
 4. For more details on installing and running Apache Atlas, please refer to https://atlas.apache.org/#/Installation.
+
+

auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java

@@ -8,7 +8,7 @@
 public interface AtlasAuthAdminClient {
     void init(RangerPluginConfig config);
 
-    ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception;
+    ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws Exception;
 
     RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception;
 

auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java

@@ -32,6 +32,7 @@ public class AtlasAuthRESTClient implements AtlasAuthAdminClient {
 
     private static final String PARAM_LAST_UPDATED_TIME = "lastUpdatedTime";
     private static final String PARAM_PLUGIN_ID         = "pluginId";
+    private static final String PARAM_USE_POLICY_DELTA  = "usePolicyDelta";
 
     @Override
     public void init(RangerPluginConfig config) {
@@ -66,20 +67,20 @@ public String getPluginId(String serviceName, String appId) {
     }
 
     @Override
-    public ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception {
-        URI uri = buildURI("/download/policies/" + serviceName, lastUpdatedTimeInMillis);
+    public ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws Exception {
+        URI uri = buildURI("/download/policies/" + serviceName, lastUpdatedTimeInMillis, usePolicyDelta);
         return sendRequestAndGetResponse(uri, ServicePolicies.class);
     }
 
     @Override
     public RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception {
-        URI uri = buildURI("/download/roles/" + serviceName, lastUpdatedTimeInMillis);
+        URI uri = buildURI("/download/roles/" + serviceName, lastUpdatedTimeInMillis, false);
         return sendRequestAndGetResponse(uri, RangerRoles.class);
     }
 
     @Override
     public RangerUserStore getUserStoreIfUpdated(long lastUpdatedTimeInMillis) throws Exception {
-        URI uri = buildURI("/download/users/" + serviceName, lastUpdatedTimeInMillis);
+        URI uri = buildURI("/download/users/" + serviceName, lastUpdatedTimeInMillis, false);
         return sendRequestAndGetResponse(uri, RangerUserStore.class);
     }
 
@@ -114,13 +115,14 @@ private <T> T sendRequestAndGetResponse(URI uri, Class<T> responseClass) throws
         return null;
     }
 
-    private URI buildURI(String path, long lastUpdatedTimeInMillis) throws URISyntaxException {
+    private URI buildURI(String path, long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws URISyntaxException {
         return new URIBuilder()
                 .setScheme(SCHEME)
                 .setHost(adminUrl)
                 .setPath(path)
                 .setParameter(PARAM_LAST_UPDATED_TIME, String.valueOf(lastUpdatedTimeInMillis))
                 .setParameter(PARAM_PLUGIN_ID, pluginId)
+                .setParameter(PARAM_USE_POLICY_DELTA, String.valueOf(usePolicyDelta))
                 .build();
     }
 

auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java

@@ -19,8 +19,9 @@
 
 package org.apache.atlas.plugin.model;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -529,6 +530,11 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) {
 		this.isDenyAllElse = isDenyAllElse == null ? Boolean.FALSE : isDenyAllElse;
 	}
 
+	@JsonIgnore
+	public String getAtlasGuid() {
+		return getGuid().length() > 36 ? getGuid().substring(0, 36) : getGuid();
+	}
+
 	@Override
 	public String toString( ) {
 		StringBuilder sb = new StringBuilder();

auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java

@@ -19,8 +19,8 @@
 
 package org.apache.atlas.plugin.model;
 
-import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonIgnore;
-import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -73,6 +73,12 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol
     @JsonIgnore
     public Long getPolicyId() { return policy != null ? policy.getId() : null; }
 
+    @JsonIgnore
+    public String getPolicyGuid() { return policy != null ? policy.getGuid() : null; }
+
+    @JsonIgnore
+    public String getPolicyAtlasGuid() { return policy != null ? policy.getAtlasGuid() : null; }
+
     @JsonIgnore
     public String getZoneName() { return policy != null ? policy.getZoneName() : null; }
 
@@ -94,6 +100,7 @@ public String toString() {
                 + ", serviceType:" + getServiceType()
                 + ", policyType:" + getPolicyType()
                 + ", policyId:[" + getPolicyId() + "]"
+                + ", policyGuid:[" + getPolicyGuid() + "]"
                 + ", policy:[" + policy +"]";
     }
 

auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java

@@ -90,7 +90,7 @@ enum AuditModeEnum {
     private       List<RangerPolicyEvaluator>       dataMaskPolicyEvaluators;
     private       List<RangerPolicyEvaluator>       rowFilterPolicyEvaluators;
     private final List<RangerPolicyEvaluator>       auditPolicyEvaluators;
-    private       Map<Long, RangerPolicyEvaluator>  policyEvaluatorsMap;
+    private       Map<String, RangerPolicyEvaluator>  policyEvaluatorsMap;
     private       boolean                           isContextEnrichersShared = false;
     private       boolean                           isPreCleaned             = false;
 
@@ -654,9 +654,9 @@ public List<RangerPolicyEvaluator> getLikelyMatchPolicyEvaluators(RangerAccessRe
     }
 
 
-    public Map<Long, RangerPolicyEvaluator> getPolicyEvaluatorsMap() { return policyEvaluatorsMap; }
+    public Map<String, RangerPolicyEvaluator> getPolicyEvaluatorsMap() { return policyEvaluatorsMap; }
 
-    RangerPolicyEvaluator getPolicyEvaluator(Long id) {
+    RangerPolicyEvaluator getPolicyEvaluator(String id) {
         return policyEvaluatorsMap.get(id);
     }
 
@@ -1252,17 +1252,17 @@ private void removeEvaluatorFromTrie(RangerPolicyEvaluator oldEvaluator, RangerR
         }
     }
 
-    private Map<Long, RangerPolicyEvaluator> createPolicyEvaluatorsMap() {
-        Map<Long, RangerPolicyEvaluator> tmpPolicyEvaluatorMap = new HashMap<>();
+    private Map<String, RangerPolicyEvaluator> createPolicyEvaluatorsMap() {
+        Map<String, RangerPolicyEvaluator> tmpPolicyEvaluatorMap = new HashMap<>();
 
         for (RangerPolicyEvaluator evaluator : getPolicyEvaluators()) {
-            tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+            tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator);
         }
         for (RangerPolicyEvaluator evaluator : getDataMaskPolicyEvaluators()) {
-            tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+            tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator);
         }
         for (RangerPolicyEvaluator evaluator : getRowFilterPolicyEvaluators()) {
-            tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+            tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator);
         }
 
         return  tmpPolicyEvaluatorMap;
@@ -1294,7 +1294,7 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) {
                     }
 
                     if (!RangerPolicy.POLICY_TYPE_AUDIT.equals(policy.getPolicyType())) {
-                        policyEvaluatorsMap.put(policy.getId(), ret);
+                        policyEvaluatorsMap.put(policy.getGuid(), ret);
                     }
                 }
             }
@@ -1306,22 +1306,22 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) {
         return ret;
     }
 
-    private void removePolicy(Long id) {
+    private void removePolicy(String guid) {
         if (LOG.isDebugEnabled()) {
-            LOG.debug("==> RangerPolicyRepository.removePolicy(" + id +")");
+            LOG.debug("==> RangerPolicyRepository.removePolicy(" + guid +")");
         }
         Iterator<RangerPolicy> iterator = policies.iterator();
         while (iterator.hasNext()) {
-            if (id.equals(iterator.next().getId())) {
+            if (guid.equals(iterator.next().getGuid())) {
                 iterator.remove();
                 //break;
             }
         }
 
-        policyEvaluatorsMap.remove(id);
+        policyEvaluatorsMap.remove(guid);
 
         if (LOG.isDebugEnabled()) {
-            LOG.debug("<== RangerPolicyRepository.removePolicy(" + id +")");
+            LOG.debug("<== RangerPolicyRepository.removePolicy(" + guid +")");
         }
     }
 
@@ -1355,13 +1355,12 @@ private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) {
     }
 
     private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) {
-
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")");
         }
         Integer changeType = delta.getChangeType();
         String policyType  = delta.getPolicyType();
-        Long    policyId   = delta.getPolicyId();
+        String policyId    = delta.getPolicyGuid();
 
         RangerPolicy policy = delta.getPolicy();
 
@@ -1472,7 +1471,7 @@ private void updateResourceTrie(List<RangerPolicyDelta> deltas) {
         for (RangerPolicyDelta delta : deltas) {
             final Integer changeType  = delta.getChangeType();
             final String  serviceType = delta.getServiceType();
-            final Long    policyId    = delta.getPolicyId();
+            final String  policyId    = delta.getPolicyGuid();
             final String  policyType  = delta.getPolicyType();
 
             if (!serviceType.equals(this.serviceDef.getName())) {

auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java

@@ -314,7 +314,7 @@ public void setPolicies(ServicePolicies policies) {
 				Boolean hasPolicyDeltas = RangerPolicyDeltaUtil.hasPolicyDeltas(policies);
 
 				if (hasPolicyDeltas == null) {
-					LOG.info("Downloaded policies do not require policy change !! [" + policies + "]");
+					LOG.info("Downloaded policies do not require policy change !! [" + (policies.getPolicies() != null ? policies.getPolicies().size() : 0) + "]");
 
 					if (this.policyEngine == null) {
 
@@ -376,9 +376,8 @@ public void setPolicies(ServicePolicies policies) {
 						}
 
 						if (oldPolicyEngine != null) {
-							RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine;
-
-							newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies);
+							// Create new evaluator for the updated policies
+							newPolicyEngine = new RangerPolicyEngineImpl(servicePolicies, pluginContext, roles);
 						}
 
 						if (newPolicyEngine != null) {
@@ -429,7 +428,8 @@ public void setPolicies(ServicePolicies policies) {
 			}
 
 		} catch (Exception e) {
-			LOG.error("setPolicies: policy engine initialization failed!  Leaving current policy engine as-is. Exception : ", e);
+			LOG.error("setPolicies: Failed to set policies, didn't set policies", e);
+			throw e;
 		}
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("<== setPolicies(" + policies + ")");