Merge branch 'main' into gvfs-pr11

apache · Feb 7, 2025 · 6d1d572 · 6d1d572
2 parents 676db20 + e85e47d
commit 6d1d572
Show file tree

Hide file tree

Showing 188 changed files with 4,753 additions and 726 deletions.
diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml
@@ -92,7 +92,7 @@ jobs:
           ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: authorizations-integrate-test-reports-${{ matrix.java-version }}

diff --git a/.github/workflows/backend-integration-test-action.yml b/.github/workflows/backend-integration-test-action.yml
@@ -64,7 +64,7 @@ jobs:
           -x :authorizations:authorization-chain:test -x :authorizations:authorization-ranger:test 
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -90,7 +90,7 @@ jobs:
           ./gradlew :spark-connector:spark-3.5:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false
 
       - name: Upload unit tests report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: unit test report
@@ -129,7 +129,7 @@ jobs:
         run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false -x :clients:client-python:build
 
       - name: Upload unit tests report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: unit test report

diff --git a/.github/workflows/cron-integration-test.yml b/.github/workflows/cron-integration-test.yml
@@ -87,7 +87,7 @@ jobs:
           ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
         with:
           name: integrate test reports

diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -32,6 +32,11 @@ on:
         description: 'Publish Docker token'
         required: true
         type: string
+      publish-latest-tag:
+        description: 'Whether to update the latest tag. This operation is only applicable to official releases and should not be used for Release Candidate (RC).'
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   publish-docker-image:
@@ -83,6 +88,12 @@ jobs:
             echo "image_type=iceberg-rest-server" >> $GITHUB_ENV
             echo "image_name=apache/gravitino-iceberg-rest" >> $GITHUB_ENV
           fi
+          
+          if [ "${{ github.event.inputs.publish-latest-tag }}" == "true" ]; then
+            echo "publish_latest=true" >> $GITHUB_ENV
+          else
+            echo "publish_latest=false" >> $GITHUB_ENV
+          fi 
 
       - name: Check publish Docker token
         run: |
@@ -115,8 +126,16 @@ jobs:
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /opt/hostedtoolcache/CodeQL
           
-          if [[ "${image_type}" == "gravitino" || "${image_type}" == "iceberg-rest-server" ]]; then
-            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${{ github.event.inputs.version }} --latest
+          if [[ -n "${tag_name}" ]]; then
+            full_tag_name="${tag_name}-${{ github.event.inputs.version }}"
+          else
+            full_tag_name="${{ github.event.inputs.version }}"
+          fi
+          
+          if [[ "${publish_latest}" == "true" ]]; then
+            echo "Publish tag ${full_tag_name}, and update latest too."
+            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${full_tag_name} --latest
           else
-            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag "${tag_name}-${{ github.event.inputs.version }}"
-          fi
+            echo "Publish tag ${full_tag_name}."
+            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${full_tag_name}
+          fi
diff --git a/.github/workflows/flink-integration-test-action.yml b/.github/workflows/flink-integration-test-action.yml
@@ -52,7 +52,7 @@ jobs:
           ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ inputs.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: flink-connector-integrate-test-reports-${{ inputs.java-version }}

diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml
@@ -88,7 +88,7 @@ jobs:
           ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: integrate-test-reports-${{ matrix.java-version }}

diff --git a/.github/workflows/gvfs-fuse-build-test.yml b/.github/workflows/gvfs-fuse-build-test.yml
@@ -88,7 +88,7 @@ jobs:
           dev/ci/util_free_space.sh
 
       - name: Upload tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: Gvfs-fuse integrate-test-reports-${{ matrix.java-version }}

diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml
@@ -82,7 +82,7 @@ jobs:
           done
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
         with:
           name: integrate test reports

diff --git a/.github/workflows/spark-integration-test-action.yml b/.github/workflows/spark-integration-test-action.yml
@@ -63,7 +63,7 @@ jobs:
           ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PscalaVersion=${{ inputs.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: spark-connector-integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}

diff --git a/.github/workflows/trino-integration-test.yml b/.github/workflows/trino-integration-test.yml
@@ -90,7 +90,7 @@ jobs:
           trino-connector/integration-test/trino-test-tools/run_test.sh
 
       - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: trino-connector-integrate-test-reports-${{ matrix.java-version }}

diff --git a/.gitignore b/.gitignore
@@ -27,6 +27,7 @@ replay_pid*
 **/.gradle
 **/.idea
 !/.idea/icon.svg
+!.idea/vcs.xml
 **/build
 gen
 **/.DS_Store

diff --git a/api/src/main/java/org/apache/gravitino/MetadataObjects.java b/api/src/main/java/org/apache/gravitino/MetadataObjects.java
@@ -126,6 +126,7 @@ public static MetadataObject parent(MetadataObject object) {
       case TABLE:
       case FILESET:
       case TOPIC:
+      case MODEL:
         parentType = MetadataObject.Type.SCHEMA;
         break;
       case SCHEMA:

diff --git a/...in/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java b/...in/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java
@@ -33,7 +33,6 @@
 import org.apache.gravitino.authorization.Role;
 import org.apache.gravitino.authorization.RoleChange;
 import org.apache.gravitino.authorization.User;
-import org.apache.gravitino.authorization.common.AuthorizationProperties;
 import org.apache.gravitino.authorization.common.ChainedAuthorizationProperties;
 import org.apache.gravitino.connector.authorization.AuthorizationPlugin;
 import org.apache.gravitino.connector.authorization.BaseAuthorization;
@@ -55,16 +54,6 @@ private void initPlugins(String catalogProvider, Map<String, String> properties)
     ChainedAuthorizationProperties chainedAuthzProperties =
         new ChainedAuthorizationProperties(properties);
     chainedAuthzProperties.validate();
-    // Validate the properties for each plugin
-    chainedAuthzProperties
-        .plugins()
-        .forEach(
-            pluginName -> {
-              Map<String, String> pluginProperties =
-                  chainedAuthzProperties.fetchAuthPluginProperties(pluginName);
-              String authzProvider = chainedAuthzProperties.getPluginProvider(pluginName);
-              AuthorizationProperties.validate(authzProvider, pluginProperties);
-            });
     // Create the plugins
     chainedAuthzProperties
         .plugins()
@@ -83,8 +72,13 @@ private void initPlugins(String catalogProvider, Map<String, String> properties)
               try {
                 BaseAuthorization<?> authorization =
                     BaseAuthorization.createAuthorization(classLoader, authzProvider);
+
+                // Load the authorization plugin with the class loader of the catalog.
+                // Because the JDBC authorization plugin may load JDBC driver using the class
+                // loader.
                 AuthorizationPlugin authorizationPlugin =
-                    authorization.newPlugin(metalake, catalogProvider, pluginConfig);
+                    classLoader.withClassLoader(
+                        cl -> authorization.newPlugin(metalake, catalogProvider, pluginConfig));
                 plugins.add(authorizationPlugin);
               } catch (Exception e) {
                 throw new RuntimeException(e);

diff --git a/authorizations/authorization-common/build.gradle.kts b/authorizations/authorization-common/build.gradle.kts
@@ -49,6 +49,7 @@ dependencies {
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.mockito.core)
   testImplementation(libs.testcontainers)
+  testImplementation(libs.h2db)
   testRuntimeOnly(libs.junit.jupiter.engine)
 }
 

diff --git a/...mmon/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java b/...mmon/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java
@@ -34,21 +34,4 @@ public AuthorizationProperties(Map<String, String> properties) {
   public abstract String getPropertiesPrefix();
 
   public abstract void validate();
-
-  public static void validate(String type, Map<String, String> properties) {
-    switch (type) {
-      case "ranger":
-        RangerAuthorizationProperties rangerAuthorizationProperties =
-            new RangerAuthorizationProperties(properties);
-        rangerAuthorizationProperties.validate();
-        break;
-      case "chain":
-        ChainedAuthorizationProperties chainedAuthzProperties =
-            new ChainedAuthorizationProperties(properties);
-        chainedAuthzProperties.validate();
-        break;
-      default:
-        throw new IllegalArgumentException("Unsupported authorization properties type: " + type);
-    }
-  }
 }
diff --git a/...mmon/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java b/...mmon/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java
@@ -122,8 +122,13 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return "MetadataObject: [fullName=" + fullName() + "],  [path=" + path == null
-        ? "null"
-        : path + "], [type=" + type + "]";
+    String strPath = path == null ? "null" : path;
+    return "MetadataObject: [fullName="
+        + fullName()
+        + "],  [path="
+        + strPath
+        + "], [type="
+        + type
+        + "]";
   }
 }
diff --git a/...common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java b/...common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java
@@ -85,6 +85,13 @@ public JdbcAuthorizationPlugin(Map<String, String> config) {
     dataSource.setTestOnReturn(BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN);
     dataSource.setLifo(BaseObjectPoolConfig.DEFAULT_LIFO);
     mappingProvider = new JdbcSecurableObjectMappingProvider();
+
+    try (Connection ignored = getConnection()) {
+      LOG.info("Load the JDBC driver first");
+    } catch (SQLException se) {
+      LOG.error("Jdbc authorization plugin exception: ", se);
+      throw toAuthorizationPluginException(se);
+    }
   }
 
   @Override
@@ -110,8 +117,13 @@ public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws Runtime
   @Override
   public Boolean onRoleCreated(Role role) throws AuthorizationPluginException {
     List<String> sqls = getCreateRoleSQL(role.name());
+    boolean createdNewly = false;
     for (String sql : sqls) {
-      executeUpdateSQL(sql, "already exists");
+      createdNewly = executeUpdateSQL(sql, "already exists");
+    }
+
+    if (!createdNewly) {
+      return true;
     }
 
     if (role.securableObjects() != null) {
@@ -140,7 +152,6 @@ public Boolean onRoleDeleted(Role role) throws AuthorizationPluginException {
   @Override
   public Boolean onRoleUpdated(Role role, RoleChange... changes)
       throws AuthorizationPluginException {
-    onRoleCreated(role);
     for (RoleChange change : changes) {
       if (change instanceof RoleChange.AddSecurableObject) {
         SecurableObject object = ((RoleChange.AddSecurableObject) change).getSecurableObject();
@@ -381,14 +392,15 @@ protected AuthorizationPluginException toAuthorizationPluginException(SQLExcepti
         "JDBC authorization plugin fail to execute SQL, error code: %d", se.getErrorCode());
   }
 
-  public void executeUpdateSQL(String sql, String ignoreErrorMsg) {
+  public boolean executeUpdateSQL(String sql, String ignoreErrorMsg) {
     try (final Connection connection = getConnection()) {
       try (final Statement statement = connection.createStatement()) {
         statement.executeUpdate(sql);
+        return true;
       }
     } catch (SQLException se) {
       if (ignoreErrorMsg != null && se.getMessage().contains(ignoreErrorMsg)) {
-        return;
+        return false;
       }
       LOG.error("JDBC authorization plugin exception: ", se);
       throw toAuthorizationPluginException(se);

diff --git a/.../src/test/java/org/apache/gravitino/authorization/common/TestPathBasedMetadataObject.java b/.../src/test/java/org/apache/gravitino/authorization/common/TestPathBasedMetadataObject.java
@@ -47,4 +47,31 @@ public void PathBasedMetadataObjectNotEquals() {
 
     Assertions.assertNotEquals(pathBasedMetadataObject1, pathBasedMetadataObject2);
   }
+
+  @Test
+  void testToString() {
+    PathBasedMetadataObject pathBasedMetadataObject1 =
+        new PathBasedMetadataObject("parent", "name", "path", PathBasedMetadataObject.Type.PATH);
+    Assertions.assertEquals(
+        "MetadataObject: [fullName=parent.name],  [path=path], [type=PATH]",
+        pathBasedMetadataObject1.toString());
+
+    PathBasedMetadataObject pathBasedMetadataObject2 =
+        new PathBasedMetadataObject("parent", "name", null, PathBasedMetadataObject.Type.PATH);
+    Assertions.assertEquals(
+        "MetadataObject: [fullName=parent.name],  [path=null], [type=PATH]",
+        pathBasedMetadataObject2.toString());
+
+    PathBasedMetadataObject pathBasedMetadataObject3 =
+        new PathBasedMetadataObject(null, "name", null, PathBasedMetadataObject.Type.PATH);
+    Assertions.assertEquals(
+        "MetadataObject: [fullName=name],  [path=null], [type=PATH]",
+        pathBasedMetadataObject3.toString());
+
+    PathBasedMetadataObject pathBasedMetadataObject4 =
+        new PathBasedMetadataObject(null, "name", "path", PathBasedMetadataObject.Type.PATH);
+    Assertions.assertEquals(
+        "MetadataObject: [fullName=name],  [path=path], [type=PATH]",
+        pathBasedMetadataObject4.toString());
+  }
 }