Fixed property overrides placement to correct cluster.

cloudera-labs · Nov 12, 2024 · 4a649aa · 4a649aa
1 parent e22cd41
commit 4a649aa
Show file tree

Hide file tree

Showing 7 changed files with 36 additions and 30 deletions.
diff --git a/pom.xml b/pom.xml
@@ -28,7 +28,7 @@
 
     <groupId>com.cloudera.utils.hadoop</groupId>
     <artifactId>hms-mirror</artifactId>
-    <version>2.2.0.18.0</version>
+    <version>2.2.0.18.1</version>
     <packaging>jar</packaging>
 
     <name>hms-mirror</name>

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java
@@ -569,8 +569,9 @@ protected Boolean buildMigrationSql(TableMirror tableMirror, Environment origina
 
         // Set Override Properties.
         if (config.getOptimization().getOverrides() != null) {
-            for (String key : config.getOptimization().getOverrides().getLeft().keySet()) {
-                targetEnvTable.addSql("Setting " + key, "set " + key + "=" + config.getOptimization().getOverrides().getLeft().get(key));
+            Map<String, String> overrides = config.getOptimization().getOverrides().getFor(targetEnv2);
+            for (String key : overrides.keySet()) {
+                targetEnvTable.addSql("Setting " + key, "set " + key + "=" + overrides.get(key));
             }
         }
 

diff --git a/.../java/com/cloudera/utils/hms/mirror/datastrategy/SQLAcidDowngradeInPlaceDataStrategy.java b/.../java/com/cloudera/utils/hms/mirror/datastrategy/SQLAcidDowngradeInPlaceDataStrategy.java
@@ -37,6 +37,7 @@
 import org.springframework.stereotype.Component;
 
 import java.text.MessageFormat;
+import java.util.Map;
 
 import static com.cloudera.utils.hms.mirror.SessionVars.SORT_DYNAMIC_PARTITION;
 import static com.cloudera.utils.hms.mirror.SessionVars.SORT_DYNAMIC_PARTITION_THRESHOLD;
@@ -125,9 +126,11 @@ public Boolean buildOutSql(TableMirror tableMirror) throws MissingDataPointExcep
 
         let.addSql(TableUtils.USE_DESC, useDb);
         // Set Override Properties.
-        if (!hmsMirrorConfig.getOptimization().getOverrides().getLeft().isEmpty()) {
-            for (String key : hmsMirrorConfig.getOptimization().getOverrides().getLeft().keySet()) {
-                let.addSql("Setting " + key, "set " + key + "=" + hmsMirrorConfig.getOptimization().getOverrides().getLeft().get(key));
+        // Get the LEFT overrides for the DOWNGRADE.
+        Map<String, String> overrides = hmsMirrorConfig.getOptimization().getOverrides().getFor(Environment.LEFT);
+        if (!overrides.isEmpty()) {
+            for (String key : overrides.keySet()) {
+                let.addSql("Setting " + key, "set " + key + "=" + overrides.get(key));
             }
         }
 

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java
@@ -489,9 +489,10 @@ public Boolean execute(TableMirror tableMirror) {
                         let.addSql(new Pair(TEZ_EXECUTION_DESC, SET_TEZ_AS_EXECUTION_ENGINE));
                     }
                     // Set Override Properties.
-                    if (config.getOptimization().getOverrides() != null) {
-                        for (String key : config.getOptimization().getOverrides().getLeft().keySet()) {
-                            let.addSql("Setting " + key, "set " + key + "=" + config.getOptimization().getOverrides().getLeft().get(key));
+                    Map<String, String> overrides = config.getOptimization().getOverrides().getFor(Environment.LEFT);
+                    if (!overrides.isEmpty()) {
+                        for (String key : overrides.keySet()) {
+                            let.addSql("Setting " + key, "set " + key + "=" + overrides.get(key));
                         }
                     }
 

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/domain/Overrides.java b/src/main/java/com/cloudera/utils/hms/mirror/domain/Overrides.java
@@ -17,6 +17,7 @@
 
 package com.cloudera.utils.hms.mirror.domain;
 
+import com.cloudera.utils.hms.mirror.domain.support.Environment;
 import com.cloudera.utils.hms.mirror.domain.support.SideType;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@@ -47,31 +48,28 @@ public void addProperty(String key, String value, SideType side) {
     }
 
     @JsonIgnore
-    public Map<String, String> getLeft() {
-        Map<String, String> left = new TreeMap<String, String>();
-        for (Map.Entry<String, Map<SideType, String>> entry : properties.entrySet()) {
-            if (entry.getValue().containsKey(SideType.LEFT)) {
-                left.put(entry.getKey(), entry.getValue().get(SideType.LEFT));
-            }
-            if (entry.getValue().containsKey(SideType.BOTH)) {
-                left.put(entry.getKey(), entry.getValue().get(SideType.BOTH));
-            }
-        }
-        return left;
-    }
+    public Map<String, String> getFor(Environment environment) {
 
-    @JsonIgnore
-    public Map<String, String> getRight() {
-        Map<String, String> right = new TreeMap<String, String>();
+        Map<String, String> rtn = new TreeMap<String, String>();
         for (Map.Entry<String, Map<SideType, String>> entry : properties.entrySet()) {
-            if (entry.getValue().containsKey(SideType.RIGHT)) {
-                right.put(entry.getKey(), entry.getValue().get(SideType.RIGHT));
+            switch (environment) {
+                case LEFT:
+                    if (entry.getValue().containsKey(SideType.LEFT)) {
+                        rtn.put(entry.getKey(), entry.getValue().get(SideType.LEFT));
+                    }
+                    break;
+                case RIGHT:
+                    if (entry.getValue().containsKey(SideType.RIGHT)) {
+                        rtn.put(entry.getKey(), entry.getValue().get(SideType.RIGHT));
+                    }
+                    break;
             }
             if (entry.getValue().containsKey(SideType.BOTH)) {
-                right.put(entry.getKey(), entry.getValue().get(SideType.BOTH));
+                rtn.put(entry.getKey(), entry.getValue().get(SideType.BOTH));
             }
         }
-        return right;
+
+        return rtn;
     }
 
     public void setPropertyOverridesStr(String[] inPropsStr, SideType side) {

diff --git a/src/test/java/com/cloudera/utils/hms/mirror/domain/HmsMirrorConfigCloneFullTest_01.java b/src/test/java/com/cloudera/utils/hms/mirror/domain/HmsMirrorConfigCloneFullTest_01.java
@@ -147,8 +147,8 @@ public void cloneTest_01() {
         assertEquals(clone.getOptimization().isAutoTune(), config.getOptimization().isAutoTune());
         assertEquals(clone.getOptimization().isCompressTextOutput(), config.getOptimization().isCompressTextOutput());
         assertEquals(clone.getOptimization().isSkipStatsCollection(), config.getOptimization().isSkipStatsCollection());
-        assertEquals(clone.getOptimization().getOverrides().getLeft(), config.getOptimization().getOverrides().getLeft());
-        assertEquals(clone.getOptimization().getOverrides().getRight(), config.getOptimization().getOverrides().getRight());
+        assertEquals(clone.getOptimization().getOverrides().getFor(Environment.LEFT), config.getOptimization().getOverrides().getFor(Environment.LEFT));
+        assertEquals(clone.getOptimization().getOverrides().getFor(Environment.RIGHT), config.getOptimization().getOverrides().getFor(Environment.RIGHT));
         assertEquals(clone.getOptimization().isBuildShadowStatistics(), config.getOptimization().isBuildShadowStatistics());
 
         assertEquals(clone.getOutputDirectory(), config.getOutputDirectory());

diff --git a/.../java/com/cloudera/utils/hms/mirror/integration/end_to_end/legacy_to_cdp/Test_sql_is.java b/.../java/com/cloudera/utils/hms/mirror/integration/end_to_end/legacy_to_cdp/Test_sql_is.java
@@ -43,6 +43,9 @@
 //                "--hms-mirror.config.target-namespace=s3a://my_cs_bucket",
 //                "--hms-mirror.config.reset-to-default-location=true",
 //                "--hms-mirror.config.distcp=true",
+                "--hms-mirror.config.property-overrides=hive.exec.orc.split.strategy=BI",
+                "--hms-mirror.config.property-overrides-left=tez.queue.name=left-side",
+                "--hms-mirror.config.property-overrides-right=tez.queue.name=right-side",
                 "--hms-mirror.conversion.test-filename=/test_data/assorted_tbls_01.yaml",
                 "--hms-mirror.config.filename=/config/default.yaml.hdp2-cdp",
                 "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/legacy_cdp/sql_is"