From 1a94e8117b35465811d09b725355ffa7db670476 Mon Sep 17 00:00:00 2001 From: "David W. Streever" Date: Thu, 21 Nov 2024 09:47:43 -0500 Subject: [PATCH] **Bug Fixes** - Doubling of mapping locations for partitions in `distcp` for STORAGE_MIGRATION. - [STORAGE_MIGRATION is setting ACID to ON, regardless.](https://github.com/cloudera-labs/hms-mirror/issues/158) **What's New** - [Validate JDBC Jar Files in config.](https://github.com/cloudera-labs/hms-mirror/issues/159) - Ability to turn-on `strict` mode for Storage Migration. This will cause `distcp` to fail when non-standard locations are used. To turn off, use the `-sms|--storage-migration-strict` flag via the CLI. **Behavior Changes** The default behavior for Storage Migration 'strict' has changed from `true` to `false`. The intent behind the `strict` mode was to ensure `distcp` would fail when non-standard locations are used. The combination of `metastore_direct` and knowing the partition location details gives us a better chance on making these mappings work for `distcp`. When the scenario arises, we do **HIGHLY** recommend that you validate the plans created. The new default behavior will allow `distcp` to continue when non-standard locations are encountered, while throwing a warning. This will allow the migration to continue, but you should validate the results. --- Writerside/topics/Release-Notes.md | 16 ++ .../topics/Transfer-Storage-Migration.md | 10 ++ pom.xml | 2 +- .../utils/hms/mirror/MessageCode.java | 15 +- .../cli/HmsMirrorCommandLineOptions.java | 34 ++-- .../StorageMigrationDataStrategy.java | 57 ++++++- .../hms/mirror/domain/StorageMigration.java | 2 +- .../end_to_end/cdp/Test_sm_smn_wd_epl_dc.java | 51 ++---- .../cdp/Test_sm_smn_wd_epl_dc_mismatch.java | 18 ++- ...Test_sm_smn_wd_epl_dc_mismatch_strict.java | 90 +++++++++++ .../cdp/Test_sm_smn_wd_epl_dc_strict.java | 80 +++++++++ .../cdp/Test_sm_smn_wd_epl_glm_fel_dc.java | 5 +- .../Test_sm_smn_wd_epl_glm_fel_dc_strict.java | 152 ++++++++++++++++++ .../end_to_end/cdp/Test_sm_wd_epl_dc.java | 15 +- .../cdp/Test_sm_wd_epl_dc_strict.java | 85 ++++++++++ .../end_to_end/cdp/Test_sm_wd_epl_glm_dc.java | 5 +- .../cdp/Test_sm_wd_epl_glm_dc_strict.java | 99 ++++++++++++ .../cdp/Test_sm_wd_epl_rdl_dc_mmd.java | 5 +- .../cdp/Test_sm_wd_epl_rdl_dc_mmd_strict.java | 131 +++++++++++++++ .../cdp/Test_sm_wd_epl_rdl_dc_odd.java | 4 +- .../cdp/Test_sm_wd_epl_rdl_dc_odd_strict.java | 135 ++++++++++++++++ 21 files changed, 925 insertions(+), 86 deletions(-) create mode 100644 src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_mismatch_strict.java create mode 100644 src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_strict.java create mode 100644 src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_glm_fel_dc_strict.java create mode 100644 src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_dc_strict.java create mode 100644 src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_glm_dc_strict.java create mode 100644 src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_mmd_strict.java create mode 100644 src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_odd_strict.java diff --git a/Writerside/topics/Release-Notes.md b/Writerside/topics/Release-Notes.md index 8a552480..d911d66b 100644 --- a/Writerside/topics/Release-Notes.md +++ b/Writerside/topics/Release-Notes.md @@ -10,6 +10,22 @@ The latest set of enhancement requests can be found [here](https://github.com/cl If there is something you'd like to see, add a new issue [here](https://github.com/cloudera-labs/hms-mirror/issues) +## 2.2.0.19.0 (pre-release) + +**Bug Fixes** + +- Doubling of mapping locations for partitions in `distcp` for STORAGE_MIGRATION. +- [STORAGE_MIGRATION is setting ACID to ON, regardless.](https://github.com/cloudera-labs/hms-mirror/issues/158) + +**What's New** + +- [Validate JDBC Jar Files in config.](https://github.com/cloudera-labs/hms-mirror/issues/159) +- Ability to turn-on `strict` mode for Storage Migration. This will cause `distcp` to fail when non-standard locations are used. To turn off, use the `-sms|--storage-migration-strict` flag via the CLI. + +**Behavior Changes** + +The default behavior for Storage Migration 'strict' has changed from `true` to `false`. The intent behind the `strict` mode was to ensure `distcp` would fail when non-standard locations are used. The combination of `metastore_direct` and knowing the partition location details gives us a better chance on making these mappings work for `distcp`. When the scenario arises, we do **HIGHLY** recommend that you validate the plans created. The new default behavior will allow `distcp` to continue when non-standard locations are encountered, while throwing a warning. This will allow the migration to continue, but you should validate the results. + ## 2.2.0.18.1 diff --git a/Writerside/topics/Transfer-Storage-Migration.md b/Writerside/topics/Transfer-Storage-Migration.md index 01f8c809..08b70a11 100644 --- a/Writerside/topics/Transfer-Storage-Migration.md +++ b/Writerside/topics/Transfer-Storage-Migration.md @@ -54,4 +54,14 @@ you expect since there's no guarantee that there isn't a lot of other 'extra' da transfer: storageMigration: consolidateTablesForDistcp: true|false +``` + +## Strict Mode + +When true and using `distcp`, the migration will fail if the table partition locations aren't standard. The default is `false`. The default mode will allow you to move forward with warnings about the non-standard partition locations. It's highly recommended that you validate the location mappings in the `distcp` plan before running the migration. + +```yaml +transfer: + storageMigration: + strict: true|false ``` \ No newline at end of file diff --git a/pom.xml b/pom.xml index 42e34e33..d678cfb9 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ com.cloudera.utils.hadoop hms-mirror - 2.2.0.18.2 + 2.2.0.19.0 jar hms-mirror diff --git a/src/main/java/com/cloudera/utils/hms/mirror/MessageCode.java b/src/main/java/com/cloudera/utils/hms/mirror/MessageCode.java index dac98c47..32153e28 100644 --- a/src/main/java/com/cloudera/utils/hms/mirror/MessageCode.java +++ b/src/main/java/com/cloudera/utils/hms/mirror/MessageCode.java @@ -67,13 +67,11 @@ public enum MessageCode { "location with SCHEMA_ONLY so you can build out the movement plan."), DISTCP_VALID_STRATEGY("The `distcp` option is not valid for this strategy ({0}) and configuration."), DISTCP_WITH_MISMATCHING_LOCATIONS("You''ve specified ''distcp'' with mismatching locations for {0} {1}: " + - "Original Location {2}, Specification {3}. When these don''t match, a valid distcp plan can''t be created to " + - "correctly align the data elements. You''ll need to use SQL to migrate the data and allow Hive to reorganize it " + - "according to your specs."), + "Original Location {2}, Specification {3}. When these don''t match, the 'distcp' plan created might not " + + "correctly align the data elements. You MUST validate the plan before executing. Proceed at your own risk!"), DISTCP_WITH_MISMATCHING_TABLE_LOCATION("You''ve specified ''distcp'' with mismatching locations for {0} {1}: " + - "Original Location: {2}, Derived table name from directory: {3}. The partition directory doesn''t match the table name and we can''t " + - "correctly align the data elements via distcp. You''ll need to use SQL to migrate the data and allow Hive to reorganize it " + - "according to your specs."), + "Original Location: {2}, Derived table name from directory: {3}. When these don''t match the table name and the plan might not " + + "correctly align the data elements via distcp. You MUST validate the plan before executing. Proceed at your own risk!"), DISTCP_WO_TABLE_FILTERS("`distcp` workbooks will include the DATABASE base directory, which may include more data than is " + "expected at the 'db' level, especially if the root directory is used by other processes outside of the tables " + "defined in the database."), @@ -214,11 +212,14 @@ public enum MessageCode { STORAGE_MIGRATION_REQUIRED_NAMESPACE("STORAGE_MIGRATION requires -smn or -cs to define the new namespace."), STORAGE_MIGRATION_REQUIRED_STRATEGY("STORAGE_MIGRATION requires -sms to set the Data Strategy. Applicable options " + "are SCHEMA_ONLY, SQL, EXPORT_IMPORT, or HYBRID"), - STORAGE_MIGRATION_STRICT("Storage Migration is in 'strict' mode. If the table and/or partition locations can't " + STORAGE_MIGRATION_STRICT("Storage Migration is in 'strict' mode. The table and/or partition locations can't " + "be mapped to the warehouse locations or there are mismatches in location standards, we can't process the table " + "without potential data loss. Add additional 'global-location-map' entries to cover the locations." + "Mismatched directories or non-standard partition locations can only be handled through the " + "SQL dataMovementStategy."), + STORAGE_MIGRATION_NOT_STRICT_ISSUE("Storage Migration is NOT in 'strict' mode. The table and/or partition locations aren't " + + "standard and could lead to data loss when migrated with 'distcp'. We strongly recommend reviewing the plans created " + + "and ensuring the data is correctly aligned before executing to avoid data loss and/or duplication."), SYNC_TBL_FILTER("'sync' with 'table filter' will be bi-directional ONLY for tables that meet the table filter '" + "' ON BOTH SIDES!!!") // WARNINGS , diff --git a/src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptions.java b/src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptions.java index f38b883c..1feb9604 100644 --- a/src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptions.java +++ b/src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptions.java @@ -1182,26 +1182,25 @@ CommandLineRunner configStorageMigrationNamespace(HmsMirrorConfig hmsMirrorConfi @Bean @Order(1) @ConditionalOnProperty( - name = "hms-mirror.config.storage-migration-strict", - havingValue = "true") - CommandLineRunner configStorageMigrationStrictTrue(HmsMirrorConfig hmsMirrorConfig) { + name = "hms-mirror.config.storage-migration-strict") + CommandLineRunner configStorageMigrationStrictTrue(HmsMirrorConfig hmsMirrorConfig, @Value("${hms-mirror.config.storage-migration-strict}") boolean value) { return args -> { - log.info("storage-migration-strict: {}", Boolean.TRUE); - hmsMirrorConfig.getTransfer().getStorageMigration().setStrict(Boolean.TRUE); + log.info("storage-migration-strict: {}", value); + hmsMirrorConfig.getTransfer().getStorageMigration().setStrict(value); }; } - @Bean - @Order(1) - @ConditionalOnProperty( - name = "hms-mirror.config.storage-migration-strict", - havingValue = "false") - CommandLineRunner configStorageMigrationStrictFalse(HmsMirrorConfig hmsMirrorConfig) { - return args -> { - log.warn("storage-migration-strict: {} is not currently supported to ensure valid migration plans.", Boolean.FALSE); +// @Bean +// @Order(1) +// @ConditionalOnProperty( +// name = "hms-mirror.config.storage-migration-strict", +// havingValue = "false") +// CommandLineRunner configStorageMigrationStrictFalse(HmsMirrorConfig hmsMirrorConfig) { +// return args -> { +// log.warn("storage-migration-strict: {}", Boolean.FALSE); // hmsMirrorConfig.getTransfer().getStorageMigration().setStrict(Boolean.FALSE); - }; - } +// }; +// } @Bean @Order(1) @@ -1873,6 +1872,11 @@ private Options getOptions() { storageMigrationNamespaceOption.setArgName("namespace"); options.addOption(storageMigrationNamespaceOption); + Option storageMigrationStrictOption = new Option("sms", "storage-migration-strict", false, + "Use 'strict' location translations for storage migration."); + storageMigrationStrictOption.setRequired(Boolean.FALSE); + options.addOption(storageMigrationStrictOption); + Option dbOption = new Option("db", "database", true, "Comma separated list of Databases (upto 100)."); dbOption.setValueSeparator(','); diff --git a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java index eb8611a9..95288783 100644 --- a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java +++ b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java @@ -212,6 +212,9 @@ public Boolean execute(TableMirror tableMirror) { EnvironmentTable let = getEnvironmentTable(Environment.LEFT, tableMirror); EnvironmentTable ret = getEnvironmentTable(Environment.RIGHT, tableMirror); + + Boolean strictIssues = Boolean.FALSE; + try { /* If using distcp, we don't need to go through and rename/recreate the tables. We just need to change the @@ -284,6 +287,14 @@ public Boolean execute(TableMirror tableMirror) { for (Map.Entry entry : let.getPartitions().entrySet()) { String partSpec = entry.getKey(); int level = StringUtils.countMatches(partSpec, "/"); + // To ensure the partition location is handled correctly when it's within the table location, + // we need to see if the partition location 'starts with' the table location. + String tableLocation = TableUtils.getLocation(tableMirror.getName(), tableMirror.getTableDefinition(Environment.LEFT)); + if (entry.getValue().startsWith(tableLocation)) { + // Increase the pruning back to the table location so we aren't build distcp commands for the partitions + // that would be handled by the table location. + level++; + } // Translate to 'partition spec'. partSpec = TableUtils.toPartitionSpec(partSpec); String partLocation = entry.getValue(); @@ -297,8 +308,12 @@ public Boolean execute(TableMirror tableMirror) { String msg = MessageFormat.format(MessageCode.DISTCP_WITH_MISMATCHING_LOCATIONS.getDesc(), "Partition", partSpec, partLocation, normalizedPartSpecLocation); tableMirror.addIssue(Environment.LEFT, msg); - noIssues = Boolean.FALSE; - continue; + if (config.getTransfer().getStorageMigration().isStrict()) { + noIssues = Boolean.FALSE; + continue; + } else { + strictIssues = Boolean.TRUE; + } } else { // Since the partition location matches the partition spec, we also need // to verify that the directory the partition is in, matches the table name @@ -310,8 +325,12 @@ public Boolean execute(TableMirror tableMirror) { String msg = MessageFormat.format(MessageCode.DISTCP_WITH_MISMATCHING_TABLE_LOCATION.getDesc(), "Partition", partSpec, partLocation, tableDirName); tableMirror.addIssue(Environment.LEFT, msg); - noIssues = Boolean.FALSE; - continue; + if (config.getTransfer().getStorageMigration().isStrict()) { + noIssues = Boolean.FALSE; + continue; + } else { + strictIssues = Boolean.TRUE; + } } } } @@ -374,6 +393,13 @@ public Boolean execute(TableMirror tableMirror) { let.getSql().clear(); rtn = Boolean.FALSE; } + if (strictIssues) { + // Strict is NOT set, but there were some issues. We need to post a warning here. + log.warn("Storage Migration is NOT strict, but there are concerns with the location mappings for {}", + tableMirror.getName()); + let.addIssue(MessageCode.STORAGE_MIGRATION_NOT_STRICT_ISSUE.getDesc()); + rtn = Boolean.TRUE; + } } else { // Distcp with Archive. The intent is to retain an archive of the table (and data) // even under the distcp movement strategy. This allows the user access to the original @@ -406,8 +432,12 @@ public Boolean execute(TableMirror tableMirror) { String msg = MessageFormat.format(MessageCode.DISTCP_WITH_MISMATCHING_LOCATIONS.getDesc(), "Partition", partSpec, partLocation, normalizedPartSpecLocation); tableMirror.addIssue(Environment.LEFT, msg); - noIssues = Boolean.FALSE; - continue; + if (config.getTransfer().getStorageMigration().isStrict()) { + noIssues = Boolean.FALSE; + continue; + } else { + strictIssues = Boolean.TRUE; + } } else { // Since the partition location matches the partition spec, we also need // to verify that the directory the partition is in, matches the table name @@ -419,8 +449,12 @@ public Boolean execute(TableMirror tableMirror) { String msg = MessageFormat.format(MessageCode.DISTCP_WITH_MISMATCHING_TABLE_LOCATION.getDesc(), "Partition", partSpec, partLocation, tableDirName); tableMirror.addIssue(Environment.LEFT, msg); - noIssues = Boolean.FALSE; - continue; + if (config.getTransfer().getStorageMigration().isStrict()) { + noIssues = Boolean.FALSE; + continue; + } else { + strictIssues = Boolean.TRUE; + } } } } @@ -468,6 +502,13 @@ public Boolean execute(TableMirror tableMirror) { let.getSql().clear(); rtn = Boolean.FALSE; } + if (strictIssues) { + // Strict is NOT set, but there were some issues. We need to post a warning here. + log.warn("Storage Migration is NOT strict, but there are concerns with the location mappings for {}", + tableMirror.getName()); + let.addIssue(MessageCode.STORAGE_MIGRATION_NOT_STRICT_ISSUE.getDesc()); + rtn = Boolean.TRUE; + } } // Build Distcp plan for moving table and partition data. diff --git a/src/main/java/com/cloudera/utils/hms/mirror/domain/StorageMigration.java b/src/main/java/com/cloudera/utils/hms/mirror/domain/StorageMigration.java index bfcb625e..a244da16 100644 --- a/src/main/java/com/cloudera/utils/hms/mirror/domain/StorageMigration.java +++ b/src/main/java/com/cloudera/utils/hms/mirror/domain/StorageMigration.java @@ -53,7 +53,7 @@ public class StorageMigration implements Cloneable { private boolean consolidateTablesForDistcp = Boolean.FALSE; @Schema(description = "When strict is true, any issues during evaluation will cause the migration to fail. When false, " + "the migration will continue but the issues will be reported. This can lead to data movement issues.") - private boolean strict = Boolean.TRUE; + private boolean strict = Boolean.FALSE; @Override public StorageMigration clone() { diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc.java index f68ebb4d..f2d1c22c 100644 --- a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc.java +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc.java @@ -54,17 +54,6 @@ STORAGE_MIGRATION test. Defining the warehouse directories (-wd and -ewd) along */ public class Test_sm_smn_wd_epl_dc extends E2EBaseTest { - @Test - public void issueTest() { - validateTableIssueCount("ext_purge_odd_parts", "web_sales", - Environment.LEFT, 2); - } - - @Test - public void phaseTest() { - validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); - } - @Test public void returnCodeTest() { // Get Runtime Return Code. @@ -72,37 +61,19 @@ public void returnCodeTest() { // Verify the return code. // A few partitions have non-standard locations and can't be migrated without addition GLM entries. - long check = 1L; + long check = 0L; assertEquals("Return Code Failure: " + rtn, check, rtn); } -// @Test -// public void sqlTest() { -// Boolean foundAT = Boolean.FALSE; -// Boolean foundOddPart = Boolean.FALSE; -// Boolean foundOddPart2 = Boolean.FALSE; -// -// for (Pair pair : getConversion().getDatabase("ext_purge_odd_parts") -// .getTableMirrors().get("web_sales") -// .getEnvironmentTable(Environment.LEFT).getSql()) { -// if (pair.getDescription().trim().equals("Alter Table Location")) { -// assertEquals("Location doesn't match", "ALTER TABLE web_sales SET LOCATION \"ofs://OHOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales\"", pair.getAction()); -// foundAT = Boolean.TRUE; -// } -// if (pair.getDescription().trim().equals("Alter Table Partition Spec `ws_sold_date_sk`='2451180' Location")) { -// assertEquals("Location doesn't match", "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451180') SET " + -// "LOCATION \"ofs://OHOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451180\"", pair.getAction()); -// foundOddPart = Boolean.TRUE; -// } -// if (pair.getDescription().trim().equals("Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location")) { -// assertEquals("Location doesn't match", "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451188') SET " + -// "LOCATION \"ofs://OHOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451188\"", pair.getAction()); -// foundOddPart2 = Boolean.TRUE; -// } -// } -// assertEquals("Alter Table Location not found", Boolean.TRUE, foundAT); -// assertEquals("Alter Odd Part Location not found", Boolean.TRUE, foundOddPart); -// assertEquals("Alter Odd Part 2 Location not found", Boolean.TRUE, foundOddPart2); -// } + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.SUCCESS); + } + + @Test + public void issueTest() { + validateTableIssueCount("ext_purge_odd_parts", "web_sales", + Environment.LEFT, 2); + } } diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_mismatch.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_mismatch.java index 395872a9..3eb8e783 100644 --- a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_mismatch.java +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_mismatch.java @@ -20,6 +20,7 @@ import com.cloudera.utils.hms.mirror.PhaseState; import com.cloudera.utils.hms.mirror.cli.Mirror; +import com.cloudera.utils.hms.mirror.domain.support.Environment; import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest; import lombok.extern.slf4j.Slf4j; import org.junit.Test; @@ -35,6 +36,7 @@ args = { "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_01.yaml", "--hms-mirror.config.filename=/config/default.yaml.cdp", + "--hms-mirror.config.storage-migration-strict=false", // This would be 'false' by default. "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_smn_wd_epl_dc_mismatch", }) @ActiveProfiles("e2e-cdp-sm_smn_wd_epl_dc") @@ -50,10 +52,6 @@ This test uses a configuration (LEFT hcfs namespace) that doesn't match the tabl but success app run. */ public class Test_sm_smn_wd_epl_dc_mismatch extends E2EBaseTest { - @Test - public void phaseTest() { - validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); - } /* String[] args = new String[]{"-d", "STORAGE_MIGRATION", @@ -81,8 +79,18 @@ public void returnCodeTest() { // Get Runtime Return Code. long rtn = getReturnCode(); // Verify the return code. - long check = 1L; + long check = 0L; assertEquals("Return Code Failure: " + rtn, check, rtn); } + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.SUCCESS); + } + + @Test + public void issueTest() { + validateTableIssueCount("ext_purge_odd_parts", "web_sales", + Environment.LEFT, 2); + } } diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_mismatch_strict.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_mismatch_strict.java new file mode 100644 index 00000000..2e591060 --- /dev/null +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_mismatch_strict.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2024. Cloudera, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp; + + +import com.cloudera.utils.hms.mirror.PhaseState; +import com.cloudera.utils.hms.mirror.cli.Mirror; +import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Mirror.class, + args = { + "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_01.yaml", + "--hms-mirror.config.filename=/config/default.yaml.cdp", + "--hms-mirror.config.storage-migration-strict=true", + "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_smn_wd_epl_dc_mismatch_strict", + }) +@ActiveProfiles("e2e-cdp-sm_smn_wd_epl_dc") +@Slf4j +/* +This test uses a configuration (LEFT hcfs namespace) that doesn't match the table/partition prefixes. This should +fail with warnings. + +FIXED:1. when namespace in table doesn't match the namespace specified in the hcfsNamespace, nothing is translated. + - This should result in an error and warnings about why this didn't work. + +TODO: We need to fix the return code to be negative on Errors and Positive on 'table' conversion failures + but success app run. + */ +public class Test_sm_smn_wd_epl_dc_mismatch_strict extends E2EBaseTest { + + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + } + + /* + String[] args = new String[]{"-d", "STORAGE_MIGRATION", + "-wd", "/finance/managed-fso", + "-ewd", "/finance/external-fso", + "-smn", "ofs://OHOME90", + "-epl", + "-dc", + "-ltd", EXT_PURGE_ODD_PARTS_03, "-cfg", CDP, + "-o", outputDir + }; + long rtn = 0; + MirrorLegacy mirror = new MirrorLegacy(); + rtn = mirror.go(args); + assertEquals("Return Code Failure: " + rtn, 1, rtn); + + // Read the output and verify the results. + DBMirror[] resultsMirrors = getResults(outputDir, EXT_PURGE_ODD_PARTS_03); + + validatePhase(resultsMirrors[0], "web_sales", PhaseState.ERROR); + + */ + @Test + public void returnCodeTest() { + // Get Runtime Return Code. + long rtn = getReturnCode(); + // Verify the return code. + long check = 1L; + assertEquals("Return Code Failure: " + rtn, check, rtn); + } + +} diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_strict.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_strict.java new file mode 100644 index 00000000..ac208ddb --- /dev/null +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_dc_strict.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2024. Cloudera, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp; + +import com.cloudera.utils.hms.mirror.PhaseState; +import com.cloudera.utils.hms.mirror.cli.Mirror; +import com.cloudera.utils.hms.mirror.domain.support.Environment; +import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Mirror.class, + args = { + "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_smn_wd_epl_dc_strict", + "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_01.yaml", +// "--hms-mirror.config.reset-to-default-location=true", + "--hms-mirror.config.data-strategy=STORAGE_MIGRATION", + "--hms-mirror.config.storage-migration-strict=true", + "--hms-mirror.config.warehouse-plans=ext_purge_odd_parts=/finance/external-fso:/finance/managed-fso", + "--hms-mirror.config.storage-migration-namespace=ofs://OHOME90", +// "--hms-mirror.config.evaluate-partition-location=true", + "--hms-mirror.config.align-locations=true", + "--hms-mirror.config.distcp=PULL", + "--hms-mirror.config.filename=/config/default.yaml.cdp" + }) +@Slf4j +/* +STORAGE_MIGRATION test. Defining the warehouse directories (-wd and -ewd) along with -epl (evaluation of partition locations). +We've also added -dc to this to produce a distcp plan for this data migration. +It should only evaluate non-acid tables. + +This storage migration doesn't require the creation of any new tables. We will simply ALTER the table and partition +locations. + */ +public class Test_sm_smn_wd_epl_dc_strict extends E2EBaseTest { + + @Test + public void returnCodeTest() { + // Get Runtime Return Code. + long rtn = getReturnCode(); + // Verify the return code. + + // A few partitions have non-standard locations and can't be migrated without addition GLM entries. + long check = 1L; + assertEquals("Return Code Failure: " + rtn, check, rtn); + } + + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + } + + @Test + public void issueTest() { + validateTableIssueCount("ext_purge_odd_parts", "web_sales", + Environment.LEFT, 2); + } + +} diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_glm_fel_dc.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_glm_fel_dc.java index 4b7890cb..c43b378b 100644 --- a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_glm_fel_dc.java +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_glm_fel_dc.java @@ -37,6 +37,7 @@ args = { "--hms-mirror.config.force-external-location=true", "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_smn_wd_epl_glm_fel_dc", + "--hms-mirror.config.storage-migration-strict=false", "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_01.yaml", "--hms-mirror.config.global-location-map=/user/dstreev/datasets/alt-locations/load_web_sales=/finance/external-fso/load_web_sales,/warehouse/tablespace/external/hive=/finance/external-fso" @@ -107,7 +108,7 @@ public class Test_sm_smn_wd_epl_glm_fel_dc extends E2EBaseTest { @Test public void phaseTest() { - validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.SUCCESS); } @Test @@ -118,7 +119,7 @@ public void returnCodeTest() { // Non-standard locations can't be migrated without additional GLM entries. // Verify the return code. - long check = 1L; + long check = 0L; assertEquals("Return Code Failure: " + rtn, check, rtn); } diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_glm_fel_dc_strict.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_glm_fel_dc_strict.java new file mode 100644 index 00000000..7248faea --- /dev/null +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_glm_fel_dc_strict.java @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2024. Cloudera, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp; + + +import com.cloudera.utils.hms.mirror.PhaseState; +import com.cloudera.utils.hms.mirror.cli.Mirror; +import com.cloudera.utils.hms.mirror.domain.support.Environment; +import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Mirror.class, + args = { + "--hms-mirror.config.force-external-location=true", + "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_smn_wd_epl_glm_fel_dc_strict", + "--hms-mirror.config.storage-migration-strict=true", + "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_01.yaml", + "--hms-mirror.config.global-location-map=/user/dstreev/datasets/alt-locations/load_web_sales=/finance/external-fso/load_web_sales,/warehouse/tablespace/external/hive=/finance/external-fso" + + }) +@ActiveProfiles("e2e-cdp-sm_smn_wd_epl_dc") +@Slf4j +/* +Issues: Need to post warning when table/partition(s) new location isn't in the -[e]wd location. + */ +public class Test_sm_smn_wd_epl_glm_fel_dc_strict extends E2EBaseTest { + + // @Test +// public void sm_smn_wd_epl_glm_fel_dc() { +// String nameofCurrMethod = new Throwable() +// .getStackTrace()[0] +// .getMethodName(); +// +// String outputDir = getOutputDirBase() + nameofCurrMethod; +// +// String[] args = new String[]{"-d", "STORAGE_MIGRATION", +// "-wd", "/finance/managed-fso", +// "-ewd", "/finance/external-fso", +// "-smn", "ofs://OHOME90", +// "-epl", +// "-glm", "/user/dstreev/datasets/alt-locations/load_web_sales=/finance/external-fso/load_web_sales,/warehouse/tablespace/external/hive=/finance/external-fso", +// "-fel", +// "-dc", +// "-ltd", EXT_PURGE_ODD_PARTS_03, "-cfg", CDP_CDP, +// "-o", outputDir +// }; +// long rtn = 0; +// MirrorLegacy mirror = new MirrorLegacy(); +// rtn = mirror.go(args); +// assertEquals("Return Code Failure: " + rtn, 0, rtn); +// +// // Read the output and verify the results. +// DBMirror[] resultsMirrors = getResults(outputDir, EXT_PURGE_ODD_PARTS_03); +// +// validatePhase(resultsMirrors[0], "web_sales", PhaseState.SUCCESS); +// validateTableIssueCount(resultsMirrors[0], "web_sales", Environment.LEFT, 3); +// /* +// - description: "Alter Table Location" +// action: "ALTER TABLE web_sales SET LOCATION \"ofs://OHOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales\"" +// - description: "Alter Table Partition Spec `ws_sold_date_sk`='2451180' Location " +// action: "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451180') SET\ +// \ LOCATION \"ofs://OHOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451180\"" +// - description: "Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location " +// action: "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451188') SET\ +// \ LOCATION \"ofs://OHOME90/user/dstreev/datasets/alt-locations/web_sales/ws_sold_date_sk=2451188\"" +// - description: "Alter Table Partition Spec `ws_sold_date_sk`='2452035' Location " +// action: "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2452035') SET\ +// \ LOCATION \"ofs://OHOME90/finance/external-fso/load_web_sales/odd\"" +// */ +// if (!validateSqlPair(resultsMirrors[0], Environment.LEFT, "web_sales", "Alter Table Location", +// "ALTER TABLE web_sales SET LOCATION \"ofs://OHOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales\"")) { +// fail("Alter Table Location not found"); +// } +// if (!validateSqlPair(resultsMirrors[0], Environment.LEFT, "web_sales", "Alter Table Partition Spec `ws_sold_date_sk`='2451180' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451180') SET LOCATION \"ofs://OHOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451180\"")) { +// fail("Alter Table Partition Location not found"); +// } +// if (!validateSqlPair(resultsMirrors[0], Environment.LEFT, "web_sales", "Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451188') SET LOCATION \"ofs://OHOME90/user/dstreev/datasets/alt-locations/web_sales/ws_sold_date_sk=2451188\"")) { +// fail("Alter Table Partition Location not found"); +// } +// +// } + + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + } + + @Test + public void returnCodeTest() { + // Get Runtime Return Code. + long rtn = getReturnCode(); + + // Non-standard locations can't be migrated without additional GLM entries. + + // Verify the return code. + long check = 1L; + assertEquals("Return Code Failure: " + rtn, check, rtn); + } + +// @Test +// public void validateSqlTest() { +// if (!validateSqlPair("ext_purge_odd_parts", Environment.LEFT, "web_sales", "Alter Table Location", +// "ALTER TABLE web_sales SET LOCATION \"ofs://OHOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales\"")) { +// fail("Alter Table Location not found"); +// } +// if (!validateSqlPair("ext_purge_odd_parts", Environment.LEFT, "web_sales", "Alter Table Partition Spec `ws_sold_date_sk`='2451180' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451180') SET LOCATION \"ofs://OHOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451180\"")) { +// fail("Alter Table Partition Location not found"); +// } +// if (!validateSqlPair("ext_purge_odd_parts", Environment.LEFT, "web_sales", "Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451188') SET LOCATION \"ofs://OHOME90/user/dstreev/datasets/alt-locations/web_sales/ws_sold_date_sk=2451188\"")) { +// fail("Alter Table Partition Location not found"); +// } +// } + + @Test + public void validateTableIssueCount() { + validateTableIssueCount("ext_purge_odd_parts", "web_sales", + Environment.LEFT, 2); + +// assertEquals("Issue Count not as expected", 3, +// getConversion().getDatabase("ext_purge_odd_parts") +// .getTableMirrors().get("web_sales") +// .getEnvironmentTable(Environment.LEFT).getIssues().size()); + } + +} diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_dc.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_dc.java index 18a4b805..5e400ad4 100644 --- a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_dc.java +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_dc.java @@ -17,7 +17,9 @@ package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp; +import com.cloudera.utils.hms.mirror.PhaseState; import com.cloudera.utils.hms.mirror.cli.Mirror; +import com.cloudera.utils.hms.mirror.domain.support.Environment; import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest; import lombok.extern.slf4j.Slf4j; import org.junit.Test; @@ -65,7 +67,18 @@ public void returnCodeTest() { // Get Runtime Return Code. long rtn = getReturnCode(); // Verify the return code. - long check = 1L; + long check = 0L; assertEquals("Return Code Failure: " + rtn, check, rtn); } + + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.SUCCESS); + } + + @Test + public void issueTest() { + validateTableIssueCount("ext_purge_odd_parts", "web_sales", + Environment.LEFT, 2); + } } diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_dc_strict.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_dc_strict.java new file mode 100644 index 00000000..a03d9ad2 --- /dev/null +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_dc_strict.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2024. Cloudera, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp; + +import com.cloudera.utils.hms.mirror.PhaseState; +import com.cloudera.utils.hms.mirror.cli.Mirror; +import com.cloudera.utils.hms.mirror.domain.support.Environment; +import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Mirror.class, + args = { + "--hms-mirror.config.target-namespace=ofs://OHOME90", + "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_01.yaml", + "--hms-mirror.config.storage-migration-strict=true", + "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_wd_epl_dc_strict" + } +) +@ActiveProfiles("e2e-cdp-sm_wd_epl_dc") +@Slf4j +public class Test_sm_wd_epl_dc_strict extends E2EBaseTest { + /* + String[] args = new String[]{"-d", "STORAGE_MIGRATION", + "-wd", "/finance/managed-fso", + "-ewd", "/finance/external-fso", +// "-smn", "ofs://OHOME90", + "-epl", +// "-rdl", + "-dc", + "-ltd", EXT_PURGE_ODD_PARTS_03, "-cfg", CDP_CDP, + "-o", outputDir + }; + long rtn = 0; + MirrorLegacy mirror = new MirrorLegacy(); + rtn = mirror.go(args); + long check = STORAGE_MIGRATION_NAMESPACE_LEFT_MISSING_RDL_GLM.getLong(); + + assertEquals("Return Code Failure: " + rtn + " doesn't match: " + check * -1, check * -1, rtn); + + + */ + @Test + public void returnCodeTest() { + // Get Runtime Return Code. + long rtn = getReturnCode(); + // Verify the return code. + long check = 1L; + assertEquals("Return Code Failure: " + rtn, check, rtn); + } + + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + } + + @Test + public void issueTest() { + validateTableIssueCount("ext_purge_odd_parts", "web_sales", + Environment.LEFT, 2); + } + +} diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_glm_dc.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_glm_dc.java index 3959acfc..a4d21801 100644 --- a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_glm_dc.java +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_glm_dc.java @@ -35,6 +35,7 @@ args = { "--hms-mirror.config.target-namespace=ofs://OHOME90", "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_wd_epl_glm_dc", + "--hms-mirror.config.storage-migration-strict=false", "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_01.yaml" // "--hms-mirror.config.global-location-map=/user/dstreev/datasets/alt-locations/load_web_sales=/finance/external-fso/load_web_sales," + // "/warehouse/tablespace/external/hive=/finance/external-fso,/user/dstreev/datasets/alt-locations=/finance/external-fso/ext_purge_odd_parts.db" @@ -73,7 +74,7 @@ public class Test_sm_wd_epl_glm_dc extends E2EBaseTest { @Test public void phaseTest() { - validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.SUCCESS); } @Test @@ -81,7 +82,7 @@ public void returnCodeTest() { // Get Runtime Return Code. long rtn = getReturnCode(); // Verify the return code. - assertEquals("Return Code Failure: " + rtn, 1L, rtn); + assertEquals("Return Code Failure: " + rtn, 0L, rtn); } @Test diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_glm_dc_strict.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_glm_dc_strict.java new file mode 100644 index 00000000..408e091a --- /dev/null +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_glm_dc_strict.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2024. Cloudera, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp; + +import com.cloudera.utils.hms.mirror.PhaseState; +import com.cloudera.utils.hms.mirror.cli.Mirror; +import com.cloudera.utils.hms.mirror.domain.support.Environment; +import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Mirror.class, + args = { + "--hms-mirror.config.target-namespace=ofs://OHOME90", + "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_wd_epl_glm_dc_strict", + "--hms-mirror.config.storage-migration-strict=true", + "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_01.yaml" +// "--hms-mirror.config.global-location-map=/user/dstreev/datasets/alt-locations/load_web_sales=/finance/external-fso/load_web_sales," + +// "/warehouse/tablespace/external/hive=/finance/external-fso,/user/dstreev/datasets/alt-locations=/finance/external-fso/ext_purge_odd_parts.db" + + // /user/dstreev/datasets/alt-locations /warehouse/tablespace/external/hive/ext_purge_odd_parts.db + }) +@ActiveProfiles("e2e-cdp-sm_wd_epl_dc") +@Slf4j +/* +Issues: Need to post warning when table/partition(s) new location isn't in the -[e]wd location. +*/ +public class Test_sm_wd_epl_glm_dc_strict extends E2EBaseTest { + /* + + String[] args = new String[]{"-d", "STORAGE_MIGRATION", + "-wd", "/finance/managed-fso", + "-ewd", "/finance/external-fso", + "-epl", + "-glm", "/user/dstreev/datasets/alt-locations/load_web_sales=/finance/external-fso/load_web_sales,/warehouse/tablespace/external/hive=/finance/external-fso", + "-dc", + "-ltd", EXT_PURGE_ODD_PARTS_03, "-cfg", CDP_CDP, + "-o", outputDir + }; + long rtn = 0; + MirrorLegacy mirror = new MirrorLegacy(); + rtn = mirror.go(args); + assertEquals("Return Code Failure: " + rtn, 1, rtn); + + // Read the output and verify the results. + DBMirror[] resultsMirrors = getResults(outputDir, EXT_PURGE_ODD_PARTS_03); + + validatePhase(resultsMirrors[0], "web_sales", PhaseState.ERROR); + validateTableIssueCount(resultsMirrors[0], "web_sales", Environment.LEFT, 3); + // One of the locations is not accounted for in the glm and isn't standard. So we can't translate it.. + */ + + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + } + + @Test + public void returnCodeTest() { + // Get Runtime Return Code. + long rtn = getReturnCode(); + // Verify the return code. + assertEquals("Return Code Failure: " + rtn, 1L, rtn); + } + + @Test + public void validateTableIssueCount() { + validateTableIssueCount("ext_purge_odd_parts", "web_sales", + Environment.LEFT, 2); + +// assertEquals("Issue Count not as expected", 3, +// getConversion().getDatabase("ext_purge_odd_parts") +// .getTableMirrors().get("web_sales") +// .getEnvironmentTable(Environment.LEFT).getIssues().size()); + } + +} diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_mmd.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_mmd.java index 76a667b9..991973c3 100644 --- a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_mmd.java +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_mmd.java @@ -41,6 +41,7 @@ "--hms-mirror.config.distcp=PULL", "--hms-mirror.config.filename=/config/default.yaml.cdp", "--hms-mirror.config.align-locations=true", + "--hms-mirror.config.storage-migration-strict=false", "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_02.yaml", "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_wd_epl_rdl_dc_mmd" } @@ -99,7 +100,7 @@ public void issueCountTest() { @Test public void phaseTest() { - validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.SUCCESS); } @Test @@ -107,7 +108,7 @@ public void returnCodeTest() { // Get Runtime Return Code. long rtn = getReturnCode(); // Verify the return code. - assertEquals("Return Code Failure: " + rtn, 1L, rtn); + assertEquals("Return Code Failure: " + rtn, 0L, rtn); } // @Test diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_mmd_strict.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_mmd_strict.java new file mode 100644 index 00000000..610ac90d --- /dev/null +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_mmd_strict.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2024. Cloudera, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp; + +import com.cloudera.utils.hms.mirror.PhaseState; +import com.cloudera.utils.hms.mirror.cli.Mirror; +import com.cloudera.utils.hms.mirror.domain.support.Environment; +import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Mirror.class, + args = { + "--hms-mirror.config.data-strategy=STORAGE_MIGRATION", + "--hms-mirror.config.warehouse-plans=ext_purge_odd_parts=/finance/external-fso:/finance/managed-fso", +// "--hms-mirror.config.warehouse-directory=/finance/managed-fso", +// "--hms-mirror.config.external-warehouse-directory=/finance/external-fso", + "--hms-mirror.config.target-namespace=ofs://OHOME90", + "--hms-mirror.config.distcp=PULL", + "--hms-mirror.config.filename=/config/default.yaml.cdp", + "--hms-mirror.config.align-locations=true", + "--hms-mirror.config.storage-migration-strict=true", + "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_02.yaml", + "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_wd_epl_rdl_dc_mmd" + } +) +@Slf4j +/* +STORAGE_MIGRATION test used to show how to move data from one directory to another, within the same namespace. + +Since the -smn is not specified, the namespace is assumed to be the same as the original table location. +The -wd and -ewd are used to define the warehouse directories. The -epl is used to evaluate the partition locations and +with -rdl, the default location is reset to the new warehouse directory. + +There should be no issue now that the default location is reset to the new warehouse directory. + + */ +public class Test_sm_wd_epl_rdl_dc_mmd_strict extends E2EBaseTest { + + // String[] args = new String[]{"-d", "STORAGE_MIGRATION", +// "-wd", "/finance/managed-fso", +// "-ewd", "/finance/external-fso", +// "-epl", +// "-rdl", +// "-dc", +// "-ltd", EXT_PURGE_ODD_PARTS_03, "-cfg", CDP_CDP, +// "-o", outputDir +// }; +// long rtn = 0; +// MirrorLegacy mirror = new MirrorLegacy(); +// rtn = mirror.go(args); +// assertEquals("Return Code Failure: " + rtn, 0, rtn); +// +// // Read the output and verify the results. +// DBMirror[] resultsMirrors = getResults(outputDir, EXT_PURGE_ODD_PARTS_03); +// +// validatePhase(resultsMirrors[0], "web_sales", PhaseState.SUCCESS); +// +// if (!validateSqlPair(resultsMirrors[0], Environment.LEFT, "web_sales", "Alter Table Location", +// "ALTER TABLE web_sales SET LOCATION \"hdfs://HDP50/finance/external-fso/ext_purge_odd_parts.db/web_sales\"")) { +// fail("Alter Table Location not found"); +// } +// if (!validateSqlPair(resultsMirrors[0], Environment.LEFT, "web_sales", +// "Alter Table Partition Spec `ws_sold_date_sk`='2451180' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451180') SET LOCATION \"hdfs://HDP50/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451180\"")) { +// fail("Alter Table Partition Location not found"); +// } +// if (!validateSqlPair(resultsMirrors[0], Environment.LEFT, "web_sales", +// "Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451188') SET LOCATION \"hdfs://HDP50/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451188\"")) { +// fail("Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location"); +// } + + @Test + public void issueCountTest() { + validateTableIssueCount("ext_purge_odd_parts", "web_sales", Environment.LEFT, 2); + } + + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + } + + @Test + public void returnCodeTest() { + // Get Runtime Return Code. + long rtn = getReturnCode(); + // Verify the return code. + assertEquals("Return Code Failure: " + rtn, 1L, rtn); + } + +// @Test +// public void sqlTest() { +// if (!validateSqlPair("ext_purge_odd_parts", Environment.LEFT, "web_sales", "Alter Table Location", +// "ALTER TABLE web_sales SET LOCATION \"hdfs://HOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales\"")) { +// fail("Alter Table Location not found"); +// } +// if (!validateSqlPair("ext_purge_odd_parts", Environment.LEFT, "web_sales", +// "Alter Table Partition Spec `ws_sold_date_sk`='2451180' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451180') SET LOCATION \"hdfs://HOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451180\"")) { +// fail("Alter Table Partition Location not found"); +// } +// if (!validateSqlPair("ext_purge_odd_parts", Environment.LEFT, "web_sales", +// "Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451188') SET LOCATION \"hdfs://HOME90/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451188\"")) { +// fail("Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location"); +// } +// } + +} diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_odd.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_odd.java index 28585dc5..1e8c64a4 100644 --- a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_odd.java +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_odd.java @@ -101,7 +101,7 @@ public void issueCountTest() { @Test public void phaseTest() { - validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.SUCCESS); } @Test @@ -113,7 +113,7 @@ public void returnCodeTest() { // GLM entries. // Verify the return code. - assertEquals("Return Code Failure: " + rtn, 1L, rtn); + assertEquals("Return Code Failure: " + rtn, 0L, rtn); } @Test diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_odd_strict.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_odd_strict.java new file mode 100644 index 00000000..2e150552 --- /dev/null +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_wd_epl_rdl_dc_odd_strict.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2024. Cloudera, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp; + +import com.cloudera.utils.hms.mirror.MessageCode; +import com.cloudera.utils.hms.mirror.PhaseState; +import com.cloudera.utils.hms.mirror.cli.Mirror; +import com.cloudera.utils.hms.mirror.domain.support.Environment; +import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Mirror.class, + args = { + "--hms-mirror.config.data-strategy=STORAGE_MIGRATION", + "--hms-mirror.config.target-namespace=ofs://OHOME90", + "--hms-mirror.config.warehouse-plans=ext_purge_odd_parts=/finance/external-fso:/finance/managed-fso", +// "--hms-mirror.config.warehouse-directory=/finance/managed-fso", +// "--hms-mirror.config.external-warehouse-directory=/finance/external-fso", +// "--hms-mirror.config.evaluate-partition-location=true", + "--hms-mirror.config.distcp=PULL", + "--hms-mirror.config.filename=/config/default.yaml.cdp", + "--hms-mirror.config.storage-migration-strict=true", + "--hms-mirror.conversion.test-filename=/test_data/ext_purge_odd_parts_01.yaml", + "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_wd_epl_rdl_dc_odd_strict" + } +) +@Slf4j +/* +STORAGE_MIGRATION test used to show how to move data from one directory to another, within the same namespace. + +Since the -smn is not specified, the namespace is assumed to be the same as the original table location. +The -wd and -ewd are used to define the warehouse directories. The -epl is used to evaluate the partition locations and +with -rdl, the default location is reset to the new warehouse directory. + +There should be no issue now that the default location is reset to the new warehouse directory. + + */ +public class Test_sm_wd_epl_rdl_dc_odd_strict extends E2EBaseTest { + + // String[] args = new String[]{"-d", "STORAGE_MIGRATION", +// "-wd", "/finance/managed-fso", +// "-ewd", "/finance/external-fso", +// "-epl", +// "-rdl", +// "-dc", +// "-ltd", EXT_PURGE_ODD_PARTS_03, "-cfg", CDP_CDP, +// "-o", outputDir +// }; +// long rtn = 0; +// MirrorLegacy mirror = new MirrorLegacy(); +// rtn = mirror.go(args); +// assertEquals("Return Code Failure: " + rtn, 0, rtn); +// +// // Read the output and verify the results. +// DBMirror[] resultsMirrors = getResults(outputDir, EXT_PURGE_ODD_PARTS_03); +// +// validatePhase(resultsMirrors[0], "web_sales", PhaseState.SUCCESS); +// +// if (!validateSqlPair(resultsMirrors[0], Environment.LEFT, "web_sales", "Alter Table Location", +// "ALTER TABLE web_sales SET LOCATION \"hdfs://HDP50/finance/external-fso/ext_purge_odd_parts.db/web_sales\"")) { +// fail("Alter Table Location not found"); +// } +// if (!validateSqlPair(resultsMirrors[0], Environment.LEFT, "web_sales", +// "Alter Table Partition Spec `ws_sold_date_sk`='2451180' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451180') SET LOCATION \"hdfs://HDP50/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451180\"")) { +// fail("Alter Table Partition Location not found"); +// } +// if (!validateSqlPair(resultsMirrors[0], Environment.LEFT, "web_sales", +// "Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location", +// "ALTER TABLE web_sales PARTITION (`ws_sold_date_sk`='2451188') SET LOCATION \"hdfs://HDP50/finance/external-fso/ext_purge_odd_parts.db/web_sales/ws_sold_date_sk=2451188\"")) { +// fail("Alter Table Partition Spec `ws_sold_date_sk`='2451188' Location"); +// } + + @Test + public void issueCountTest() { + validateTableIssueCount("ext_purge_odd_parts", "web_sales", Environment.LEFT, 2); + } + + @Test + public void phaseTest() { + validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.ERROR); + } + + @Test + public void returnCodeTest() { + // Get Runtime Return Code. + long rtn = getReturnCode(); + + // Has non-standard partition locations which can't be translated without additional + // GLM entries. + + // Verify the return code. + assertEquals("Return Code Failure: " + rtn, 1L, rtn); + } + + @Test + public void warningCodeTest() { + // Get Runtime Return Code. + long actual = getWarningCode(); + // Verify the return code. + long expected = getCheckCode( + MessageCode.ALIGNED_DISTCP_EXECUTE, + MessageCode.DATASTRATEGY_FILTER_CONTROLLED_BY, +// MessageCode.RDL_DC_WARNING_TABLE_ALIGNMENT, +// MessageCode.STORAGE_MIGRATION_NAMESPACE_LEFT, + MessageCode.DISTCP_WO_TABLE_FILTERS + ); + + assertEquals("Warning Code Failure: ", expected, actual); + + } + +}