CORE-426 Optimize duplicate query (sodadata#1781)

* CORE-426 Optimize duplicate query
atlanhq · Feb 6, 2023 · f000c70 · f000c70
1 parent 13f3e20
commit f000c70
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 13 deletions.
diff --git a/soda/core/soda/execution/data_source.py b/soda/core/soda/execution/data_source.py
@@ -713,20 +713,17 @@ def sql_get_duplicates_count(
         column_names: str,
         table_name: str,
         filter: str,
-        exclude_patterns: list[str] | None = None,
     ) -> str | None:
-        main_query_columns = f"{column_names}, frequency" if exclude_patterns else "*"
         sql = dedent(
             f"""
             WITH frequencies AS (
-              SELECT {column_names}, COUNT(*) AS frequency
+              SELECT COUNT(*) AS frequency
               FROM {table_name}
               WHERE {filter}
               GROUP BY {column_names})
-            SELECT {main_query_columns}
+            SELECT count(*)
             FROM frequencies
-            WHERE frequency > 1
-            ORDER BY frequency DESC"""
+            WHERE frequency > 1"""
         )
 
         return sql

diff --git a/soda/core/soda/execution/query/duplicates_query.py b/soda/core/soda/execution/query/duplicates_query.py
@@ -47,7 +47,6 @@ def __init__(self, partition: "Partition", metric: "Metric"):
                 column_names,
                 table_name,
                 values_filter,
-                exclude_patterns=exclude_patterns,
             )
         )
 
@@ -96,10 +95,11 @@ def __init__(self, partition: "Partition", metric: "Metric"):
         )
 
     def execute(self):
-        self.fetchall()
-        self.metric.set_value(len(self.rows))
+        self.fetchone()
+        duplicates_count = self.row[0]
+        self.metric.set_value(duplicates_count)
 
-        if self.rows:
+        if duplicates_count:
             sample_query = SampleQuery(
                 self.data_source_scan,
                 self.metric,

diff --git a/soda/core/tests/data_source/test_duplicates.py b/soda/core/tests/data_source/test_duplicates.py
@@ -17,7 +17,7 @@ def test_duplicates_single_column(data_source_fixture: DataSourceFixture):
     scan.assert_all_checks_pass()
 
     # This is a simple use case, verify that * is used in the main query.
-    scan.assert_log("SELECT *")
+    scan.assert_log("count(*)")
 
 
 def test_duplicates_multiple_columns(data_source_fixture: DataSourceFixture):
@@ -51,8 +51,8 @@ def test_duplicates_with_exclude_columns(data_source_fixture: DataSourceFixture)
     scan.assert_all_checks_pass()
 
     # Exclude columns present, query should list the columns explicitly
-    scan.assert_log("SELECT cat, frequency")
-    scan.assert_no_log("SELECT *")
+    scan.assert_log("cat, frequency")
+    scan.assert_no_log(" * ")
 
 
 def test_duplicates_with_filter(data_source_fixture: DataSourceFixture):