diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index a2d69d9e552916..04bc51f1ebd3f5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -491,15 +491,25 @@ def fetch_secure_view_definition( try: view_definitions = self.data_dictionary.get_secure_view_definitions() return view_definitions[db_name][schema_name][table_name] + except KeyError: + # Received secure view definitions but the view is not present in results + self.structured_reporter.info( + title="Secure view definition not found", + message="Lineage will be missing for the view.", + context=f"{db_name}.{schema_name}.{table_name}", + ) + return None except Exception as e: - if isinstance(e, SnowflakePermissionError): - error_msg = ( - "Failed to get secure views definitions. Please check permissions." - ) - else: - error_msg = "Failed to get secure views definitions" + action_msg = ( + "Please check permissions." + if isinstance(e, SnowflakePermissionError) + else "" + ) + self.structured_reporter.warning( - error_msg, + title="Failed to get secure views definitions", + message=f"Lineage will be missing for the view. {action_msg}", + context=f"{db_name}.{schema_name}.{table_name}", exc=e, ) return None diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py index de6e996a52642b..4cb6cec4906efa 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py @@ -260,3 +260,63 @@ def test_snowflake_missing_snowflake_operations_permission_causes_pipeline_failu assert "usage-permission-error" in [ failure.message for failure in pipeline.source.get_report().failures ] + + +@freeze_time(FROZEN_TIME) +def test_snowflake_missing_snowflake_secure_view_definitions_raises_pipeline_info( + pytestconfig, + snowflake_pipeline_config, +): + with mock.patch("snowflake.connector.connect") as mock_connect: + sf_connection = mock.MagicMock() + sf_cursor = mock.MagicMock() + mock_connect.return_value = sf_connection + sf_connection.cursor.return_value = sf_cursor + + # Empty secure view definitions + sf_cursor.execute.side_effect = query_permission_response_override( + default_query_results, + [snowflake_query.SnowflakeQuery.get_secure_view_definitions()], + [], + ) + pipeline = Pipeline(snowflake_pipeline_config) + pipeline.run() + + pipeline.raise_from_status(raise_warnings=True) + assert pipeline.source.get_report().infos.as_obj() == [ + { + "title": "Secure view definition not found", + "message": "Lineage will be missing for the view.", + "context": ["TEST_DB.TEST_SCHEMA.VIEW_1"], + } + ] + + +@freeze_time(FROZEN_TIME) +def test_snowflake_failed_secure_view_definitions_query_raises_pipeline_warning( + pytestconfig, + snowflake_pipeline_config, +): + with mock.patch("snowflake.connector.connect") as mock_connect: + sf_connection = mock.MagicMock() + sf_cursor = mock.MagicMock() + mock_connect.return_value = sf_connection + sf_connection.cursor.return_value = sf_cursor + + # Error in getting secure view definitions + sf_cursor.execute.side_effect = query_permission_error_override( + default_query_results, + [snowflake_query.SnowflakeQuery.get_secure_view_definitions()], + "Database 'SNOWFLAKE' does not exist or not authorized.", + ) + pipeline = Pipeline(snowflake_pipeline_config) + pipeline.run() + assert pipeline.source.get_report().warnings.as_obj() == [ + { + "title": "Failed to get secure views definitions", + "message": "Lineage will be missing for the view. Please check permissions.", + "context": [ + "TEST_DB.TEST_SCHEMA.VIEW_1 : Database 'SNOWFLAKE' does not exist or not authorized." + ], + } + ]