From c2e0dacbcaa3c1887041513ae6354285eaf4032d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Wed, 22 Jan 2025 10:18:30 +0100 Subject: [PATCH 1/7] fix(model): fixes DashboardContainsDashboard relationship in DashboardInfo aspect --- .../pegasus/com/linkedin/dashboard/DashboardInfo.pdl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl index 0ce19b32c8930..877d42f52b1a1 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl @@ -111,11 +111,11 @@ record DashboardInfo includes CustomProperties, ExternalReference { "name": "DashboardContainsDashboard", "entityTypes": [ "dashboard" ], "isLineage": true, - "createdOn": "datasetEdges/*/created/time" - "createdActor": "datasetEdges/*/created/actor" - "updatedOn": "datasetEdges/*/lastModified/time" - "updatedActor": "datasetEdges/*/lastModified/actor" - "properties": "datasetEdges/*/properties" + "createdOn": "dashboards/*/created/time" + "createdActor": "dashboards/*/created/actor" + "updatedOn": "dashboards/*/lastModified/time" + "updatedActor": "dashboards/*/lastModified/actor" + "properties": "dashboards/*/properties" } } dashboards: array[Edge] = [ ] From cad0b3756c9786d07ae47507f1229a071514b0f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Thu, 23 Jan 2025 07:47:20 +0100 Subject: [PATCH 2/7] Update updating-datahub.md --- docs/how/updating-datahub.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 945a00088f1f7..beece8884aa08 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -28,6 +28,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Other Notable Changes +- #12433: Fixes the searchable annotations in the model supporting `Dashboard` to `Dashboard` lineage within the `DashboardInfo` aspect. Users of Sigma and PowerBI Apps ingestion may need to run a targeted reindex to update metadata on these edges. + ## 0.15.0 - OpenAPI Update: PIT Keep Alive parameter added to scroll endpoints. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set. From 43072d506eb1a0d2c6aa4a10079fe026f805b8fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Fri, 24 Jan 2025 17:32:35 +0100 Subject: [PATCH 3/7] fix chardEdges destination type validation in DashboardPatchBuilder + add missing case for dashboards --- .../api/incremental_lineage_helper.py | 4 + .../src/datahub/specific/dashboard.py | 44 +++++- .../test_incremental_lineage_helper.py | 138 +++++++++++++++++- 3 files changed, 183 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py index 92ee158661d3d..47e7e4b276309 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py @@ -102,6 +102,10 @@ def convert_dashboard_info_to_patch( if aspect.datasets: patch_builder.add_datasets(aspect.datasets) + if aspect.dashboards: + for dashboard in aspect.dashboards: + patch_builder.add_dashboard(dashboard) + if aspect.access: patch_builder.set_access(aspect.access) diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py index 515fcf0c6da95..1dc97dc42a7b4 100644 --- a/metadata-ingestion/src/datahub/specific/dashboard.py +++ b/metadata-ingestion/src/datahub/specific/dashboard.py @@ -161,7 +161,7 @@ def add_chart_edge(self, chart: Union[Edge, Urn, str]) -> "DashboardPatchBuilder lastModified=self._mint_auditstamp(), ) - self._ensure_urn_type("dataset", [chart_edge], "add_chart_edge") + self._ensure_urn_type("chart", [chart_edge], "add_chart_edge") self._add_patch( DashboardInfo.ASPECT_NAME, "add", @@ -271,6 +271,48 @@ def add_datasets( return self + def add_dashboard( + self, dashboard: Union[Edge, Urn, str] + ) -> "DashboardPatchBuilder": + """ + Adds an dashboard to the DashboardPatchBuilder. + + Args: + dashboard: The dashboard, which can be an Edge object, Urn object, or a string. + + Returns: + The DashboardPatchBuilder instance. + + Raises: + ValueError: If the dashboard is not a Dashboard urn. + + Notes: + If `dashboard` is an Edge object, it is used directly. If `dashboard` is a Urn object or string, + it is converted to an Edge object and added with default audit stamps. + """ + if isinstance(dashboard, Edge): + dashboard_urn: str = dashboard.destinationUrn + dashboard_edge: Edge = dashboard + elif isinstance(dashboard, (Urn, str)): + dashboard_urn = str(dashboard) + if not dashboard_urn.startswith("urn:li:dashboard:"): + raise ValueError(f"Input {dashboard} is not a Dashboard urn") + + dashboard_edge = Edge( + destinationUrn=dashboard_urn, + created=self._mint_auditstamp(), + lastModified=self._mint_auditstamp(), + ) + + self._ensure_urn_type("dashboard", [dashboard_edge], "add_dashboard") + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path=("dashboards", dashboard_urn), + value=dashboard_edge, + ) + return self + def set_dashboard_url( self, dashboard_url: Optional[str] ) -> "DashboardPatchBuilder": diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py index c5c4a378894c3..6dd2332fec2ba 100644 --- a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py +++ b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py @@ -1,11 +1,19 @@ -from typing import List +import re +import unittest +from typing import List, Optional + +import pytest import datahub.emitter.mce_builder as builder import datahub.metadata.schema_classes as models from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage +from datahub.ingestion.api.incremental_lineage_helper import ( + auto_incremental_lineage, + convert_dashboard_info_to_patch, +) from datahub.ingestion.api.source_helpers import auto_workunit +from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.sink.file import write_metadata_file from tests.test_helpers import mce_helpers @@ -183,3 +191,129 @@ def test_incremental_lineage_pass_through(tmp_path, pytestconfig): mce_helpers.check_golden_file( pytestconfig=pytestconfig, output_path=test_file, golden_path=golden_file ) + + +class TestConvertDashboardInfoToPatch(unittest.TestCase): + def setUp(self): + self.urn = builder.make_dashboard_urn( + platform="platform", name="test", platform_instance="instance" + ) + self.system_metadata = models.SystemMetadataClass() + self.aspect = models.DashboardInfoClass( + title="Test Dashboard", + description="This is a test dashboard", + lastModified=models.ChangeAuditStampsClass(), + ) + + def _validate_dashboard_info_patch( + self, result: Optional[MetadataWorkUnit], expected_aspect_value: bytes + ) -> None: + assert ( + result + and result.metadata + and isinstance(result.metadata, models.MetadataChangeProposalClass) + ) + mcp: models.MetadataChangeProposalClass = result.metadata + assert ( + mcp.entityUrn == self.urn + and mcp.entityType == "dashboard" + and mcp.aspectName == "dashboardInfo" + and mcp.changeType == "PATCH" + ) + assert mcp.aspect and isinstance(mcp.aspect, models.GenericAspectClass) + aspect: models.GenericAspectClass = mcp.aspect + assert aspect.value == expected_aspect_value + + def test_convert_dashboard_info_to_patch_with_no_additional_values(self): + result = convert_dashboard_info_to_patch( + self.urn, self.aspect, self.system_metadata + ) + self._validate_dashboard_info_patch( + result=result, + expected_aspect_value=b'[{"op": "add", "path": "/title", "value": "Test Dashboard"}, {"op": "add", "path": "/description", "value": "This is a test dashboard"}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}]', + ) + + def test_convert_dashboard_info_to_patch_with_custom_properties(self): + self.aspect.customProperties = { + "key1": "value1", + "key2": "value2", + } + result = convert_dashboard_info_to_patch( + self.urn, self.aspect, self.system_metadata + ) + self._validate_dashboard_info_patch( + result=result, + expected_aspect_value=b'[{"op": "add", "path": "/customProperties/key1", "value": "value1"}, {"op": "add", "path": "/customProperties/key2", "value": "value2"}, {"op": "add", "path": "/title", "value": "Test Dashboard"}, {"op": "add", "path": "/description", "value": "This is a test dashboard"}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}]', + ) + + def test_convert_dashboard_info_to_patch_with_chart_edges(self): + self.aspect.chartEdges = [ + models.EdgeClass( + destinationUrn=builder.make_chart_urn( + platform="platform", + name="target-test", + platform_instance="instance", + ), + ) + ] + result = convert_dashboard_info_to_patch( + self.urn, self.aspect, self.system_metadata + ) + self._validate_dashboard_info_patch( + result, + b'[{"op": "add", "path": "/chartEdges/urn:li:chart:(platform,instance.target-test)", "value": {"destinationUrn": "urn:li:chart:(platform,instance.target-test)"}}, {"op": "add", "path": "/title", "value": "Test Dashboard"}, {"op": "add", "path": "/description", "value": "This is a test dashboard"}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}]', + ) + + def test_convert_dashboard_info_to_patch_with_chart_edges_no_chart(self): + self.aspect.chartEdges = [ + models.EdgeClass( + destinationUrn=builder.make_dashboard_urn( + platform="platform", + name="target-test", + platform_instance="instance", + ), + ) + ] + with pytest.raises( + ValueError, + match=re.escape( + "add_chart_edge: urn:li:dashboard:(platform,instance.target-test) is not of type chart" + ), + ): + convert_dashboard_info_to_patch(self.urn, self.aspect, self.system_metadata) + + def test_convert_dashboard_info_to_patch_with_dashboards(self): + self.aspect.dashboards = [ + models.EdgeClass( + destinationUrn=builder.make_dashboard_urn( + platform="platform", + name="target-test", + platform_instance="instance", + ), + ) + ] + result = convert_dashboard_info_to_patch( + self.urn, self.aspect, self.system_metadata + ) + self._validate_dashboard_info_patch( + result, + b'[{"op": "add", "path": "/title", "value": "Test Dashboard"}, {"op": "add", "path": "/description", "value": "This is a test dashboard"}, {"op": "add", "path": "/dashboards/urn:li:dashboard:(platform,instance.target-test)", "value": {"destinationUrn": "urn:li:dashboard:(platform,instance.target-test)"}}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}]', + ) + + def test_convert_dashboard_info_to_patch_with_dashboards_no_dashboard(self): + self.aspect.dashboards = [ + models.EdgeClass( + destinationUrn=builder.make_chart_urn( + platform="platform", + name="target-test", + platform_instance="instance", + ), + ) + ] + with pytest.raises( + ValueError, + match=re.escape( + "add_dashboard: urn:li:chart:(platform,instance.target-test) is not of type dashboard" + ), + ): + convert_dashboard_info_to_patch(self.urn, self.aspect, self.system_metadata) From 3c330638fb7847b0f8fa4723f346784dfed021d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Mon, 27 Jan 2025 12:06:04 +0100 Subject: [PATCH 4/7] fix missing "dashboards" field in DashboardInfoTemplate --- .../builder/DashboardInfoPatchBuilder.java | 23 +++++++++++++++++++ .../dashboard/DashboardInfoTemplate.java | 13 +++++++++++ .../template/DashboardInfoTemplateTest.java | 22 ++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java index 9156b304a394e..1e398e1daddd9 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.Edge; import com.linkedin.common.urn.ChartUrn; +import com.linkedin.common.urn.DashboardUrn; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.aspect.patch.PatchOperationType; @@ -19,6 +20,7 @@ public class DashboardInfoPatchBuilder extends AbstractMultiFieldPatchBuilder { private static final String CHART_EDGES_PATH_START = "/chartEdges/"; private static final String DATASET_EDGES_PATH_START = "/datasetEdges/"; + private static final String DASHBOARDS_PATH_START = "/dashboards/"; // Simplified with just Urn public DashboardInfoPatchBuilder addChartEdge(@Nonnull ChartUrn urn) { @@ -36,6 +38,7 @@ public DashboardInfoPatchBuilder removeChartEdge(@Nonnull ChartUrn urn) { return this; } + // Simplified with just Urn public DashboardInfoPatchBuilder addDatasetEdge(@Nonnull DatasetUrn urn) { ObjectNode value = createEdgeValue(urn); @@ -52,6 +55,22 @@ public DashboardInfoPatchBuilder removeDatasetEdge(@Nonnull DatasetUrn urn) { return this; } + // Simplified with just Urn + public DashboardInfoPatchBuilder addDashboard(@Nonnull DashboardUrn urn) { + ObjectNode value = createEdgeValue(urn); + + pathValues.add( + ImmutableTriple.of(PatchOperationType.ADD.getValue(), DASHBOARDS_PATH_START + urn, value)); + return this; + } + + public DashboardInfoPatchBuilder removeDashboard(@Nonnull DashboardUrn urn) { + pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), DASHBOARDS_PATH_START + urn, null)); + return this; + } + // Full Edge modification public DashboardInfoPatchBuilder addEdge(@Nonnull Edge edge) { ObjectNode value = createEdgeValue(edge); @@ -87,6 +106,10 @@ private String getEdgePath(@Nonnull Edge edge) { return CHART_EDGES_PATH_START + destinationUrn; } + if (DASHBOARD_ENTITY_NAME.equals(destinationUrn.getEntityType())) { + return DASHBOARDS_PATH_START + destinationUrn; + } + // TODO: Output Data Jobs not supported by aspect, add here if this changes throw new IllegalArgumentException( diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java index 85ce06b01c1d7..1eccb6254a2c9 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java @@ -22,6 +22,7 @@ public class DashboardInfoTemplate implements ArrayMergingTemplate Date: Tue, 28 Jan 2025 07:29:44 +0100 Subject: [PATCH 5/7] doc restore-indices --- docs/how/updating-datahub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index beece8884aa08..312953d46f1ed 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -28,7 +28,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Other Notable Changes -- #12433: Fixes the searchable annotations in the model supporting `Dashboard` to `Dashboard` lineage within the `DashboardInfo` aspect. Users of Sigma and PowerBI Apps ingestion may need to run a targeted reindex to update metadata on these edges. +- #12433: Fixes the searchable annotations in the model supporting `Dashboard` to `Dashboard` lineage within the `DashboardInfo` aspect. Users of Sigma and PowerBI Apps ingestion may need to run a targeted [reindex](https://datahubproject.io/docs/how/restore-indices/) to update metadata on these edges. ## 0.15.0 From 49e0dc2738303ae381c3e2b3be1e3436649c06df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Wed, 29 Jan 2025 09:52:47 +0100 Subject: [PATCH 6/7] Update PatchTest.java --- .../src/test/java/datahub/client/patch/PatchTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java index 95d2060079780..e7bba709f238d 100644 --- a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java @@ -10,6 +10,7 @@ import com.linkedin.common.TagAssociation; import com.linkedin.common.urn.ChartUrn; import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.DashboardUrn; import com.linkedin.common.urn.DataJobUrn; import com.linkedin.common.urn.DataPlatformUrn; import com.linkedin.common.urn.DatasetUrn; @@ -622,6 +623,9 @@ public void testLocalDashboardInfoAdd() { DatasetUrn.createFromString( "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleHiveDataset,PROD)")) .addChartEdge(ChartUrn.createFromString("urn:li:chart:(dashboartTool, chartId)")) + .addDashboard( + DashboardUrn.createFromString( + "urn:li:dashboard:(dashboardTool, childDashboardId)")) .build(); Future response = restEmitter.emit(dashboardInfoPatch); @@ -644,6 +648,9 @@ public void testLocalDashboardInfoRemove() { DatasetUrn.createFromString( "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleHiveDataset,PROD)")) .removeChartEdge(ChartUrn.createFromString("urn:li:chart:(dashboardTool, chartId)")) + .removeDashboard( + DashboardUrn.createFromString( + "urn:li:dashboard:(dashboardTool, childDashboardId)")) .build(); Future response = restEmitter.emit(dashboardInfoPatch); From b582b473e82ff6543d01fd63d4faffa2a5836876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Wed, 29 Jan 2025 09:59:13 +0100 Subject: [PATCH 7/7] Reindex DashboardInfo --- .../ReindexDashboardInfoConfig.java | 30 ++++++++++++ .../dashboardinfo/ReindexDashboardInfo.java | 49 +++++++++++++++++++ .../ReindexDashboardInfoStep.java | 42 ++++++++++++++++ .../src/main/resources/application.yaml | 5 ++ 4 files changed, 126 insertions(+) create mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexDashboardInfoConfig.java create mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfo.java create mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfoStep.java diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexDashboardInfoConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexDashboardInfoConfig.java new file mode 100644 index 0000000000000..54beed11f9cb4 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/restoreindices/ReindexDashboardInfoConfig.java @@ -0,0 +1,30 @@ +package com.linkedin.datahub.upgrade.config.restoreindices; + +import com.linkedin.datahub.upgrade.config.SystemUpdateCondition; +import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; +import com.linkedin.datahub.upgrade.system.restoreindices.dashboardinfo.ReindexDashboardInfo; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; + +@Configuration +@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class) +public class ReindexDashboardInfoConfig { + + @Bean + public NonBlockingSystemUpgrade reindexDashboardInfo( + final OperationContext opContext, + final EntityService entityService, + final AspectDao aspectDao, + @Value("${systemUpdate.dashboardInfo.enabled}") final boolean enabled, + @Value("${systemUpdate.dashboardInfo.batchSize}") final Integer batchSize, + @Value("${systemUpdate.dashboardInfo.delayMs}") final Integer delayMs, + @Value("${systemUpdate.dashboardInfo.limit}") final Integer limit) { + return new ReindexDashboardInfo( + opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfo.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfo.java new file mode 100644 index 0000000000000..dd9a6c6f01472 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfo.java @@ -0,0 +1,49 @@ +package com.linkedin.datahub.upgrade.system.restoreindices.dashboardinfo; + +import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; +import java.util.List; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +/** + * A job that reindexes all dashboard info aspects as part of reindexing dashboards relationship. + * This is required to fix the dashboards relationships for dashboards + */ +@Slf4j +public class ReindexDashboardInfo implements NonBlockingSystemUpgrade { + + private final List _steps; + + public ReindexDashboardInfo( + @Nonnull OperationContext opContext, + EntityService entityService, + AspectDao aspectDao, + boolean enabled, + Integer batchSize, + Integer batchDelayMs, + Integer limit) { + if (enabled) { + _steps = + ImmutableList.of( + new ReindexDashboardInfoStep( + opContext, entityService, aspectDao, batchSize, batchDelayMs, limit)); + } else { + _steps = ImmutableList.of(); + } + } + + @Override + public String id() { + return this.getClass().getName(); + } + + @Override + public List steps() { + return _steps; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfoStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfoStep.java new file mode 100644 index 0000000000000..7d0780c18c29c --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/restoreindices/dashboardinfo/ReindexDashboardInfoStep.java @@ -0,0 +1,42 @@ +package com.linkedin.datahub.upgrade.system.restoreindices.dashboardinfo; + +import static com.linkedin.metadata.Constants.*; + +import com.linkedin.datahub.upgrade.system.AbstractMCLStep; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import io.datahubproject.metadata.context.OperationContext; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; + +@Slf4j +public class ReindexDashboardInfoStep extends AbstractMCLStep { + + public ReindexDashboardInfoStep( + OperationContext opContext, + EntityService entityService, + AspectDao aspectDao, + Integer batchSize, + Integer batchDelayMs, + Integer limit) { + super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit); + } + + @Override + public String id() { + return "dashboard-info-v1"; + } + + @Nonnull + @Override + protected String getAspectName() { + return DASHBOARD_INFO_ASPECT_NAME; + } + + @Nullable + @Override + protected String getUrnLike() { + return "urn:li:" + DASHBOARD_ENTITY_NAME + ":%"; + } +} diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 04f7409d5c39a..a990a8100ff76 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -403,6 +403,11 @@ systemUpdate: batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_BATCH_SIZE:1000} delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_DELAY_MS:30000} limit: ${BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_CLL_LIMIT:0} + dashboardInfo: + enabled: ${BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_ENABLED:true} + batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_BATCH_SIZE:1000} + delayMs: ${BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_DELAY_MS:30000} + limit: ${BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_CLL_LIMIT:0} browsePathsV2: enabled: ${BOOTSTRAP_SYSTEM_UPDATE_BROWSE_PATHS_V2_ENABLED:true} batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_BROWSE_PATHS_V2_BATCH_SIZE:5000}