Use Dataverse metadata to improve reverse engineering

.github/workflows/vsix.yml

@@ -58,7 +58,7 @@ jobs:
         mkdir vsix
         7z x src/GUI/lib/efreveng80.exe.zip -oefreveng80 -y
         dir /a:-d /s /b "efreveng80" | find /c ":\" > filecount.txt
-        findstr "157" filecount.txt
+        findstr "166" filecount.txt
 
     - name: Extract and verify efreveng90.exe.zip file count
       if: github.event_name != 'pull_request'
@@ -67,7 +67,7 @@ jobs:
           mkdir vsix
           7z x src/GUI/lib/efreveng90.exe.zip -oefreveng90 -y
           dir /a:-d /s /b "efreveng90" | find /c ":\" > filecount.txt
-          findstr "156" filecount.txt
+          findstr "165" filecount.txt
 
     - name: Setup MSBuild.exe
       uses: microsoft/setup-msbuild@v2

src/Core/RevEng.Core.80/DataverseModelFactoryExtension.cs

@@ -0,0 +1,196 @@
+using System;
+using System.Collections.Generic;
+using System.Data.Common;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Data.SqlClient;
+using Microsoft.EntityFrameworkCore.Scaffolding.Metadata;
+using Microsoft.PowerPlatform.Dataverse.Client;
+using Microsoft.Xrm.Sdk.Messages;
+using Microsoft.Xrm.Sdk.Metadata;
+using Microsoft.Xrm.Sdk.Metadata.Query;
+
+namespace RevEng.Core
+{
+    public class DataverseModelFactoryExtension
+    {
+        private readonly ServiceClient serviceClient;
+
+        public DataverseModelFactoryExtension(ServiceClient serviceClient)
+        {
+            this.serviceClient = serviceClient;
+        }
+
+        /// <summary>
+        /// Checks if the connection is for a Dataverse instance and creates a new <see cref="DataverseModelFactoryExtension"/>
+        /// if so.
+        /// </summary>
+        /// <param name="connection">The database connection to use to connect to Dataverse.</param>
+        /// <param name="dataverse">Set to a new <see cref="DataverseModelFactoryExtension"/> if the <paramref name="connection"/> represents a connection to a Dataverse TDS Endpoint.</param>
+        /// <returns><see langword="true"/> if the <paramref name="connection"/> represents a connection to a Dataverse TDS Endpoint, or <see langword="false"/> otherwise.</returns>
+        public static bool TryCreate(DbConnection connection, out DataverseModelFactoryExtension dataverse)
+        {
+            ArgumentNullException.ThrowIfNull(connection);
+
+            var connectionStringParser = new SqlConnectionStringBuilder(connection.ConnectionString);
+            if (connection is SqlConnection sqlConnection &&
+                connectionStringParser.DataSource.Contains("dynamics.com", StringComparison.OrdinalIgnoreCase))
+            {
+                connectionStringParser.Authentication = SqlAuthenticationMethod.NotSpecified;
+                connection.ConnectionString = connectionStringParser.ToString();
+                var serviceClient = new ServiceClient($"AuthType=OAuth;Username={connectionStringParser.UserID};Url=https://{connectionStringParser.DataSource};AppId=51f81489-12ee-4a9e-aaae-a2591f45987d;RedirectUri=http://localhost;LoginPrompt=Auto");
+                sqlConnection.AccessTokenCallback = (_, __) => Task.FromResult(new SqlAuthenticationToken(serviceClient.CurrentAccessToken, DateTimeOffset.MaxValue));
+
+                dataverse = new DataverseModelFactoryExtension(serviceClient);
+                return true;
+            }
+
+            dataverse = null;
+            return false;
+        }
+
+        /// <summary>
+        /// Updates the <paramref name="tables"/> with metadata from the Dataverse instance.
+        /// </summary>
+        /// <param name="tables">The table definitions that have already been loaded from the Dataverse TDS Endpoint.</param>
+        public void GetDataverseMetadata(List<DatabaseTable> tables)
+        {
+            var metadataQuery = new EntityQueryExpression
+            {
+                Properties = new MetadataPropertiesExpression(
+                    nameof(EntityMetadata.LogicalName),
+                    nameof(EntityMetadata.SchemaName),
+                    nameof(EntityMetadata.PrimaryIdAttribute),
+                    nameof(EntityMetadata.Attributes),
+                    nameof(EntityMetadata.ManyToOneRelationships),
+                    nameof(EntityMetadata.Keys)),
+                AttributeQuery = new AttributeQueryExpression
+                {
+                    Properties = new MetadataPropertiesExpression(
+                        nameof(AttributeMetadata.LogicalName),
+                        nameof(AttributeMetadata.SchemaName)),
+                },
+                RelationshipQuery = new RelationshipQueryExpression
+                {
+                    Properties = new MetadataPropertiesExpression(
+                        nameof(OneToManyRelationshipMetadata.SchemaName),
+                        nameof(OneToManyRelationshipMetadata.ReferencingAttribute),
+                        nameof(OneToManyRelationshipMetadata.ReferencedEntity),
+                        nameof(OneToManyRelationshipMetadata.ReferencedAttribute),
+                        nameof(OneToManyRelationshipMetadata.CascadeConfiguration)),
+                },
+                KeyQuery = new EntityKeyQueryExpression
+                {
+                    Properties = new MetadataPropertiesExpression(
+                        nameof(EntityKeyMetadata.SchemaName),
+                        nameof(EntityKeyMetadata.KeyAttributes)),
+                },
+            };
+            var metadata = (RetrieveMetadataChangesResponse)serviceClient.Execute(new RetrieveMetadataChangesRequest { Query = metadataQuery });
+
+            foreach (var entity in metadata.EntityMetadata)
+            {
+                // Check if the entity is in the table list
+                var table = tables.SingleOrDefault(t => t.Name == entity.LogicalName);
+                if (table is null)
+                {
+                    continue;
+                }
+
+                // Use the schema names for tables and columns instead of the default logical names for more standard .NET naming
+                // Only switch to the schema names if they are the same as the logical name except in a different case.
+                if (entity.SchemaName != entity.LogicalName && entity.LogicalName.Equals(entity.SchemaName, StringComparison.OrdinalIgnoreCase))
+                {
+                    table.Name = entity.SchemaName;
+                }
+
+                foreach (var attr in entity.Attributes)
+                {
+                    var col = table.Columns.SingleOrDefault(c => c.Name == attr.LogicalName);
+
+                    if (col != null && attr.SchemaName != attr.LogicalName && attr.LogicalName.Equals(attr.SchemaName, StringComparison.OrdinalIgnoreCase))
+                    {
+                        col.Name = attr.SchemaName;
+                    }
+                }
+
+                // Add the primary key column
+                table.PrimaryKey = new DatabasePrimaryKey
+                {
+                    Table = table,
+                    Name = entity.PrimaryIdAttribute,
+                    Columns =
+                    {
+                        table.Columns.Single(c => c.Name.Equals(entity.PrimaryIdAttribute, StringComparison.OrdinalIgnoreCase)),
+                    },
+                };
+
+                // Add the alternate keys
+                foreach (var key in entity.Keys)
+                {
+                    var uniqueConstraint = new DatabaseUniqueConstraint
+                    {
+                        Table = table,
+                        Name = key.SchemaName,
+                    };
+
+                    var hasAllColumns = true;
+
+                    foreach (var attr in key.KeyAttributes)
+                    {
+                        var col = table.Columns.SingleOrDefault(c => c.Name.Equals(attr, StringComparison.OrdinalIgnoreCase));
+
+                        if (col != null)
+                        {
+                            uniqueConstraint.Columns.Add(col);
+                        }
+                        else
+                        {
+                            hasAllColumns = false;
+                        }
+                    }
+
+                    if (hasAllColumns)
+                    {
+                        table.UniqueConstraints.Add(uniqueConstraint);
+                    }
+                }
+
+                // Add the foreign key relationships
+                foreach (var relationship in entity.ManyToOneRelationships)
+                {
+                    var referencedTable = tables.SingleOrDefault(t => t.Name.Equals(relationship.ReferencedEntity, StringComparison.OrdinalIgnoreCase));
+
+                    if (referencedTable is null)
+                    {
+                        continue;
+                    }
+
+                    var referencingColumn = table.Columns.SingleOrDefault(c => c.Name.Equals(relationship.ReferencingAttribute, StringComparison.OrdinalIgnoreCase));
+                    var referencedColumn = referencedTable.Columns.SingleOrDefault(c => c.Name.Equals(relationship.ReferencedAttribute, StringComparison.OrdinalIgnoreCase));
+
+                    if (referencingColumn is null || referencedColumn is null)
+                    {
+                        continue;
+                    }
+
+                    table.ForeignKeys.Add(new DatabaseForeignKey
+                    {
+                        Table = table,
+                        PrincipalTable = referencedTable,
+                        Name = relationship.SchemaName,
+                        Columns =
+                        {
+                            referencingColumn,
+                        },
+                        PrincipalColumns =
+                        {
+                            referencedColumn,
+                        },
+                    });
+                }
+            }
+        }
+    }
+}

src/Core/RevEng.Core.80/PatchedSqlServerDatabaseModelFactory.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Data;
 using System.Data.Common;
@@ -7,6 +7,7 @@
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using Microsoft.Data.SqlClient;
 using Microsoft.EntityFrameworkCore.Diagnostics;
 using Microsoft.EntityFrameworkCore.Metadata;
@@ -18,7 +19,12 @@
 using Microsoft.EntityFrameworkCore.SqlServer.Metadata.Internal;
 using Microsoft.EntityFrameworkCore.Storage;
 using Microsoft.Extensions.Logging;
+using Microsoft.PowerPlatform.Dataverse.Client;
+using Microsoft.Xrm.Sdk.Messages;
+using Microsoft.Xrm.Sdk.Metadata;
+using Microsoft.Xrm.Sdk.Metadata.Query;
 using RevEng.Common;
+using RevEng.Core;
 
 #nullable enable
 namespace Microsoft.EntityFrameworkCore.SqlServer.Scaffolding.Internal;
@@ -83,6 +89,7 @@ private static readonly Regex PartExtractor
     private byte? _compatibilityLevel;
     private EngineEdition? _engineEdition;
     private string? _version;
+    private DataverseModelFactoryExtension? _dataverse;
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -123,6 +130,11 @@ public DatabaseModel Create(DbConnection connection, DatabaseModelFactoryOptions
         var databaseModel = new DatabaseModel();
 
         var connectionStartedOpen = connection.State == ConnectionState.Open;
+
+        // Avoid multiple login prompts for Dataverse by authenticating once with the Dataverse service client
+        // and reusing the access token for the TDS Endpoint connection as well.
+        DataverseModelFactoryExtension.TryCreate(connection, out _dataverse);
+
         if (!connectionStartedOpen)
         {
             connection.Open();
@@ -699,13 +711,20 @@ FROM [sys].[views] AS [v]
         // This is done separately due to MARS property may be turned off
         GetColumns(connection, tables, tableFilterSql, viewFilter, typeAliases, databaseCollation);
 
-        GetIndexes(connection, tables, tableFilterSql);
+        if (_dataverse != null)
+        {
+            _dataverse.GetDataverseMetadata(tables);
+        }
+        else
+        {
+            GetIndexes(connection, tables, tableFilterSql);
 
-        GetForeignKeys(connection, tables, tableFilterSql);
+            GetForeignKeys(connection, tables, tableFilterSql);
 
-        if (SupportsTriggers())
-        {
-            GetTriggers(connection, tables, tableFilterSql);
+            if (SupportsTriggers())
+            {
+                GetTriggers(connection, tables, tableFilterSql);
+            }
         }
 
         foreach (var table in tables)

src/Core/RevEng.Core.80/RevEng.Core.80.csproj

@@ -22,6 +22,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.11" />   
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer.HierarchyId" Version="8.0.11" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer.NetTopologySuite" Version="8.0.11" />
+    <PackageReference Include="Microsoft.PowerPlatform.Dataverse.Client" Version="1.2.2" />
     <PackageReference Include="Mono.TextTemplating" Version="3.0.0" />
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime" Version="8.0.11" />
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.NetTopologySuite" Version="8.0.11" />

src/Core/RevEng.Core.90/RevEng.Core.90.csproj

@@ -36,6 +36,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer.NetTopologySuite" Version="9.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer.HierarchyId" Version="9.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="9.0.0" />
+    <PackageReference Include="Microsoft.PowerPlatform.Dataverse.Client" Version="1.2.2" />
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime" Version="9.0.1" />
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.NetTopologySuite" Version="9.0.1" />
     <PackageReference Include="Oracle.EntityFrameworkCore" Version="9.23.60" />