Add evaluation client

src/Microsoft.Agents.SDK.sln

@@ -112,6 +112,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WeatherBot", "samples\Seman
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Agents.Telemetry", "libraries\Core\Telemetry\Microsoft.Agents.Telemetry.csproj", "{8E43D789-4F70-4FF2-9B19-DEB6234C5BC0}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EvalClient", "samples\EvalClient\EvalClient.csproj", "{DC0B528D-64A5-4A67-8925-42F459B90CA6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -278,6 +280,10 @@ Global
 		{1889C7A6-B006-4F47-8578-73D95236E94C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1889C7A6-B006-4F47-8578-73D95236E94C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1889C7A6-B006-4F47-8578-73D95236E94C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DC0B528D-64A5-4A67-8925-42F459B90CA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DC0B528D-64A5-4A67-8925-42F459B90CA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DC0B528D-64A5-4A67-8925-42F459B90CA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DC0B528D-64A5-4A67-8925-42F459B90CA6}.Release|Any CPU.Build.0 = Release|Any CPU
 		{8E43D789-4F70-4FF2-9B19-DEB6234C5BC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{8E43D789-4F70-4FF2-9B19-DEB6234C5BC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8E43D789-4F70-4FF2-9B19-DEB6234C5BC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -336,6 +342,7 @@ Global
 		{FAC68927-EC35-464F-BEE0-824B04D39B62} = {674A812C-7287-4883-97F9-697D83750648}
 		{5059670B-CB2F-4D97-8B24-872FD1EC9D99} = {674A812C-7287-4883-97F9-697D83750648}
 		{1889C7A6-B006-4F47-8578-73D95236E94C} = {5059670B-CB2F-4D97-8B24-872FD1EC9D99}
+		{DC0B528D-64A5-4A67-8925-42F459B90CA6} = {674A812C-7287-4883-97F9-697D83750648}
 		{8E43D789-4F70-4FF2-9B19-DEB6234C5BC0} = {7A18F0C9-F8AF-4168-B954-6563BB2C1A90}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution

src/samples/EvalClient/.gitignore

@@ -0,0 +1,2 @@
+.Idea/
+.DS_Store

src/samples/EvalClient/AddTokenHandler.cs

@@ -0,0 +1,84 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System.Net.Http.Headers;
+using System.Runtime.InteropServices;
+using Microsoft.Identity.Client;
+using Microsoft.Identity.Client.Extensions.Msal;
+
+namespace EvalClient
+{
+    /// <summary>
+    /// This sample uses an HttpClientHandler to add an authentication token to the request.
+    /// </summary>
+    /// <param name="settings">Direct To engine connection settings.</param>
+    internal class AddTokenHandler(SampleConnectionSettings settings) : DelegatingHandler(new HttpClientHandler())
+    {
+        private static readonly string _keyChainServiceName = "copilot_studio_client_app";
+        private static readonly string _keyChainAccountName = "copilot_studio_client";
+
+        private async Task<AuthenticationResult> AuthenticateAsync(CancellationToken ct = default!)
+        {
+            ArgumentNullException.ThrowIfNull(settings);
+
+            string[] scopes = ["https://api.powerplatform.com/.default"];
+            //string[] scopes = ["https://api.gov.powerplatform.microsoft.us/CopilotStudio.Copilots.Invoke"];
+
+            IPublicClientApplication app = PublicClientApplicationBuilder.Create(settings.AppClientId)
+                 .WithAuthority(AadAuthorityAudience.AzureAdMyOrg)
+                 .WithTenantId(settings.TenantId)
+                 .WithRedirectUri("http://localhost")
+                 .Build();
+
+            string currentDir = Path.Combine(AppContext.BaseDirectory, "mcs_client_console");
+
+            if (!Directory.Exists(currentDir))
+            {
+                Directory.CreateDirectory(currentDir);
+            }
+
+            StorageCreationPropertiesBuilder storageProperties = new("TokenCache", currentDir);
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+            {
+                storageProperties.WithLinuxUnprotectedFile();
+            }
+
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                storageProperties.WithMacKeyChain(_keyChainServiceName, _keyChainAccountName);
+            }
+
+            MsalCacheHelper tokenCacheHelper = await MsalCacheHelper.CreateAsync(storageProperties.Build());
+            tokenCacheHelper.RegisterCache(app.UserTokenCache);
+
+            IAccount? account = (await app.GetAccountsAsync()).FirstOrDefault();
+
+            AuthenticationResult authResponse;
+            try
+            {
+                authResponse = await app.AcquireTokenSilent(scopes, account).ExecuteAsync(ct);
+            }
+            catch (MsalUiRequiredException)
+            {
+                authResponse = await app.AcquireTokenInteractive(scopes).ExecuteAsync(ct);
+            }
+            return authResponse;
+        }
+
+        /// <summary>
+        /// Handles sending the request and adding the token to the request.
+        /// </summary>
+        /// <param name="request">Request to be sent</param>
+        /// <param name="cancellationToken"></param>
+        /// <returns></returns>
+        protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            if (request.Headers.Authorization is null)
+            {
+                AuthenticationResult authResponse = await AuthenticateAsync(cancellationToken);
+                request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authResponse.AccessToken);
+            }
+            return await base.SendAsync(request, cancellationToken);
+        }
+    }
+}

src/samples/EvalClient/Data/Evaluation Dataset.csv

@@ -0,0 +1,3 @@
+Name,Test Type,Test Utterance,Expected Response,Sources
+Evaluation 1 name,Respone Match,This is the first question asked.,This is the first, expected, optimal response or "ground truth".,https://tenant.sharepoint.com/library/Document%20Library%201/souce_document_1.pdf;https://tenant.sharepoint.com/library/Document%20Library%201/souce_document_2.pdf
+Evaluation 2 name,Respone Match,This is the second question asked.,This is the second, expected, optimal response or "ground truth".https://tenant.sharepoint.com/library/Document%20Library%201/souce_document_3.pdf

src/samples/EvalClient/EvalClient.csproj

@@ -0,0 +1,33 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Azure.AI.OpenAI" Version="2.1.0" />
+      <PackageReference Include="Azure.Identity" Version="1.13.1" />
+      <PackageReference Include="CsvHelper" Version="33.0.1" />
+      <PackageReference Include="Microsoft.Agents.Authentication" Version="0.1.26" />
+      <PackageReference Include="Microsoft.Agents.Client" Version="0.1.26" />
+      <PackageReference Include="Microsoft.Agents.CopilotStudio.Client" Version="0.1.26" />
+      <PackageReference Include="Microsoft.Extensions.AI.OpenAI" Version="9.0.1-preview.1.24570.5" />
+      <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.0" />
+      <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
+      <PackageReference Include="Microsoft.Identity.Client" Version="4.66.2" />
+      <PackageReference Include="Microsoft.Identity.Client.Extensions.Msal" Version="4.66.2" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Update="appsettings.json">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+      <None Update="Data\Evaluation Dataset.csv">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+
+</Project>

src/samples/EvalClient/EvalClient.sln

@@ -0,0 +1,15 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EvalClient", "EvalClient.csproj", "{92313BE6-7239-4F5B-80C9-C50011E4BD75}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{92313BE6-7239-4F5B-80C9-C50011E4BD75}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{92313BE6-7239-4F5B-80C9-C50011E4BD75}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{92313BE6-7239-4F5B-80C9-C50011E4BD75}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{92313BE6-7239-4F5B-80C9-C50011E4BD75}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal

src/samples/EvalClient/EvalDataset.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+namespace EvalClient;
+
+/// <summary>
+/// This class is responsible for deserializing the evaluation dataset
+/// </summary>
+public class EvalDataset
+{
+    public string Name { get; set; }
+    public string TestType { get; set; }
+    public string TestUtterance { get; set; }
+    public string ExpectedResponse { get; set; }
+    public string Sources { get; set; }
+    public string AgentResponse { get; set; }
+    public string AnswerScore { get; set; }
+    public string SourcesScore { get; set; }
+}
+

src/samples/EvalClient/EvalDatasetCsvMap.cs

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+using CsvHelper.Configuration;
+
+namespace EvalClient;
+
+
+/// <summary>
+/// This class is responsible for deserializing the evaluation dataset
+/// </summary>
+public class EvalDatasetCsvMap : ClassMap<EvalDataset>
+{
+    public EvalDatasetCsvMap()
+    {
+        Map(m => m.Name).Name("Name");
+        Map(m => m.TestType).Name("Test Type");
+        Map(m => m.TestUtterance).Name("Test Utterance");
+        Map(m => m.ExpectedResponse).Name("Expected Response");
+        Map(m => m.Sources).Name("Sources");
+    }
+}

src/samples/EvalClient/EvalDatasetResultCsvMap.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+using CsvHelper.Configuration;
+
+namespace EvalClient;
+
+
+/// <summary>
+/// This class is responsible for deserializing the evaluation dataset
+/// </summary>
+public class EvalDatasetResultCsvMap : ClassMap<EvalDataset>
+{
+    public EvalDatasetResultCsvMap()
+    {
+        Map(m => m.Name).Name("Name");
+        Map(m => m.TestType).Name("Test Type");
+        Map(m => m.TestUtterance).Name("Test Utterance");
+        Map(m => m.ExpectedResponse).Name("Expected Response");
+        Map(m => m.Sources).Name("Sources");
+        Map(m => m.AgentResponse).Name("Agent Response");
+        Map(m => m.AnswerScore).Name("Answer Score");
+        Map(m => m.SourcesScore).Name("Sources Score");
+    }
+}