[Backport 1.x] Re-add ApiGenerator

OpenSearch.sln

@@ -61,6 +61,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{3EA11364-051
 		src\_PublishArtifacts.Build.props = src\_PublishArtifacts.Build.props
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApiGenerator", "src\ApiGenerator\ApiGenerator.csproj", "{CA508E92-50AE-4858-BD94-8637E88A8FAC}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.Net.VirtualizedCluster", "src\OpenSearch.Net.VirtualizedCluster\OpenSearch.Net.VirtualizedCluster.csproj", "{CFE97627-8DD3-470B-B7CF-78B62E1D305D}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{29E53C13-34F7-4F0D-8D28-41EF768793E7}"
@@ -124,6 +126,7 @@ Global
 		{81473437-5722-4829-A5CD-125B17CCA238} = {6C4A2627-AF22-4388-9DF7-7A9AEACFD635}
 		{E97CCF40-0BA6-43FE-9F2D-58D454134088} = {3EA11364-0513-44B7-AD6D-A675485E7448}
 		{072BA7DA-7B60-407D-8B6E-95E3186BE70C} = {3EA11364-0513-44B7-AD6D-A675485E7448}
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC} = {3EA11364-0513-44B7-AD6D-A675485E7448}
 		{CFE97627-8DD3-470B-B7CF-78B62E1D305D} = {3EA11364-0513-44B7-AD6D-A675485E7448}
 		{D6997ADC-E933-418E-831C-DE1A78897493} = {29E53C13-34F7-4F0D-8D28-41EF768793E7}
 		{432D5575-2347-4D3C-BF8C-3E38410C46CA} = {29E53C13-34F7-4F0D-8D28-41EF768793E7}
@@ -189,6 +192,10 @@ Global
 		{81473437-5722-4829-A5CD-125B17CCA238}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{81473437-5722-4829-A5CD-125B17CCA238}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{81473437-5722-4829-A5CD-125B17CCA238}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CA508E92-50AE-4858-BD94-8637E88A8FAC}.Release|Any CPU.Build.0 = Release|Any CPU
 		{CFE97627-8DD3-470B-B7CF-78B62E1D305D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{CFE97627-8DD3-470B-B7CF-78B62E1D305D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CFE97627-8DD3-470B-B7CF-78B62E1D305D}.Release|Any CPU.ActiveCfg = Release|Any CPU

build/scripts/Commandline.fs

@@ -62,6 +62,8 @@ Targets:
 * benchmark [non-interactive] [url] [username] [password] 
   - Runs a benchmark from Tests.Benchmarking and indexes the results to [url] when provided.
     If non-interactive runs all benchmarks without prompting
+* codegen
+  - runs the code generator interactively
 
 NOTE: both the `test` and `integrate` targets can be suffixed with `-all` to force the tests against all suported TFM's
 
@@ -193,6 +195,7 @@ Execution hints can be provided anywhere on the command line
         | ["clean"]
         | ["benchmark"]
         | ["profile"] -> parsed
+        | "codegen" :: tail -> { parsed with RemainingArguments = tail }
         | "rest-spec-tests" :: tail -> { parsed with RemainingArguments = tail }
 
         | ["release"; version] -> { parsed with CommandArguments = SetVersion { Version = version; OutputLocation = None }; }

build/scripts/ReposTooling.fs

@@ -52,6 +52,14 @@ module ReposTooling =
 
         Shell.deleteDir tempDir
 
+    let GenerateApi args =
+        //TODO allow branch name to be passed for CI
+        let folder = Path.getDirectory (Paths.ProjFile "ApiGenerator")
+        let timeout = TimeSpan.FromMinutes(120.)
+        // Building to make sure XML docs files are there, faster then relying on the ApiGenerator to emit these
+        // from a compilation unit
+        Tooling.DotNet.ExecInWithTimeout folder (["run"; "-c"; " Release"; "--" ] @ args) timeout  |> ignore
+
     let RestSpecTests args =
         let folder = Path.getDirectory (Paths.TestProjFile "Tests.YamlRunner")
         let timeout = TimeSpan.FromMinutes(120.)

build/scripts/Targets.fs

@@ -133,6 +133,9 @@ module Main =
             command "cluster" [ "restore"; "full-build" ] <| fun _ ->
                 ReposTooling.LaunchCluster parsed
 
+            command "codegen" [ ] <| fun _ ->
+                ReposTooling.GenerateApi parsed.RemainingArguments
+
             command "rest-spec-tests" [ ] <| fun _ ->
                 ReposTooling.RestSpecTests parsed.RemainingArguments
 

src/ApiGenerator/ApiGenerator.csproj

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <IsPackable>false</IsPackable>
+
+    <NoWarn>CS1591;NU1701</NoWarn>
+    <PreserveCompilationContext>true</PreserveCompilationContext>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="CSharpier.Core" Version="0.26.3" />
+    <PackageReference Include="Glob" Version="1.1.9" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="NSwag.Core.Yaml" Version="13.20.0" />
+    <PackageReference Include="SemanticVersioning" Version="2.0.2" />
+    <PackageReference Include="ShellProgressBar" Version="5.2.0" />
+    <PackageReference Include="Spectre.Console" Version="0.48.0" />
+    <PackageReference Include="System.CommandLine.DragonFruit" Version="0.3.0-alpha.20371.2" />
+    <PackageReference Include="RazorLight" Version="2.3.1" />
+    <PackageReference Include="System.Text.Encodings.Web" Version="7.0.0" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Views\**\*.cshtml" />
+  </ItemGroup>
+</Project>

src/ApiGenerator/CodeTemplatePage.cs

@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+/*
+* Modifications Copyright OpenSearch Contributors. See
+* GitHub history for details.
+*
+*  Licensed to Elasticsearch B.V. under one or more contributor
+*  license agreements. See the NOTICE file distributed with
+*  this work for additional information regarding copyright
+*  ownership. Elasticsearch B.V. licenses this file to you under
+*  the Apache License, Version 2.0 (the "License"); you may
+*  not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing,
+*  software distributed under the License is distributed on an
+*  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+*  KIND, either express or implied.  See the License for the
+*  specific language governing permissions and limitations
+*  under the License.
+*/
+
+using System.Threading.Tasks;
+using RazorLight;
+
+namespace ApiGenerator;
+
+public abstract class CodeTemplatePage<TModel> : TemplatePage<TModel>
+{
+	protected new Task IncludeAsync(string key, object model = null)
+		=> base.IncludeAsync(key.Replace('/', '.'), model);
+
+	protected async Task IncludeLegacyGeneratorNotice() => await IncludeAsync("GeneratorNotice", true);
+
+	protected async Task IncludeGeneratorNotice() => await IncludeAsync("GeneratorNotice", false);
+}

src/ApiGenerator/Configuration/CodeConfiguration.cs

@@ -0,0 +1,228 @@
+/* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+/*
+* Modifications Copyright OpenSearch Contributors. See
+* GitHub history for details.
+*
+*  Licensed to Elasticsearch B.V. under one or more contributor
+*  license agreements. See the NOTICE file distributed with
+*  this work for additional information regarding copyright
+*  ownership. Elasticsearch B.V. licenses this file to you under
+*  the Apache License, Version 2.0 (the "License"); you may
+*  not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing,
+*  software distributed under the License is distributed on an
+*  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+*  KIND, either express or implied.  See the License for the
+*  specific language governing permissions and limitations
+*  under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using GlobExpressions;
+
+namespace ApiGenerator.Configuration
+{
+    public static class CodeConfiguration
+	{
+		private static readonly Glob[] OperationsToInclude =
+		{
+			new("{create,delete}_pit"),
+			new("{delete,get}_all_pits"),
+
+			new("cluster.*"),
+			new("dangling_indices.*"),
+
+			new("indices.{delete,exists,get,put}_index_template"),
+
+			new("ingest.*"),
+            new("nodes.*"),
+			new("snapshot.*"),
+			new("tasks.*")
+		};
+
+		public static bool IncludeOperation(string name) => OperationsToInclude.Any(g => g.IsMatch(name));
+
+		/// <summary>
+        /// Map API default names for API's we are only supporting on the low level client first
+        /// </summary>
+        private static readonly Dictionary<string, string> LowLevelApiNameMapping = new()
+		{
+        };
+
+        /// <summary>
+        /// Scan all OSC source code files for Requests and look for the [MapsApi(filename)] attribute.
+        /// The class name minus Request is used as the canonical .NET name for the API.
+        /// </summary>
+        public static readonly Dictionary<string, string> HighLevelApiNameMapping =
+            (from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*.cs", SearchOption.AllDirectories)
+                let contents = File.ReadAllText(f.FullName)
+                let c = Regex.Replace(contents, @"^.+\[MapsApi\(""([^ \r\n]+)""\)\].*$", "$1", RegexOptions.Singleline)
+                where !c.Contains(" ") //filter results that did not match
+                select new { Value = f.Name.Replace("Request", ""), Key = c.Replace(".json", "") })
+            .DistinctBy(v => v.Key)
+            .ToDictionary(k => k.Key, v => v.Value.Replace(".cs", ""));
+
+        public static readonly HashSet<string> EnableHighLevelCodeGen = new HashSet<string>();
+
+        public static bool IsNewHighLevelApi(string apiFileName) =>
+            // no requests with [MapsApi("filename.json")] found
+            !HighLevelApiNameMapping.ContainsKey(apiFileName.Replace(".json", ""));
+
+        public static bool IgnoreHighLevelApi(string apiFileName)
+        {
+			//always generate already mapped requests
+
+            if (HighLevelApiNameMapping.ContainsKey(apiFileName.Replace(".json", ""))) return false;
+
+            return !EnableHighLevelCodeGen.Contains(apiFileName);
+        }
+
+        private static Dictionary<string, string> _apiNameMapping;
+
+        public static Dictionary<string, string> ApiNameMapping
+        {
+            get
+            {
+                if (_apiNameMapping != null) return _apiNameMapping;
+                lock (LowLevelApiNameMapping)
+                {
+                    if (_apiNameMapping == null)
+                    {
+                        var mapping = new Dictionary<string, string>(HighLevelApiNameMapping);
+                        foreach (var (k, v) in LowLevelApiNameMapping)
+                            mapping[k] = v;
+                        _apiNameMapping = mapping;
+                    }
+                    return _apiNameMapping;
+                }
+            }
+        }
+
+        private static readonly string ResponseBuilderAttributeRegex = @"^.+\[ResponseBuilderWithGeneric\(""([^ \r\n]+)""\)\].*$";
+        /// <summary>
+        /// Scan all OSC source code files for Requests and look for the [MapsApi(filename)] attribute.
+        /// The class name minus Request is used as the canonical .NET name for the API.
+        /// </summary>
+        public static readonly Dictionary<string, string> ResponseBuilderInClientCalls =
+            (from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*.cs", SearchOption.AllDirectories)
+                from l in File.ReadLines(f.FullName)
+                where Regex.IsMatch(l, ResponseBuilderAttributeRegex)
+                let c = Regex.Replace(l, @"^.+\[ResponseBuilderWithGeneric\(""([^ \r\n]+)""\)\].*$", "$1", RegexOptions.Singleline)
+                select new { Key = f.Name.Replace(".cs", ""), Value = c })
+            .DistinctBy(v => v.Key)
+            .ToDictionary(k => k.Key, v => v.Value);
+
+        public static readonly Dictionary<string, string> DescriptorGenericsLookup =
+            (from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
+                let name = Path.GetFileNameWithoutExtension(f.Name).Replace("Request", "")
+                let contents = File.ReadAllText(f.FullName)
+                let c = Regex.Replace(contents, $@"^.+class ({name}Descriptor(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
+                let key = $"{name}Descriptor"
+                select new { Key = key, Value = Regex.Replace(c, @"^.*?(?:(\<.+>).*?)?$", "$1") })
+            .DistinctBy(v => v.Key)
+            .OrderBy(v => v.Key)
+            .ToDictionary(k => k.Key, v => v.Value);
+
+        /// <summary> Scan all OSC files for request interfaces and note any generics declared on them </summary>
+        private static readonly List<Tuple<string, string>> AllKnownRequestInterfaces = (
+            // find all files in OSC ending with Request.cs
+            from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
+            from l in File.ReadLines(f.FullName)
+            // attempt to locate all Request interfaces lines
+            where Regex.IsMatch(l, @"^.+interface [^ \r\n]+Request")
+            //grab the interface name including any generics declared on it
+            let c = Regex.Replace(l, @"^.+interface ([^ \r\n]+Request(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
+            where c.StartsWith("I") && c.Contains("Request")
+            let request = Regex.Replace(c, "<.*$", "")
+            let generics = Regex.Replace(c, @"^.*?(?:(\<.+>).*?)?$", "$1")
+            select Tuple.Create(request,  generics)
+            )
+            .OrderBy(v=>v.Item1)
+            .ToList();
+
+        public static readonly HashSet<string> GenericOnlyInterfaces = new HashSet<string>(AllKnownRequestInterfaces
+            .GroupBy(v => v.Item1)
+            .Where(g => g.All(v => !string.IsNullOrEmpty(v.Item2)))
+            .Select(g => g.Key)
+            .ToList());
+
+        public static readonly HashSet<string> DocumentRequests = new HashSet<string>((
+            // find all files in OSC ending with Request.cs
+            from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
+            from l in File.ReadLines(f.FullName)
+            // attempt to locate all Request interfaces lines
+            where Regex.IsMatch(l, @"^.+interface [^ \r\n]+Request")
+            where l.Contains("IDocumentRequest")
+            let c = Regex.Replace(l, @"^.+interface ([^ \r\n]+Request(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
+            //grab the interface name including any generics declared on it
+            let request = Regex.Replace(c, "<.*$", "")
+            select request
+            )
+            .ToList());
+
+        public static readonly Dictionary<string, string> DescriptorConstructors = (
+            // find all files in OSC ending with Request.cs
+            from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Request.cs", SearchOption.AllDirectories)
+            let descriptor = Path.GetFileNameWithoutExtension(f.Name).Replace("Request", "Descriptor")
+            let re = $@"^.+public {descriptor}\(([^\r\n\)]+?)\).*$"
+            from l in File.ReadLines(f.FullName)
+            where Regex.IsMatch(l, re)
+            let args = Regex.Replace(l, re, "$1", RegexOptions.Singleline)
+            where !string.IsNullOrWhiteSpace(args) && !args.Contains(": base")
+            select (Descriptor: descriptor, Args: args)
+            )
+            .ToDictionary(r => r.Descriptor, r => r.Args);
+
+        public static readonly Dictionary<string, string> RequestInterfaceGenericsLookup =
+            AllKnownRequestInterfaces
+            .GroupBy(v=>v.Item1)
+            .Select(g=>g.Last())
+            .ToDictionary(k => k.Item1, v => v.Item2);
+
+        /// <summary>
+        /// Some API's reuse response this is a hardcoded map of these cases
+        /// </summary>
+        private static Dictionary<string, (string, string)> ResponseReroute = new Dictionary<string, (string, string)>
+        {
+            {"UpdateByQueryRethrottleResponse", ("ListTasksResponse", "")},
+            {"DeleteByQueryRethrottleResponse", ("ListTasksResponse", "")},
+            {"MultiSearchTemplateResponse", ("MultiSearchResponse", "")},
+            {"ScrollResponse", ("SearchResponse", "<TDocument>")},
+            {"SearchTemplateResponse", ("SearchResponse", "<TDocument>")},
+
+        };
+
+
+        /// <summary> Create a dictionary lookup of all responses and their generics </summary>
+        public static readonly SortedDictionary<string, (string, string)> ResponseLookup = new SortedDictionary<string, (string, string)>(
+        (
+            // find all files in OSC ending with Request.cs
+            from f in new DirectoryInfo(GeneratorLocations.OpenSearchClientFolder).GetFiles("*Response.cs", SearchOption.AllDirectories)
+            from l in File.ReadLines(f.FullName)
+            // attempt to locate all Response class lines
+            where Regex.IsMatch(l, @"^.+public class [^ \r\n]+Response")
+            //grab the response name including any generics declared on it
+            let c = Regex.Replace(l, @"^.+public class ([^ \r\n]+Response(?:<[^>\r\n]+>)?[^ \r\n]*).*$", "$1", RegexOptions.Singleline)
+            where c.Contains("Response")
+            let response = Regex.Replace(c, "<.*$", "")
+            let generics = Regex.Replace(c, @"^.*?(?:(\<.+>).*?)?$", "$1")
+            select (response,  (response, generics))
+        )
+            .Concat(ResponseReroute.Select(kv=>(kv.Key, (kv.Value.Item1, kv.Value.Item2))))
+            .ToDictionary(t=>t.Item1, t=>t.Item2));
+
+    }
+}