diff --git a/.github/actions/run-released-opensearch/action.yml b/.github/actions/run-released-opensearch/action.yml new file mode 100644 index 0000000000..c87f63e341 --- /dev/null +++ b/.github/actions/run-released-opensearch/action.yml @@ -0,0 +1,52 @@ +name: Run OpenSearch +description: Runs a released version of OpenSearch +inputs: + version: + description: The version of OpenSearch to run + required: true + secured: + description: Whether to enable the security plugin + required: true +outputs: + opensearch_url: + description: The URL where the OpenSearch node is accessible + value: ${{ steps.opensearch.outputs.opensearch_url }} + admin_password: + description: The initial admin password + value: ${{ steps.opensearch.outputs.admin_password }} +runs: + using: composite + steps: + - name: Restore cached OpenSearch distro + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: opensearch-* + key: opensearch-${{ inputs.version }}-${{ runner.os }} + + - name: Download OpenSearch + if: steps.cache-restore.outputs.cache-hit != 'true' + shell: bash -eo pipefail {0} + run: | + if [[ "$RUNNER_OS" != "Windows" ]]; then + curl -sSLO https://artifacts.opensearch.org/releases/bundle/opensearch/${{ inputs.version }}/opensearch-${{ inputs.version }}-linux-x64.tar.gz + tar -xzf opensearch-*.tar.gz + rm -f opensearch-*.tar.gz + else + curl -sSLO https://artifacts.opensearch.org/releases/bundle/opensearch/${{ inputs.version }}/opensearch-${{ inputs.version }}-windows-x64.zip + unzip opensearch-*.zip + rm -f opensearch-*.zip + fi + + - name: Save cached OpenSearch distro + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: opensearch-* + key: opensearch-${{ inputs.version }}-${{ runner.os }} + + - name: Start OpenSearch + id: opensearch + uses: ./client/.github/actions/start-opensearch + with: + secured: ${{ inputs.secured }} diff --git a/.github/actions/start-opensearch/action.yml b/.github/actions/start-opensearch/action.yml new file mode 100644 index 0000000000..8e4eb154e5 --- /dev/null +++ b/.github/actions/start-opensearch/action.yml @@ -0,0 +1,98 @@ +name: Start OpenSearch +description: Configures and starts an OpenSearch daemon +inputs: + secured: + description: Whether to enable the security plugin + default: 'false' +outputs: + opensearch_url: + description: The URL where the OpenSearch node is accessible + value: ${{ steps.opensearch.outputs.url }} + admin_password: + description: The initial admin password + value: ${{ steps.opensearch.outputs.password }} +runs: + using: composite + steps: + - name: Install Java + uses: actions/setup-java@v3 + with: + distribution: zulu + java-version: 11 + + - name: Start OpenSearch + id: opensearch + shell: bash -eo pipefail {0} + run: | + if [[ "$RUNNER_OS" == "macOS" ]]; then + brew install -q coreutils + fi + OPENSEARCH_HOME=$(realpath ./opensearch-[1-9]*) + CONFIG_DIR=$OPENSEARCH_HOME/config + CONFIG_FILE=$CONFIG_DIR/opensearch.yml + SECURITY_DIR=$OPENSEARCH_HOME/plugins/opensearch-security + OPENSEARCH_JAVA_OPTS="-Djava.net.preferIPv4Stack=true" + + URL="http://localhost:9200" + cp ./client/.ci/opensearch/opensearch.yml $CONFIG_FILE + + bash ./client/.ci/generate-certs.sh + + export OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin + + if [[ -d "$SECURITY_DIR" ]]; then + if [[ "$SECURED" == "true" ]]; then + SECURITY_VERSION=$(cat $SECURITY_DIR/plugin-descriptor.properties | grep '^version=' | cut -d'=' -f 2) + SECURITY_VERSION_COMPONENTS=(${SECURITY_VERSION//./ }) + SECURITY_MAJOR="${SECURITY_VERSION_COMPONENTS[0]}" + SECURITY_MINOR="${SECURITY_VERSION_COMPONENTS[1]}" + + if (( $SECURITY_MAJOR > 2 || ( $SECURITY_MAJOR == 2 && $SECURITY_MINOR >= 12 ) )); then + export OPENSEARCH_INITIAL_ADMIN_PASSWORD=$(LC_ALL=C tr -dc A-Za-z0-9 > $CONFIG_FILE + fi + fi + + if [[ "$RUNNER_OS" == "macOS" ]]; then + sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $CONFIG_FILE + fi + + { + echo "url=$URL" + echo "password=$OPENSEARCH_INITIAL_ADMIN_PASSWORD" + } | tee -a $GITHUB_OUTPUT + + if [[ "$RUNNER_OS" == "Linux" ]]; then + sudo swapoff -a + sudo sysctl -w vm.swappiness=1 + sudo sysctl -w fs.file-max=262144 + sudo sysctl -w vm.max_map_count=262144 + sudo prlimit --pid $$ --memlock=unlimited:unlimited + fi + + if [[ "$RUNNER_OS" != "Windows" ]]; then + $OPENSEARCH_HOME/bin/opensearch & + else + $OPENSEARCH_HOME/bin/opensearch.bat -d & + fi + + for attempt in {1..20}; do + sleep 5 + if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} $URL; then + echo '=====> ready' + exit 0 + fi + echo '=====> waiting...' + done + exit 1 + env: + SECURED: ${{ inputs.secured }} + RUNNER_OS: ${{ runner.os }} diff --git a/.github/workflows/integration-yaml-tests.yml b/.github/workflows/integration-yaml-tests.yml new file mode 100644 index 0000000000..b8799a62fe --- /dev/null +++ b/.github/workflows/integration-yaml-tests.yml @@ -0,0 +1,152 @@ +name: YAML Tests + +on: + push: + branches-ignore: + - 'dependabot/**' + pull_request: {} + +jobs: + test-yaml: + name: YAML Tests (Released OpenSearch) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: + - 2.11.1 + - 2.10.0 + - 2.8.0 + - 2.6.0 + - 2.4.1 + - 2.2.1 + - 2.0.1 + - 1.3.14 + - 1.2.4 + - 1.1.0 + steps: + - name: Checkout Client + uses: actions/checkout@v3 + with: + path: client + + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: | + 5.0.x + 6.0.x + + - uses: actions/cache@v3 + with: + path: ~/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.?sproj') }} + restore-keys: | + ${{ runner.os }}-nuget- + + - name: Run OpenSearch + id: opensearch + uses: ./client/.github/actions/run-released-opensearch + with: + version: ${{ matrix.version }} + secured: true + + - name: Run YAML tests + working-directory: client + run: | + dotnet run \ + --project ./tests/Tests.YamlRunner/Tests.YamlRunner.fsproj \ + -- \ + --endpoint $OPENSEARCH_URL \ + --auth-cert ./.ci/certs/kirk.p12 \ + --auth-cert-pass kirk \ + --junit-output-file ./test-results.xml + env: + OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }} + + - name: Save OpenSearch logs + if: failure() + uses: actions/upload-artifact@v3 + with: + name: opensearch-logs-${{ matrix.version }} + path: | + opensearch-*/logs/* + + - name: Upload test report + if: failure() + uses: actions/upload-artifact@v3 + with: + name: report-yaml-${{ matrix.version }} + path: client/test-results.xml + + test-yaml-unreleased: + name: YAML Tests (Unreleased OpenSearch) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + opensearch_ref: ['1.x', '2.x', 'main'] + steps: + - name: Checkout Client + uses: actions/checkout@v3 + with: + path: client + + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: | + 5.0.x + 6.0.x + + - uses: actions/cache@v3 + with: + path: ~/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.?sproj') }} + restore-keys: | + ${{ runner.os }}-nuget- + + - name: Restore or Build OpenSearch + id: opensearch_build + uses: ./client/.github/actions/build-opensearch + with: + ref: ${{ matrix.opensearch_ref }} + security_plugin: true + + - name: Unpack OpenSearch + run: | + tar -xzf ${{ steps.opensearch_build.outputs.distribution }} \ + && ./opensearch-*/bin/opensearch-plugin install --batch file://$(realpath ./opensearch-security/build/distributions/opensearch-security-*-SNAPSHOT.zip) + + - name: Start OpenSearch + id: opensearch + uses: ./client/.github/actions/start-opensearch + with: + secured: true + + - name: Run YAML tests + working-directory: client + run: | + dotnet run \ + --project ./tests/Tests.YamlRunner/Tests.YamlRunner.fsproj \ + -- \ + --endpoint $OPENSEARCH_URL \ + --auth-cert ./.ci/certs/kirk.p12 \ + --auth-cert-pass kirk \ + --junit-output-file ./test-results.xml + env: + OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }} + ADMIN_PASS: ${{ steps.opensearch.outputs.admin_password }} + + - name: Save OpenSearch logs + if: failure() + uses: actions/upload-artifact@v3 + with: + name: opensearch-logs-${{ matrix.opensearch_ref }} + path: | + opensearch-*/logs/* + + - name: Upload test report + if: failure() + uses: actions/upload-artifact@v3 + with: + name: report-yaml-unreleased-${{ matrix.opensearch_ref }} + path: client/test-results.xml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 06b8e4c078..068000e68c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -17,17 +17,14 @@ jobs: fail-fast: false matrix: version: - - 2.9.0 + - 2.11.1 + - 2.10.0 - 2.8.0 - - 2.7.0 - 2.6.0 - - 2.5.0 - 2.4.1 - - 2.3.0 - 2.2.1 - - 2.1.0 - 2.0.1 - - 1.3.11 + - 1.3.14 - 1.2.4 - 1.1.0 @@ -62,7 +59,6 @@ jobs: path: client/build/output/* integration-opensearch-unreleased: - if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268) name: Integration OpenSearch Unreleased runs-on: ubuntu-latest strategy: diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs index bba2455f5f..f0e3769c96 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs @@ -34,6 +34,7 @@ using System.Security.Cryptography; using System.Text; using OpenSearch.OpenSearch.Managed; +using OpenSearch.OpenSearch.Managed.Configuration; using OpenSearch.Stack.ArtifactsApi; namespace OpenSearch.OpenSearch.Ephemeral @@ -59,6 +60,13 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo protected EphemeralClusterComposer Composer { get; } + protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) + { + base.ModifyNodeConfiguration(nodeConfiguration, port); + + if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true"); + } + public virtual ICollection NodesUris(string hostName = null) { hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any() diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs index 40b200f28a..74441951ef 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs @@ -172,22 +172,35 @@ protected static void WriteFileIfNotExist(string fileLocation, string contents) protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, string binary, string description, params string[] arguments) => - ExecuteBinaryInternal(config, writer, binary, description, arguments); + ExecuteBinaryInternal(config, writer, binary, description, null, arguments); + + protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, IDictionary environmentVariables, + params string[] arguments) => + ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments); private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer, - string binary, string description, params string[] arguments) + string binary, string description, IDictionary environmentVariables, params string[] arguments) { var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}"; writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}"); + var environment = new Dictionary + { + {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, + {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome} + }; + + if (environmentVariables != null) + { + foreach (var kvp in environmentVariables) + environment[kvp.Key] = kvp.Value; + } + var timeout = TimeSpan.FromSeconds(420); var processStartArguments = new StartArguments(binary, arguments) { - Environment = new Dictionary - { - {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, - {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome}, - } + Environment = environment }; var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter()); diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs index b938890102..9540c07edb 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs @@ -26,10 +26,12 @@ * under the License. */ +using System.Collections.Generic; using System.IO; using System.Linq; using OpenSearch.OpenSearch.Managed.ConsoleWriters; -using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; +using SemanticVersioning; namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks { @@ -38,30 +40,39 @@ public class InitialConfiguration : ClusterComposeTask public override void Run(IEphemeralCluster cluster) { var fs = cluster.FileSystem; - var configFile = Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml"); - if (File.Exists(configFile) && File.ReadLines(configFile).Any(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#"))) - { - cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} opensearch.yml already exists, skipping initial configuration"); + var installConfigDir = Path.Combine(fs.OpenSearchHome, "config"); + var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml"); + var pluginSecurity = Path.Combine(fs.OpenSearchHome, "plugins/opensearch-security"); + + if (!Directory.Exists(pluginSecurity)) return; - } - var securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh"; - var securityInstallDemoConfig = Path.Combine(fs.OpenSearchHome, securityInstallDemoConfigSubPath); + var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0); + + const string securityInstallDemoConfigSubPath = "tools/install_demo_configuration.sh"; + var securityInstallDemoConfig = Path.Combine(pluginSecurity, securityInstallDemoConfigSubPath); cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]"); + if (File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"))) return; + + var env = new Dictionary(); + var args = new List { securityInstallDemoConfig, "-y", "-i" }; + + if (isNewDemoScript) + { + env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin"); + args.Add("-t"); + } + ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, "/bin/bash", "install security plugin demo configuration", - securityInstallDemoConfig, - "-y", "-i", "-s"); - - if (cluster.ClusterConfiguration.EnableSsl) return; - - File.AppendAllText(configFile, "plugins.security.disabled: true"); + env, + args.ToArray()); } } } diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs index c9509b7d03..3225476353 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs @@ -27,6 +27,7 @@ */ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; @@ -93,12 +94,21 @@ public override void Run(IEphemeralCluster cluste cluster.Writer?.WriteDiagnostic( $"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}"); - if (!Directory.Exists(fs.ConfigPath)) Directory.CreateDirectory(fs.ConfigPath); + var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config"); + + if (!Directory.Exists(homeConfigPath)) Directory.CreateDirectory(homeConfigPath); + + var env = new Dictionary + { + { fs.ConfigEnvironmentVariableName, homeConfigPath } + }; + ExecuteBinary( cluster.ClusterConfiguration, cluster.Writer, fs.PluginBinary, $"install opensearch plugin: {plugin.SubProductName}", + env, "install", "--batch", GetPluginLocation(plugin, v)); CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin); diff --git a/guides/document-lifecycle.md b/guides/document-lifecycle.md index e0746f7e7c..67cb37588d 100644 --- a/guides/document-lifecycle.md +++ b/guides/document-lifecycle.md @@ -7,7 +7,7 @@ Assuming you have OpenSearch running locally on port 9200, you can create a clie var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin") + .BasicAuthentication("admin", ) .DisableDirectStreaming(); var client = new OpenSearchClient(config); diff --git a/guides/index-template.md b/guides/index-template.md index 495d873b39..81049f128b 100644 --- a/guides/index-template.md +++ b/guides/index-template.md @@ -12,7 +12,7 @@ using OpenSearch.Net; var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin"); + .BasicAuthentication("admin", ); var client = new OpenSearchClient(config);; ``` diff --git a/guides/search.md b/guides/search.md index ed8d022a97..e7f00e4bec 100644 --- a/guides/search.md +++ b/guides/search.md @@ -12,7 +12,7 @@ var node = new Uri("https://localhost:9200"); var config = new ConnectionSettings(node) .ThrowExceptions() .ServerCertificateValidationCallback(CertificateValidations.AllowAll) - .BasicAuthentication("admin", "admin"); + .BasicAuthentication("admin", ); var client = new OpenSearchClient(config); class Movie diff --git a/tests/Tests.YamlRunner/Models.fs b/tests/Tests.YamlRunner/Models.fs index 761cdf0269..5d4f372df5 100644 --- a/tests/Tests.YamlRunner/Models.fs +++ b/tests/Tests.YamlRunner/Models.fs @@ -61,9 +61,9 @@ let (|IsDoCatch|_|) (s:string) = | "conflict" -> Some Conflict | "unavailable" -> Some Unavailable | "param" -> Some UnknownParameter - | "request" -> Some OtherBadResponse - | s -> Some <| CatchRegex (s.Trim('/')) - + | "request" -> Some OtherBadResponse + | s -> Some <| CatchRegex (Regex.Replace(s.Trim('/'), @"(?