build.gradle

/*
 * SPDX-License-Identifier: Apache-2.0
 *
 * The OpenSearch Contributors require contributions made to
 * this file be licensed under the Apache-2.0 license or a
 * compatible open source license.
 *
 * Modifications Copyright OpenSearch Contributors. See
 * GitHub history for details.
 */

/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

buildscript {
    dependencies {
        classpath group: 'com.google.guava', name: 'guava', version: '30.1-jre'
        classpath 'org.ajoberstar:gradle-git:0.2.3'
    }
}

plugins {
    id 'java'
    id 'application'
    id 'maven-publish'
    id 'com.google.protobuf' version '0.8.8'
    id 'jacoco'
    id 'idea'
    id 'com.github.spotbugs' version '4.6.0'
    id "de.undercouch.download" version "4.0.4"
    id 'com.adarshr.test-logger' version '2.1.0'
    id 'org.gradle.test-retry' version '1.3.1'
    id 'com.diffplug.spotless' version '5.11.0'
}

application {
    mainClassName = 'org.opensearch.performanceanalyzer.PerformanceAnalyzerApp'
    applicationDefaultJvmArgs = ['-Xms64M',
                                 '-Xmx64M',
                                 '-XX:+UseSerialGC',
                                 '-XX:CICompilerCount=1',
                                 '-XX:-TieredCompilation',
                                 '-XX:InitialCodeCacheSize=4096',
                                 '-XX:MaxRAM=400m']
}

// Include pa_config folder in the distribution.
applicationDistribution.from(".") {
    include 'pa_config/*'
    include 'pa_bin/*'
}

distributions {
    main {
        contents {
            eachFile {
                it.path = it.path.replace("-$version/", '/')
            }
        }
    }
}

ext {
    opensearch_version = System.getProperty("opensearch.version", "1.3.0-SNAPSHOT")
    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
    gitPaBranch = 'main'
    gitPaRepo = "https://github.com/opensearch-project/performance-analyzer.git"
    runGauntletTests = "true" == System.getProperty("run.gauntlet.tests", "false")
}

spotless {
    java {
        licenseHeaderFile(file('license-header'))
        googleJavaFormat().aosp()
        importOrder()
        removeUnusedImports()
        trimTrailingWhitespace()
        endWithNewline()

        // add support for spotless:off and spotless:on tags to exclude sections of code
        toggleOffOn()
    }
}

tasks.withType(Checkstyle) {
    reports {
        xml.enabled false
        html.enabled true
        html.stylesheet resources.text.fromFile('configs/xsl/severity-sorted.xsl')
    }
}

testlogger {
    theme 'standard'
    showExceptions true
    showStackTraces false
    showFullStackTraces false
    showCauses true
    slowThreshold 4000
    showSummary true
    showSimpleNames false
    showPassed true
    showSkipped true
    showFailed true
    showStandardStreams false
    showPassedStandardStreams true
    showSkippedStandardStreams true
    showFailedStandardStreams true
    logLevel 'lifecycle'
}

spotbugsMain {
    excludeFilter = file("checkstyle/findbugs-exclude.xml")
    effort = 'max'
    ignoreFailures = false

    reports {
        xml.enabled = false
        html.enabled = true
    }
}

spotbugsTest {
    ignoreFailures = true
}

check {
    dependsOn spotbugsMain
    //dependsOn spotbugsTest
}

jacoco {
    toolVersion = "0.8.5"
}

jacocoTestReport {
    reports {
        xml.enabled true
        html.enabled true
        csv.enabled false
    }

    afterEvaluate {
        classDirectories.from = files(classDirectories.files.collect {
            fileTree(dir: it,
                    include: [
                            '**/org/opensearch/performanceanalyzer/**'
                    ],
                    exclude: [
                            '**/org/opensearch/performanceanalyzer/grpc/**',
                            '**/org/opensearch/performanceanalyzer/rca/framework/api/metrics/**',
                            '**/org/opensearch/performanceanalyzer/rca/net/requests**'
                    ])
        })
    }
}

jacocoTestCoverageVerification {
    afterEvaluate {
        classDirectories.from = files(classDirectories.files.collect {
            fileTree(dir: it,
                    include: [
                            '**/org/opensearch/performanceanalyzer/rca/**',
                            '**/org/opensearch/performanceanalyzer/reader/**'
                    ],
                    exclude: [
                            '**/org/opensearch/performanceanalyzer/grpc/**'])
        })
    }
    if (runGauntletTests) {
        violationRules {
            rule {
                limit {
                    minimum = 0.4
                }
            }
        }
    } 
    else {
        violationRules {
            rule {
                limit {
                    minimum = 0.7
                }
            }
        }    
    }
}

// to run coverage verification during the build (and fail when appropriate)
check.dependsOn jacocoTestCoverageVerification

version = opensearch_version - '-SNAPSHOT' + '.0'
if (isSnapshot) {
    version += "-SNAPSHOT"
}

distZip {
    archiveName "performance-analyzer-rca-${version}.zip"
}

publishing {
    publications {
        maven(MavenPublication) {
            groupId = 'org.opensearch'
            artifactId = 'performanceanalyzer-rca'
            from components.java
        }
    }
}

test {
    if (!runGauntletTests) {
        filter {
            excludeTestsMatching 'org.opensearch.performanceanalyzer.rca.integTests.*'
            // TODO: Fix this test as it causes OutOfMemoryError: Java heap space error and runs forever
            excludeTestsMatching 'org.opensearch.performanceanalyzer.reader.OSMetricsSnapshotTests'
        }
    }
    else {
        filter {
            includeTestsMatching 'org.opensearch.performanceanalyzer.rca.integTests.*'
            // Ignore failing tests (known issues with the test)
            excludeTestsMatching 'org.opensearch.performanceanalyzer.rca.integTests.tests.jvmsizing.HeapSizeIncreaseMissingMetricsTest'
            excludeTestsMatching 'org.opensearch.performanceanalyzer.rca.integTests.tests.consolidate_tuning.JvmFlipFlopITest'
        }
    }
    enabled = true
    retry {
        maxRetries = 2
        maxFailures = 20
        filter {
            // filter by qualified class name (* matches zero or more of any character)
            // ignoring retries on all the integ test as the retries on them fails
            // retry plugin is unable to retry the following test methods as the method name differs for these tests.
            // java.lang.Class#testDataNodeMissingGcMetrics
            excludeClasses.add("*integTests*")
        }
    }
}

task rcaTest(type: Test) {
    useJUnit {
        includeCategories 'org.opensearch.performanceanalyzer.rca.GradleTaskForRca'
    }
    testLogging.showStandardStreams = true
}

task rcaIt(type: Test) {
    dependsOn spotbugsMain
    useJUnit {
        includeCategories 'org.opensearch.performanceanalyzer.rca.integTests.framework.RcaItMarker'
    }
    //testLogging.showStandardStreams = true
}

sourceCompatibility = 1.8
targetCompatibility = 1.8

compileJava {
    dependsOn spotlessApply
    JavaVersion targetVersion = JavaVersion.toVersion(targetCompatibility);
    if (targetVersion.isJava9Compatible()) {
        options.compilerArgs += ["--add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED"]
    }
}

javadoc {
    options.addStringOption("-add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED")
}

project.afterEvaluate {
    JavaVersion targetVersion = JavaVersion.toVersion(targetCompatibility)

    // cannot contain the first version
    // should be '8' or '11' etc.
    String version = targetVersion.toString()
    if (version.length() > 2) {
        version = targetVersion.toString().substring(2)
    }

    compileJava.options.compilerArgs.removeAll(['--release', version])
}

repositories {
    mavenCentral()
}


configurations {
    includeJars
}

tasks.withType(JavaCompile) {
    options.warnings = false
}

dependencies {
    if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
        compile files("${System.properties['java.home']}/../lib/tools.jar")
    }

    def jacksonVersion = "2.12.6"

    compile 'org.jooq:jooq:3.10.8'
    compile 'org.bouncycastle:bcprov-jdk15on:1.70'
    compile 'org.bouncycastle:bcpkix-jdk15on:1.70'
    compile 'org.xerial:sqlite-jdbc:3.32.3.2'
    compile 'com.google.guava:guava:30.1-jre'
    compile 'com.google.code.gson:gson:2.8.9'
    compile 'org.checkerframework:checker-qual:3.5.0'
    compile "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}"
    compile "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
    compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.1'
    compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1'
    compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
    compile group: 'commons-io', name: 'commons-io', version: '2.7'
    compile group: 'com.google.errorprone', name: 'error_prone_annotations', version: '2.9.0'
    compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.19.2'
    implementation 'io.grpc:grpc-netty:1.44.0'
    implementation 'io.grpc:grpc-protobuf:1.44.0'
    implementation 'io.grpc:grpc-stub:1.44.0'
    implementation 'javax.annotation:javax.annotation-api:1.3.2'

    // JDK9+ has to run powermock 2+. https://github.com/powermock/powermock/issues/888
    testCompile group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.0'
    testCompile group: 'org.powermock', name: 'powermock-module-junit4', version: '2.0.0'
    testCompile group: 'org.mockito', name: 'mockito-core', version: '2.23.0'
    testCompile group: 'org.powermock', name: 'powermock-core', version: '2.0.0'
    testCompile group: 'org.powermock', name: 'powermock-api-support', version: '2.0.0'
    testCompile group: 'org.powermock', name: 'powermock-module-junit4-common', version: '2.0.0'
    testCompile group: 'org.javassist', name: 'javassist', version: '3.24.0-GA'
    testCompile group: 'org.powermock', name: 'powermock-reflect', version: '2.0.0'
    testCompile group: 'net.bytebuddy', name: 'byte-buddy', version: '1.9.3'
    testCompile group: 'org.objenesis', name: 'objenesis', version: '3.0.1'
    testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1'
    testCompile group: 'org.hamcrest', name: 'hamcrest', version: '2.1'
    testCompile group: 'junit', name: 'junit', version: '4.12'
}

protobuf {
    protoc {
        artifact = "com.google.protobuf:protoc:3.19.2"
    }

    plugins {
        grpc {
            artifact = 'io.grpc:protoc-gen-grpc-java:1.44.0'
        }

    }
    generateProtoTasks {
        all()*.plugins {
            grpc {}
        }
    }
}

idea {
    module {
        sourceDirs += file("${projectDir}/build/generated/source/proto/main/java");
        sourceDirs += file("${projectDir}/build/generated/source/proto/main/grpc");
    }
}

import groovy.json.JsonSlurper
import java.nio.file.Paths
import org.ajoberstar.gradle.git.tasks.GitClone

String paDir
String dockerBuildDir
String dockerArtifactsDir

task clonePaGitRepo(type: GitClone) {
    dockerBuildDir = Paths.get(getProject().getBuildDir().toString(), "docker-build").toString()
    mkdir dockerBuildDir
    if (localPaDir != null && !localPaDir.empty) {
        // If you already have the performance Analyzer sources checked out then update the
        // localPaDir in gradle.properties with the path.
        paDir = localPaDir
    } else {
        paDir = Paths.get(dockerBuildDir, "pa").toString()
    }

    def destination = file(paDir)
    uri = gitPaRepo
    branch = gitPaBranch
    destinationPath = destination
    bare = false
    enabled = !destination.exists() //to clone only once
}

task removeLicenses(type: Delete) {
    dependsOn(clonePaGitRepo)
    def toDelete = Paths.get(paDir, 'licenses', "performanceanalyzer-rca-${version}.jar.sha1")
    delete(toDelete)
}

task regenerateLicenses(type: Exec) {
    dependsOn(removeLicenses)
    workingDir(paDir)
    commandLine './gradlew', 'updateShas'
}

task buildPa(type: Exec) {
    dependsOn(assemble, publishToMavenLocal, regenerateLicenses)
    workingDir paDir
    println String.format('pa in dir: (%s) will be built.', paDir)
    commandLine './gradlew', 'assemble', "-Dopensearch.version=${opensearch_version}"
    println "PA repo located at '" + paDir + "' will be used."
}

task copyAllArtifacts(type: Copy) {
    dependsOn(buildPa)
    def projectPathStr = getProjectDir().toPath().toString()
    def dockerArtifacts = Paths.get(projectPathStr, 'docker').toString()
    def rcaArtifacts = Paths.get(projectPathStr, 'build', 'distributions', "performance-analyzer-rca-${version}.zip")
    def paArtifacts = Paths.get(paDir, 'build', 'distributions', "opensearch-performance-analyzer-${version}.zip")

    dockerArtifactsDir = Paths.get(dockerBuildDir, 'rca-docker')
    from(dockerArtifacts)
    from(rcaArtifacts)
    from(paArtifacts)
    into(dockerArtifactsDir)
}

task copyReaderMetricsFiles(type: Copy) {
    dependsOn(buildPa)
    copy{
        from ('src/test/resources/reader'){
        }
        into 'build/private/test_resources'
    }
}

task buildDocker(type: Exec) {
    dependsOn(copyAllArtifacts)

    workingDir(dockerArtifactsDir)
    commandLine 'docker', 'build', '-t', 'opensearch/pa-rca:1.3', '.'
}

task runDocker(type: Exec) {
    dependsOn(buildDocker)

    workingDir(dockerArtifactsDir)

    // This block is included to make the runDocker task work with Github Actions
    // It sets the path to the docker-compose program from an environment variable
    // The DOCKER_COMPOSE_LOCATION environment variable is set in the gradle.yml file inside the
    // performance-analyzer repository.
    def docker_compose_location = "docker-compose"
    if (System.getenv("DOCKER_COMPOSE_LOCATION") != null) {
        docker_compose_location = System.getenv("DOCKER_COMPOSE_LOCATION")
    }

    environment 'DATA_VOLUME1', 'opensearchdata1'
    environment 'DATA_VOLUME2', 'opensearchdata2'

    commandLine(docker_compose_location,
            '-f', 'docker-compose.yml',
            '-f', 'docker-compose.hostports.yml',
            '-f', 'docker-compose.cluster.yml',
            'up', 'opensearch1', 'opensearch2')

}

def printCommandOutput(proc) {
    Thread.start {
        BufferedReader stdInput = new BufferedReader(new
                InputStreamReader(proc.getInputStream()));

        BufferedReader stdError = new BufferedReader(new
                InputStreamReader(proc.getErrorStream()));

        System.out.println("Here is the standard output of the command:\n");
        String s = null;
        while ((s = stdInput.readLine()) != null) {
            System.out.println(s);
        }

        System.out.println("Here is the standard error of the command (if any):\n");
        while ((s = stdError.readLine()) != null) {
            System.out.println(s);
        }
    }
    int exitVal = proc.waitFor()
    System.out.println("Process exitValue: " + exitVal);
}

def runCommand(commandArr, envArr, workingDir, printOutput) {
    try {
        Process proc = Runtime.getRuntime().exec(commandArr, envArr, file(workingDir))
        if (printOutput) {
            printCommandOutput(proc)
        }
    } catch (Throwable th) {
        th.printStackTrace()
    }
}

def runInProcess(commandArr) {
    Process proc = new ProcessBuilder(commandArr).start();
    printCommandOutput(proc)
}

def runDockerCompose = { executionPath ->
    // This block is included to make the runDocker task work with Github Actions
    // It sets the path to the docker-compose program from an environment variable
    // The DOCKER_COMPOSE_LOCATION environment variable is set in the gradle.yml file inside the
    // performance-analyzer repository.
    String dockerCompose = System.getenv("DOCKER_COMPOSE_LOCATION") == null ?
            "docker-compose" : System.getenv("DOCKER_COMPOSE_LOCATION")
    String[] commandArray = [dockerCompose,
                             '-f', 'docker-compose.yml',
                             '-f', 'docker-compose.hostports.yml',
                             '-f', 'docker-compose.cluster.yml',
                             'up', 'opensearch1', 'opensearch2']
    String[] env = ['DATA_VOLUME1=opensearchdata1',
                    'DATA_VOLUME2=opensearchdata2']
    runCommand(commandArray, env, file(executionPath), true)
}

def getHttpResponse(server, timeoutSeconds) {
    URL url = new URL(server)
    long timeout = timeoutSeconds * 1000
    long increments = 1000
    long passed = 0
    println 'waiting for ' + server
    while (passed < timeout) {
        try {
            HttpURLConnection con = (HttpURLConnection) url.openConnection()
            con.setRequestMethod("GET")
            InputStreamReader isr = new InputStreamReader(con.getInputStream())
            BufferedReader bufR = new BufferedReader(isr)
            String inputLine;
            StringBuffer content = new StringBuffer()
            while ((inputLine = bufR.readLine()) != null) {
                content.append(inputLine);
            }
            println(content.toString())
            bufR.close()
            break
        } catch (IOException) {
            Thread.sleep(increments);
            passed += increments
        }
    }
    if (passed == timeout) {
        throw new GradleException(String.format("ERROR:  %s not ready.", server));
    }
}

/**
 * enableComponent is used to enable PA or RCA based on the given parameters
 * @param endpoint the endpoint to make the request against e.g. localhost:9200/pa/rca/example
 * @param desiredState
 *          The list of enabled components is represented as an n-bit binary number
 *          The LSB represents the state of PA and the LSB+1th bit is RCA
 *          0x03 (00000011 in binary) is therefore a state where both PA and RCA are enabled
 * @param timeoutSeconds we will attempt to enable the component for this many seconds before
 *          giving up and throwing a GradleException
 */
def enableComponent(endpoint, desiredState, timeoutSeconds) {
    long timeout = timeoutSeconds * 1000
    long increments = 1000
    long passed = 0
    println 'waiting for ' + endpoint
    while (passed < timeout) {
        try {
            def p = ['curl', endpoint, '-H', 'Content-Type: application/json', '-d', '{"enabled": true}'].execute()
            def json = new JsonSlurper().parseText(p.text)
            if (json.get("currentPerformanceAnalyzerClusterState").equals(desiredState) ||
                    // desiredState+2 means that the "next" significant component is enabled
                    json.get("currentPerformanceAnalyzerClusterState").equals(desiredState + 2)) {
                break
            }
        } catch (Exception) {
        }
        Thread.sleep(increments);
        passed += increments
    }
    if (passed == timeout) {
        throw new GradleException(String.format("ERROR:  %s not ready.", endpoint));
    }
}

Boolean openSearchIsUp

def openSearchUpChecker = {
    String server = "http://localhost:9200"
    int timeoutSeconds = 2 * 60
    getHttpResponse(server, timeoutSeconds)
    openSearchIsUp = true
}

// Attempts to enable PA for up to 2 minutes. Returns when PA is successfully enabled or
// throws an Exception if the timeout is exceeded
def paUpChecker = {
    String server = "localhost:9200/_plugins/_performanceanalyzer/cluster/config"
    int timeoutSeconds = 2 * 60
    enableComponent(server, 1, timeoutSeconds)
}

// Attempts to enable RCA for up to 2 minutes. Returns when RCA is successfully enabled or
// throws an Exception if the timeout is exceeded
def rcaUpChecker = {
    String server = "localhost:9200/_plugins/_performanceanalyzer/rca/cluster/config"
    int timeoutSeconds = 2 * 60
    enableComponent(server, 3, timeoutSeconds)
}

Thread openSearchRunnerThread

def runInThread(cl, input) {
    return Thread.start {
        cl(input)
    }
}

task waitForOpenSearch {
    dependsOn(buildDocker)
    doLast {
        openSearchRunnerThread = runInThread(runDockerCompose, dockerArtifactsDir)
        openSearchUpChecker()
    }
}

String perfTopDir
String perfTopZip
String perfTopBin

task downloadPerfTop(type: Download) {
    String zip = 'perf-top-1.9.0.0-MACOS.zip'
    perfTopDir = Paths.get(dockerBuildDir, 'perftop').toString()
    mkdir perfTopDir

    src 'https://d3g5vo6xdbdb9a.cloudfront.net/downloads/perftop/' + zip
    dest perfTopDir
    overwrite false

    perfTopZip = Paths.get(perfTopDir, zip).toString()
    File f = file(perfTopZip)
    enabled = !f.exists() //to clone only once
}

task unzipPerfTop(type: Copy) {
    dependsOn(downloadPerfTop)

    perfTopBin = Paths.get(perfTopDir, 'perf-top-macos').toString()

    def zipFile = file(perfTopZip)
    def outputDir = file(perfTopDir)

    from zipTree(zipFile)
    into outputDir

    enabled = !file(perfTopBin).exists()
}

// Attempts to enable PA for up to 2 minutes. Returns when PA is successfully enabled or
// throws an Exception if the timeout is exceeded
task enablePa() {
    dependsOn(waitForOpenSearch)
    doLast {
        paUpChecker()
    }
}

// Attempts to enable RCA for up to 2 minutes. Returns when RCA is successfully enabled or
// throws an Exception if the timeout is exceeded
task enableRca() {
    dependsOn(enablePa)
    doLast {
        rcaUpChecker()
    }
}

task runPerftop {
    dependsOn(enableRca, unzipPerfTop)
    doLast {
        println '============================================================================'
        println 'invoke perftop as: '
        println String.format("%s --dashboard %s", Paths.get(perfTopBin).toString(), 'ClusterOverview')
        println '============================================================================'
    }
}

task generateLoad {
    dependsOn(enableRca, runPerftop)

    doLast {
        String[] deleteIndex = ['curl', '-X', 'DELETE', 'localhost:9200/accounts?pretty']
        String[] bulk = ['curl', '-s', '-H', "Content-Type: application/x-ndjson",
                         '-XPOST', 'localhost:9200/_bulk', '--data-binary', '@accounts.json']
        String[] search = ['curl', '-X', 'GET', 'localhost:9200/_search?pretty', '-H',
                           'Content-Type:application/json', '-d',
                           '{"query": {"range": {"balance": {"gte": 100, "lte": 100000}}}}']

        String wdir = Paths.get(getProjectDir().getAbsolutePath(), 'src', 'test', 'resources')
                .toString()
        while (true) {
            // runCommand(deleteIndex, null, wdir, false)
            //println("starting indexing: " + System.currentTimeMillis())
            runCommand(bulk, null, wdir, false)
            //println("completed indexing: " + System.currentTimeMillis())
            Thread.sleep(3000)
            //println("starting search" + System.currentTimeMillis())
            runCommand(search, null, wdir, false)
            Thread.sleep(1000)
            //println("completed search" + System.currentTimeMillis())
            runCommand(deleteIndex, null, wdir, false)
            Thread.sleep(1000)
        }
    }
}

task runRally {
    dependsOn(enableRca, runPerftop)

    doLast {
        String rallyTrack='pmc'

        String[] opensearchrally = ['docker', 'run', '--network=host', 'elastic/rally',
                                '--track=' + rallyTrack, '--test-mode', '--pipeline=benchmark-only',
                '--target-hosts=localhost:9200']

        String wdir = Paths.get(getProjectDir().getAbsolutePath(), 'src', 'test', 'resources')
                .toString()
        while (true) {
            runCommand(opensearchrally, null, wdir, true)
            println('Get all RCAs using:')
            println('curl localhost:9600/_plugins/_performanceanalyzer/rca?all | python -m json.tool')
            Thread.sleep(1000)
        }
    }
}