From db13f31b3bc9eccff9a9833b7b732abc3a42ff6c Mon Sep 17 00:00:00 2001 From: andrerodrigues Date: Mon, 18 Mar 2024 23:39:06 +0000 Subject: [PATCH 1/8] Update mysql connector version and remove mxj --- deploy-service/common/pom.xml | 9 +- .../deployservice/common/DBUtils.java | 208 ++++++++++++++++++ .../deployservice/db/DatabaseUtil.java | 10 + .../pinterest/deployservice/db/DBDAOTest.java | 42 +--- deploy-service/teletraanservice/pom.xml | 5 - .../config/EmbeddedDataSourceFactory.java | 43 +--- 6 files changed, 231 insertions(+), 86 deletions(-) create mode 100644 deploy-service/common/src/main/java/com/pinterest/deployservice/common/DBUtils.java diff --git a/deploy-service/common/pom.xml b/deploy-service/common/pom.xml index 7bfc60789b..f4f33681d4 100644 --- a/deploy-service/common/pom.xml +++ b/deploy-service/common/pom.xml @@ -79,7 +79,7 @@ mysql mysql-connector-java - 5.1.49 + 8.0.33 commons-dbcp @@ -171,17 +171,10 @@ 3.1 test - - mysql - mysql-connector-mxj - 5.0.12 - test - com.ibatis ibatis2-common 2.1.7.597 - test org.kohsuke diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/common/DBUtils.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/common/DBUtils.java new file mode 100644 index 0000000000..578dbc3102 --- /dev/null +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/common/DBUtils.java @@ -0,0 +1,208 @@ +package com.pinterest.deployservice.common; + +import com.ibatis.common.jdbc.ScriptRunner; +import com.pinterest.deployservice.db.DatabaseUtil; +import org.apache.commons.dbcp.BasicDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; + +public class DBUtils { + private static final String DEFAULT_DB_NAME = "deploy"; + private static final int DEFAULT_PORT = 3303; + private static final String CONTAINER_NAME = "local-mysql8"; + private static final String IMAGE_NAME = "mysql:8-oracle"; + private static final Logger LOG = LoggerFactory.getLogger(DatabaseUtil.class); + private static final int MAX_RETRY = 5; + private static final int START_MYSQL_MAX_RETRY = 60; + + public static BasicDataSource setupDataSource() throws Exception { + boolean createDBInstance = !Boolean.parseBoolean(System.getenv("USE_LOCAL_MYSQL_INSTANCE")); + return DBUtils.setupDataSource(DEFAULT_PORT, createDBInstance); + } + + public static void startMysqlContainer(int port) throws Exception { + String runDocker = + String.format( + "docker run --rm -d -p %d:%d -e MYSQL_TCP_PORT=%d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -e" + + " MYSQL_DATABASE=%s --name %s %s", + port, port, port, DEFAULT_DB_NAME, CONTAINER_NAME, IMAGE_NAME); + + LOG.info("run docker command: " + runDocker); + + Process process = Runtime.getRuntime().exec(runDocker); + process.waitFor(); + + // wait for mysql db is ready + int counter = 0; + String logs = null; + while (true) { + if (counter == START_MYSQL_MAX_RETRY) { + throw new DatabaseStartException(logs); + } + Thread.sleep(1000); + runDocker = String.format("docker exec %s mysql -u root deploy", CONTAINER_NAME); + process = Runtime.getRuntime().exec(runDocker); + int exitValue = process.waitFor(); + if (exitValue == 0) { + break; + } + + logs = getLogs(process.getErrorStream()); + LOG.info( + "wait for mysql ready: command = {}, exitValue = {}, logs = {}", + runDocker, + exitValue, + logs); + counter++; + } + } + + public static void stopMysqlContainer() throws Exception { + String runDocker = String.format("docker stop %s", CONTAINER_NAME); + + LOG.info("run docker command: " + runDocker); + Process process = Runtime.getRuntime().exec(runDocker); + process.waitFor(); + } + + public static BasicDataSource setupDataSource(String baseDir, int port) throws Exception { + return DBUtils.setupDataSource(port, true); + } + + public static BasicDataSource setupDataSource(int port, boolean createDBInstance) + throws Exception { + if (createDBInstance) { + try { + // making sure we do not have anything running + stopMysqlContainer(); + } catch (Exception e) { + // ignore + } + + // start the docker container + startMysqlContainer(port); + } + + // use local mysql container for unit testing because the MXJ embedded db only support mysql5 in + // AMD. + // mysql docker container opens up our test db to be mysql8+, and different arch like ARM64 + BasicDataSource dataSource = DatabaseUtil.createLocalDataSource(DEFAULT_DB_NAME, port); + + Connection conn = getConnectionWithRetry(dataSource); + ScriptRunner runner = new ScriptRunner(conn, false, true); + runner.runScript( + new BufferedReader( + new InputStreamReader(DBUtils.class.getResourceAsStream("/sql/cleanup.sql")))); + runner.runScript( + new BufferedReader( + new InputStreamReader(DBUtils.class.getResourceAsStream("/sql/deploy.sql")))); + + //runSchemaUpdateScripts(runner); + + conn.close(); + return dataSource; + } + + public static Connection getConnectionWithRetry(BasicDataSource dataSource) + throws SQLException, InterruptedException { + int retryCount = 0; + while (retryCount < MAX_RETRY) { + try { + return dataSource.getConnection(); + } catch (SQLException ex) { + retryCount++; + LOG.warn("Failed to get connection after {} attempt(s)", retryCount); + if (retryCount == MAX_RETRY) { + throw ex; + } + TimeUnit.MILLISECONDS.sleep(1000 * retryCount); + } + } + return null; + } + + public static void tearDownDataSource() throws Exception { + if (!Boolean.parseBoolean(System.getenv("USE_LOCAL_MYSQL_INSTANCE"))) { + stopMysqlContainer(); + } + } + + public static void truncateAllTables(BasicDataSource dataSource) throws Exception { + try (Connection conn = dataSource.getConnection(); + Statement query = conn.createStatement(); + Statement stmt = conn.createStatement(); ) { + ResultSet rs = + query.executeQuery( + "SELECT table_name FROM information_schema.tables WHERE table_schema = SCHEMA()"); + stmt.addBatch("SET FOREIGN_KEY_CHECKS=0"); + while (rs.next()) { + String sqlStatement = String.format("TRUNCATE `%s`", rs.getString(1)); + stmt.addBatch(sqlStatement); + } + stmt.addBatch("SET FOREIGN_KEY_CHECKS=1"); + stmt.executeBatch(); + LOG.info(String.format("Truncated all tables")); + } + } + + private static void runSchemaUpdateScripts(ScriptRunner runner) { + int version = 22; + int executed = 0; + while (true) { + String scriptName = String.format("/sql/tools/mysql/schema-update-%d.sql", version); + try { + runner.runScript(new InputStreamReader(DBUtils.class.getResourceAsStream(scriptName))); + LOG.info(String.format("Applied script: %s", scriptName)); + } catch (Exception e) { + if (executed == 0) { + throw new Error( + "Could not run a single update script. Check starting version is correct"); + } + break; + } + version++; + executed++; + } + } + + private static String getLogs(InputStream in) { + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + StringBuilder sb = new StringBuilder(); + + while (true) { + try { + String line = br.readLine(); + if (line == null) { + break; + } + sb.append(line); + } catch (Exception e) { + throw new LogReadException(e); + } + } + return sb.toString(); + } + + private static class DatabaseStartException extends RuntimeException { + + DatabaseStartException(String logs) { + super("Can't connect to MySQL database: " + logs); + } + } + + private static class LogReadException extends RuntimeException { + + LogReadException(Exception e) { + super("Can't read logs", e); + } + } +} diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java index 2ab7ba9bf9..c44fc52121 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java @@ -50,6 +50,16 @@ public static BasicDataSource createMysqlDataSource(String host, int port, MAX_WAIT_TIME_FOR_CONN_IN_MS); } + // local mysql source, for unit test only that depends on a local mysql db (that can be launched + // in docker container) + // example of command to launch local mysql container: + // common/src/main/java/com/pinterest/clusterservice/common/DBUtils.java:startMysqlContainer + public static BasicDataSource createLocalDataSource(String dbName, int port) { + String url = String.format("jdbc:mysql://0.0.0.0:%d/%s?useSSL=false", port, dbName); + return createDataSource( + MYSQL_JDBC_DRIVER, url, "root", "", "0:8:8:0", MAX_WAIT_TIME_FOR_CONN_IN_MS, null); + } + public static BasicDataSource createMysqlDataSource(String host, int port, String user, String passwd, String poolSize, Map connectionProperties) { diff --git a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java index c8e6de8ebf..24e1f25554 100644 --- a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java +++ b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java @@ -15,12 +15,6 @@ */ package com.pinterest.deployservice.db; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - import com.pinterest.deployservice.bean.AcceptanceStatus; import com.pinterest.deployservice.bean.AcceptanceType; import com.pinterest.deployservice.bean.AgentBean; @@ -58,6 +52,7 @@ import com.pinterest.deployservice.bean.UserRolesBean; import com.pinterest.deployservice.common.CommonUtils; import com.pinterest.deployservice.common.Constants; +import com.pinterest.deployservice.common.DBUtils; import com.pinterest.deployservice.dao.AgentDAO; import com.pinterest.deployservice.dao.AgentErrorDAO; import com.pinterest.deployservice.dao.BuildDAO; @@ -75,10 +70,6 @@ import com.pinterest.deployservice.dao.TokenRolesDAO; import com.pinterest.deployservice.dao.UserRolesDAO; import com.pinterest.deployservice.dao.UtilDAO; - - -import com.ibatis.common.jdbc.ScriptRunner; -import com.mysql.management.driverlaunched.ServerLauncherSocketFactory; import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.lang.builder.EqualsBuilder; import org.joda.time.Interval; @@ -86,9 +77,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; import java.sql.Connection; import java.util.ArrayList; import java.util.Arrays; @@ -98,13 +86,14 @@ import java.util.List; import java.util.Set; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; -public class DBDAOTest { - - private final static String DEFAULT_BASE_DIR = "/tmp/deploy-unit-test"; - private final static String DEFAULT_DB_NAME = "deploy"; - private final static int DEFAULT_PORT = 3303; +public class DBDAOTest { private static BuildDAO buildDAO; private static AgentDAO agentDAO; private static AgentErrorDAO agentErrorDAO; @@ -125,20 +114,9 @@ public class DBDAOTest { @BeforeClass public static void setUpClass() throws Exception { - try { - // making sure we do not have anything running - ServerLauncherSocketFactory.shutdown(new File(DEFAULT_BASE_DIR), null); - } catch (Exception e) { - // ignore - } - BasicDataSource DATASOURCE = DatabaseUtil.createMXJDataSource(DEFAULT_DB_NAME, - DEFAULT_BASE_DIR, DEFAULT_PORT); + BasicDataSource DATASOURCE = DBUtils.setupDataSource(); Connection conn = DATASOURCE.getConnection(); - ScriptRunner runner = new ScriptRunner(conn, false, true); - runner.runScript(new BufferedReader(new InputStreamReader( - DBDAOTest.class.getResourceAsStream("/sql/cleanup.sql")))); - runner.runScript(new BufferedReader(new InputStreamReader( - DBDAOTest.class.getResourceAsStream("/sql/deploy.sql")))); + conn.prepareStatement("SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));").execute(); buildDAO = new DBBuildDAOImpl(DATASOURCE); agentDAO = new DBAgentDAOImpl(DATASOURCE); agentErrorDAO = new DBAgentErrorDAOImpl(DATASOURCE); @@ -160,7 +138,7 @@ public static void setUpClass() throws Exception { @AfterClass public static void tearDownClass() throws Exception { - ServerLauncherSocketFactory.shutdown(new File(DEFAULT_BASE_DIR), null); + DBUtils.tearDownDataSource(); } @Test diff --git a/deploy-service/teletraanservice/pom.xml b/deploy-service/teletraanservice/pom.xml index 577ab5072d..0f0d0638ef 100644 --- a/deploy-service/teletraanservice/pom.xml +++ b/deploy-service/teletraanservice/pom.xml @@ -73,11 +73,6 @@ quartz - - mysql - mysql-connector-mxj - 5.0.12 - com.ibatis ibatis2-common diff --git a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java index c43c8854ed..a6b87d2f88 100644 --- a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java +++ b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java @@ -15,53 +15,14 @@ */ package com.pinterest.teletraan.config; -import com.pinterest.deployservice.db.DatabaseUtil; - -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import com.ibatis.common.jdbc.ScriptRunner; -import com.mysql.management.driverlaunched.ServerLauncherSocketFactory; - +import com.pinterest.deployservice.common.DBUtils; import org.apache.commons.dbcp.BasicDataSource; -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; -import java.sql.Connection; - @JsonTypeName("embedded") public class EmbeddedDataSourceFactory implements DataSourceFactory { - private final static String DEFAULT_BASE_DIR = "/tmp/teletraan/db"; - private final static String DEFAULT_DB_NAME = "deploy"; - private final static int DEFAULT_PORT = 3305; - - @JsonProperty - private String workDir = DEFAULT_BASE_DIR; - - public String getWorkDir() { - return workDir; - } - - public void setWorkDir(String workDir) { - this.workDir = workDir; - } - public BasicDataSource build() throws Exception { - try { - // making sure we do not have anything running - ServerLauncherSocketFactory.shutdown(new File(workDir), null); - } catch (Exception e) { - // ignore - } - - BasicDataSource - DATASOURCE = - DatabaseUtil.createMXJDataSource(DEFAULT_DB_NAME, workDir, DEFAULT_PORT); - Connection conn = DATASOURCE.getConnection(); - ScriptRunner runner = new ScriptRunner(conn, false, false); - runner.runScript(new BufferedReader(new InputStreamReader( - DatabaseUtil.class.getResourceAsStream("/sql/deploy.sql")))); - return DATASOURCE; + return DBUtils.setupDataSource(); } } From b662aadeb524ad74f34889b27d80bf8a8d7eb019 Mon Sep 17 00:00:00 2001 From: andrerodrigues Date: Tue, 19 Mar 2024 20:41:26 +0000 Subject: [PATCH 2/8] Add testcontainers to database tests --- deploy-service/common/pom.xml | 18 ++ .../deployservice/common/DBUtils.java | 208 ------------------ .../pinterest/deployservice/db/DBDAOTest.java | 22 +- .../pinterest/deployservice/db/DBUtils.java | 25 +++ .../config/EmbeddedDataSourceFactory.java | 3 +- 5 files changed, 57 insertions(+), 219 deletions(-) delete mode 100644 deploy-service/common/src/main/java/com/pinterest/deployservice/common/DBUtils.java create mode 100644 deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java diff --git a/deploy-service/common/pom.xml b/deploy-service/common/pom.xml index f4f33681d4..164e828139 100644 --- a/deploy-service/common/pom.xml +++ b/deploy-service/common/pom.xml @@ -171,6 +171,24 @@ 3.1 test + + org.testcontainers + testcontainers + 1.19.7 + test + + + org.testcontainers + junit-jupiter + 1.19.7 + test + + + org.testcontainers + mysql + 1.19.7 + test + com.ibatis ibatis2-common diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/common/DBUtils.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/common/DBUtils.java deleted file mode 100644 index 578dbc3102..0000000000 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/common/DBUtils.java +++ /dev/null @@ -1,208 +0,0 @@ -package com.pinterest.deployservice.common; - -import com.ibatis.common.jdbc.ScriptRunner; -import com.pinterest.deployservice.db.DatabaseUtil; -import org.apache.commons.dbcp.BasicDataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.concurrent.TimeUnit; - -public class DBUtils { - private static final String DEFAULT_DB_NAME = "deploy"; - private static final int DEFAULT_PORT = 3303; - private static final String CONTAINER_NAME = "local-mysql8"; - private static final String IMAGE_NAME = "mysql:8-oracle"; - private static final Logger LOG = LoggerFactory.getLogger(DatabaseUtil.class); - private static final int MAX_RETRY = 5; - private static final int START_MYSQL_MAX_RETRY = 60; - - public static BasicDataSource setupDataSource() throws Exception { - boolean createDBInstance = !Boolean.parseBoolean(System.getenv("USE_LOCAL_MYSQL_INSTANCE")); - return DBUtils.setupDataSource(DEFAULT_PORT, createDBInstance); - } - - public static void startMysqlContainer(int port) throws Exception { - String runDocker = - String.format( - "docker run --rm -d -p %d:%d -e MYSQL_TCP_PORT=%d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -e" - + " MYSQL_DATABASE=%s --name %s %s", - port, port, port, DEFAULT_DB_NAME, CONTAINER_NAME, IMAGE_NAME); - - LOG.info("run docker command: " + runDocker); - - Process process = Runtime.getRuntime().exec(runDocker); - process.waitFor(); - - // wait for mysql db is ready - int counter = 0; - String logs = null; - while (true) { - if (counter == START_MYSQL_MAX_RETRY) { - throw new DatabaseStartException(logs); - } - Thread.sleep(1000); - runDocker = String.format("docker exec %s mysql -u root deploy", CONTAINER_NAME); - process = Runtime.getRuntime().exec(runDocker); - int exitValue = process.waitFor(); - if (exitValue == 0) { - break; - } - - logs = getLogs(process.getErrorStream()); - LOG.info( - "wait for mysql ready: command = {}, exitValue = {}, logs = {}", - runDocker, - exitValue, - logs); - counter++; - } - } - - public static void stopMysqlContainer() throws Exception { - String runDocker = String.format("docker stop %s", CONTAINER_NAME); - - LOG.info("run docker command: " + runDocker); - Process process = Runtime.getRuntime().exec(runDocker); - process.waitFor(); - } - - public static BasicDataSource setupDataSource(String baseDir, int port) throws Exception { - return DBUtils.setupDataSource(port, true); - } - - public static BasicDataSource setupDataSource(int port, boolean createDBInstance) - throws Exception { - if (createDBInstance) { - try { - // making sure we do not have anything running - stopMysqlContainer(); - } catch (Exception e) { - // ignore - } - - // start the docker container - startMysqlContainer(port); - } - - // use local mysql container for unit testing because the MXJ embedded db only support mysql5 in - // AMD. - // mysql docker container opens up our test db to be mysql8+, and different arch like ARM64 - BasicDataSource dataSource = DatabaseUtil.createLocalDataSource(DEFAULT_DB_NAME, port); - - Connection conn = getConnectionWithRetry(dataSource); - ScriptRunner runner = new ScriptRunner(conn, false, true); - runner.runScript( - new BufferedReader( - new InputStreamReader(DBUtils.class.getResourceAsStream("/sql/cleanup.sql")))); - runner.runScript( - new BufferedReader( - new InputStreamReader(DBUtils.class.getResourceAsStream("/sql/deploy.sql")))); - - //runSchemaUpdateScripts(runner); - - conn.close(); - return dataSource; - } - - public static Connection getConnectionWithRetry(BasicDataSource dataSource) - throws SQLException, InterruptedException { - int retryCount = 0; - while (retryCount < MAX_RETRY) { - try { - return dataSource.getConnection(); - } catch (SQLException ex) { - retryCount++; - LOG.warn("Failed to get connection after {} attempt(s)", retryCount); - if (retryCount == MAX_RETRY) { - throw ex; - } - TimeUnit.MILLISECONDS.sleep(1000 * retryCount); - } - } - return null; - } - - public static void tearDownDataSource() throws Exception { - if (!Boolean.parseBoolean(System.getenv("USE_LOCAL_MYSQL_INSTANCE"))) { - stopMysqlContainer(); - } - } - - public static void truncateAllTables(BasicDataSource dataSource) throws Exception { - try (Connection conn = dataSource.getConnection(); - Statement query = conn.createStatement(); - Statement stmt = conn.createStatement(); ) { - ResultSet rs = - query.executeQuery( - "SELECT table_name FROM information_schema.tables WHERE table_schema = SCHEMA()"); - stmt.addBatch("SET FOREIGN_KEY_CHECKS=0"); - while (rs.next()) { - String sqlStatement = String.format("TRUNCATE `%s`", rs.getString(1)); - stmt.addBatch(sqlStatement); - } - stmt.addBatch("SET FOREIGN_KEY_CHECKS=1"); - stmt.executeBatch(); - LOG.info(String.format("Truncated all tables")); - } - } - - private static void runSchemaUpdateScripts(ScriptRunner runner) { - int version = 22; - int executed = 0; - while (true) { - String scriptName = String.format("/sql/tools/mysql/schema-update-%d.sql", version); - try { - runner.runScript(new InputStreamReader(DBUtils.class.getResourceAsStream(scriptName))); - LOG.info(String.format("Applied script: %s", scriptName)); - } catch (Exception e) { - if (executed == 0) { - throw new Error( - "Could not run a single update script. Check starting version is correct"); - } - break; - } - version++; - executed++; - } - } - - private static String getLogs(InputStream in) { - BufferedReader br = new BufferedReader(new InputStreamReader(in)); - StringBuilder sb = new StringBuilder(); - - while (true) { - try { - String line = br.readLine(); - if (line == null) { - break; - } - sb.append(line); - } catch (Exception e) { - throw new LogReadException(e); - } - } - return sb.toString(); - } - - private static class DatabaseStartException extends RuntimeException { - - DatabaseStartException(String logs) { - super("Can't connect to MySQL database: " + logs); - } - } - - private static class LogReadException extends RuntimeException { - - LogReadException(Exception e) { - super("Can't read logs", e); - } - } -} diff --git a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java index 24e1f25554..4a005294d0 100644 --- a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java +++ b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java @@ -52,7 +52,6 @@ import com.pinterest.deployservice.bean.UserRolesBean; import com.pinterest.deployservice.common.CommonUtils; import com.pinterest.deployservice.common.Constants; -import com.pinterest.deployservice.common.DBUtils; import com.pinterest.deployservice.dao.AgentDAO; import com.pinterest.deployservice.dao.AgentErrorDAO; import com.pinterest.deployservice.dao.BuildDAO; @@ -75,7 +74,9 @@ import org.joda.time.Interval; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.testcontainers.containers.MySQLContainer; import java.sql.Connection; import java.util.ArrayList; @@ -112,11 +113,19 @@ public class DBDAOTest { private static ScheduleDAO scheduleDAO; private static UtilDAO utilDAO; + @ClassRule + public static final MySQLContainer mysql = new MySQLContainer().withDatabaseName("deploy"); + + @BeforeClass public static void setUpClass() throws Exception { - BasicDataSource DATASOURCE = DBUtils.setupDataSource(); - Connection conn = DATASOURCE.getConnection(); - conn.prepareStatement("SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));").execute(); + BasicDataSource DATASOURCE = new BasicDataSource(); + DATASOURCE.setDriverClassName("com.mysql.jdbc.Driver"); + DATASOURCE.setUrl(mysql.getJdbcUrl()); + DATASOURCE.setUsername(mysql.getUsername()); + DATASOURCE.setPassword(mysql.getPassword()); + DBUtils.runMigrations(DATASOURCE); + buildDAO = new DBBuildDAOImpl(DATASOURCE); agentDAO = new DBAgentDAOImpl(DATASOURCE); agentErrorDAO = new DBAgentErrorDAOImpl(DATASOURCE); @@ -136,11 +145,6 @@ public static void setUpClass() throws Exception { utilDAO = new DBUtilDAOImpl(DATASOURCE); } - @AfterClass - public static void tearDownClass() throws Exception { - DBUtils.tearDownDataSource(); - } - @Test public void testDeploymentQueries() throws Exception { long now = System.currentTimeMillis(); diff --git a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java new file mode 100644 index 0000000000..e97d8905bc --- /dev/null +++ b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java @@ -0,0 +1,25 @@ +package com.pinterest.deployservice.db; + +import com.ibatis.common.jdbc.ScriptRunner; +import org.apache.commons.dbcp.BasicDataSource; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.sql.Connection; + + +public class DBUtils { + public static BasicDataSource runMigrations(BasicDataSource dataSource) throws Exception { + Connection conn = dataSource.getConnection(); + ScriptRunner runner = new ScriptRunner(conn, false, true); + runner.runScript( + new BufferedReader( + new InputStreamReader(DBUtils.class.getResourceAsStream("/sql/cleanup.sql")))); + runner.runScript( + new BufferedReader( + new InputStreamReader(DBUtils.class.getResourceAsStream("/sql/deploy.sql")))); + conn.prepareStatement("SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));").execute(); + conn.close(); + return dataSource; + } +} diff --git a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java index a6b87d2f88..d433814e3b 100644 --- a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java +++ b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java @@ -16,13 +16,12 @@ package com.pinterest.teletraan.config; import com.fasterxml.jackson.annotation.JsonTypeName; -import com.pinterest.deployservice.common.DBUtils; import org.apache.commons.dbcp.BasicDataSource; @JsonTypeName("embedded") public class EmbeddedDataSourceFactory implements DataSourceFactory { public BasicDataSource build() throws Exception { - return DBUtils.setupDataSource(); + return new BasicDataSource(); } } From 9d6a62a4870885958f885ed99c3c3b6eb8f66da3 Mon Sep 17 00:00:00 2001 From: andrerodrigues Date: Tue, 19 Mar 2024 23:55:35 +0000 Subject: [PATCH 3/8] Update JUnit usage and clean up code --- deploy-service/common/pom.xml | 7 +- .../deployservice/db/DatabaseUtil.java | 304 +++++++++--------- .../pinterest/deployservice/db/DBDAOTest.java | 304 +++++++++--------- .../pinterest/deployservice/db/DBUtils.java | 60 +++- .../config/EmbeddedDataSourceFactory.java | 6 +- 5 files changed, 357 insertions(+), 324 deletions(-) diff --git a/deploy-service/common/pom.xml b/deploy-service/common/pom.xml index 46f88814d5..d9d0429e2c 100644 --- a/deploy-service/common/pom.xml +++ b/deploy-service/common/pom.xml @@ -54,9 +54,9 @@ slf4j-api - mysql - mysql-connector-java - 8.0.33 + com.mysql + mysql-connector-j + 8.3.0 commons-dbcp @@ -176,6 +176,7 @@ com.ibatis ibatis2-common 2.1.7.597 + test org.kohsuke diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java index c44fc52121..5d923e2188 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2016 Pinterest, Inc. + * Copyright (c) 2016-2024 Pinterest, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,169 +16,173 @@ package com.pinterest.deployservice.db; import com.pinterest.deployservice.bean.UpdateStatement; - +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.dbutils.DbUtils; import org.apache.commons.dbutils.QueryRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; - -/** - * Utility class to insertOrUpdate MySQL datasource. - */ +/** Utility class to insertOrUpdate MySQL datasource. */ public class DatabaseUtil { - public static final int MAX_WAIT_TIME_FOR_CONN_IN_MS = 200; - public static final String MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; - private static final Logger LOG = LoggerFactory.getLogger(DatabaseUtil.class); - - public static BasicDataSource createMysqlDataSource(String host, int port, - String user, String passwd, String poolSize) { - // autoReconnect by default is false - // TODO: queryTimeoutKillsConnection to true, "?queryTimeoutKillsConnection=true" - String url = String.format( - "jdbc:mysql://%s:%d/deploy?connectTimeout=5000&socketTimeout=3000&characterEncoding=UTF-8" - + "&connectionCollation=utf8mb4_general_ci", - host, port); - LOG.info("mtls is disabled --- host:{}, port:{}, user:{}", host, port, user); - return createDataSource(MYSQL_JDBC_DRIVER, url, user, passwd, poolSize, - MAX_WAIT_TIME_FOR_CONN_IN_MS); - } - - // local mysql source, for unit test only that depends on a local mysql db (that can be launched - // in docker container) - // example of command to launch local mysql container: - // common/src/main/java/com/pinterest/clusterservice/common/DBUtils.java:startMysqlContainer - public static BasicDataSource createLocalDataSource(String dbName, int port) { - String url = String.format("jdbc:mysql://0.0.0.0:%d/%s?useSSL=false", port, dbName); - return createDataSource( - MYSQL_JDBC_DRIVER, url, "root", "", "0:8:8:0", MAX_WAIT_TIME_FOR_CONN_IN_MS, null); - } - - public static BasicDataSource createMysqlDataSource(String host, int port, - String user, String passwd, String poolSize, - Map connectionProperties) { - // autoReconnect by default is false - // TODO: queryTimeoutKillsConnection to true, "?queryTimeoutKillsConnection=true" - String url = String.format( - "jdbc:mysql://%s:%d/deploy?connectTimeout=5000&socketTimeout=3000&characterEncoding=UTF-8" - + "&connectionCollation=utf8mb4_general_ci", - host, port); - LOG.info("mtls is enabled --- host:{}, port:{}, user:{}", host, port, user); - return createDataSource(MYSQL_JDBC_DRIVER, url, user, passwd, poolSize, - MAX_WAIT_TIME_FOR_CONN_IN_MS, connectionProperties); - } - - // Embedded mysql source, for unit test only - public static BasicDataSource createMXJDataSource(String dbName, String baseDir, int port) { - String - url = - String.format( - "jdbc:mysql:mxj://localhost:%d/%s?server" - + ".basedir=%s&createDatabaseIfNotExist=true&server.initialize-user=true", - port, dbName, baseDir); - return createDataSource(MYSQL_JDBC_DRIVER, url, "tester", "passwd", "0:8:8:0", - MAX_WAIT_TIME_FOR_CONN_IN_MS); - } - - /** - * Create a MySQL datasource. - * - * @param url the url of the DB. - * @param user the user name to connect to MySQL as. - * @param passwd the password for the corresponding MySQL user. - * @param poolSize the connection pool size string, in the format of - * initialSize:maxActive:maxIdle:minIdle. - * @param maxWaitInMillis the max wait time in milliseconds to get a connection from the pool. - * @return a BasicDataSource for the target MySQL instance. - */ - public static BasicDataSource createDataSource(String driverClassName, String url, - String user, String passwd, String poolSize, - int maxWaitInMillis) { - return createDataSource(driverClassName, url, user, passwd, poolSize, maxWaitInMillis, null); - } - - public static BasicDataSource createDataSource(String driverClassName, String url, - String user, String passwd, String poolSize, - int maxWaitInMillis, - Map connectionProperties) { - BasicDataSource dataSource = new BasicDataSource(); - dataSource.setDriverClassName(driverClassName); - dataSource.setUrl(url); - dataSource.setUsername(user); - dataSource.setPassword(passwd); - dataSource.setDefaultAutoCommit(true); - dataSource.setDefaultReadOnly(false); + public static final int MAX_WAIT_TIME_FOR_CONN_IN_MS = 200; + public static final String MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; + private static final Logger LOG = LoggerFactory.getLogger(DatabaseUtil.class); + + public static BasicDataSource createMysqlDataSource( + String host, int port, String user, String passwd, String poolSize) { + // autoReconnect by default is false + // TODO: queryTimeoutKillsConnection to true, "?queryTimeoutKillsConnection=true" + String url = + String.format( + "jdbc:mysql://%s:%d/deploy?connectTimeout=5000&socketTimeout=3000&characterEncoding=UTF-8" + + "&connectionCollation=utf8mb4_general_ci", + host, port); + LOG.info("mtls is disabled --- host:{}, port:{}, user:{}", host, port, user); + return createDataSource( + MYSQL_JDBC_DRIVER, url, user, passwd, poolSize, MAX_WAIT_TIME_FOR_CONN_IN_MS); + } - // poolSize parsing, the poolsize string passed in the following format - // initialSize:maxActive:maxIdle:minIdle - String[] sizeStrs = poolSize.split(":"); - dataSource.setInitialSize(Integer.parseInt(sizeStrs[0])); - dataSource.setMaxActive(Integer.parseInt(sizeStrs[1])); - dataSource.setMaxIdle(Integer.parseInt(sizeStrs[2])); - dataSource.setMinIdle(Integer.parseInt(sizeStrs[3])); + public static BasicDataSource createMysqlDataSource( + String host, + int port, + String user, + String passwd, + String poolSize, + Map connectionProperties) { + // autoReconnect by default is false + // TODO: queryTimeoutKillsConnection to true, "?queryTimeoutKillsConnection=true" + String url = + String.format( + "jdbc:mysql://%s:%d/deploy?connectTimeout=5000&socketTimeout=3000&characterEncoding=UTF-8" + + "&connectionCollation=utf8mb4_general_ci", + host, port); + LOG.info("mtls is enabled --- host:{}, port:{}, user:{}", host, port, user); + return createDataSource( + MYSQL_JDBC_DRIVER, + url, + user, + passwd, + poolSize, + MAX_WAIT_TIME_FOR_CONN_IN_MS, + connectionProperties); + } - dataSource.setValidationQuery("SELECT 1"); - dataSource.setTestOnBorrow(true); - dataSource.setTestOnReturn(false); - dataSource.setTestWhileIdle(true); - dataSource.setMinEvictableIdleTimeMillis(5 * 60 * 1000); - dataSource.setTimeBetweenEvictionRunsMillis(3 * 60 * 1000); - // dataSource.setNumTestsPerEvictionRun(3); - // max wait in milliseconds for a connection. - dataSource.setMaxWait(maxWaitInMillis); + // Embedded mysql source, for unit test only + public static BasicDataSource createLocalDataSource(String url) { + return createDataSource( + MYSQL_JDBC_DRIVER, url, "root", "", "0:8:8:0", MAX_WAIT_TIME_FOR_CONN_IN_MS, null); + } - if (connectionProperties != null) { - for (Map.Entry entry : connectionProperties.entrySet()) { - LOG.info( - String.format("Add connection properties %s=%s", entry.getKey(), entry.getValue())); - dataSource.addConnectionProperty( - entry.getKey(), - entry.getValue() - ); - } + /** + * Create a MySQL datasource. + * + * @param url the url of the DB. + * @param user the user name to connect to MySQL as. + * @param passwd the password for the corresponding MySQL user. + * @param poolSize the connection pool size string, in the format of + * initialSize:maxActive:maxIdle:minIdle. + * @param maxWaitInMillis the max wait time in milliseconds to get a connection from the pool. + * @return a BasicDataSource for the target MySQL instance. + */ + public static BasicDataSource createDataSource( + String driverClassName, + String url, + String user, + String passwd, + String poolSize, + int maxWaitInMillis) { + return createDataSource( + driverClassName, url, user, passwd, poolSize, maxWaitInMillis, null); } - // force connection pool initialization. - Connection conn = null; - try { - // Here not getting the connection from ThreadLocal no need to worry about that. - conn = dataSource.getConnection(); - LOG.info("mysql conn: {}", conn); - } catch (SQLException e) { - LOG.error( - String.format("Failed to get a db connection when creating DataSource, url = %s", url), - e); - } finally { - DbUtils.closeQuietly(conn); + public static BasicDataSource createDataSource( + String driverClassName, + String url, + String user, + String passwd, + String poolSize, + int maxWaitInMillis, + Map connectionProperties) { + BasicDataSource dataSource = new BasicDataSource(); + dataSource.setDriverClassName(driverClassName); + dataSource.setUrl(url); + dataSource.setUsername(user); + dataSource.setPassword(passwd); + dataSource.setDefaultAutoCommit(true); + dataSource.setDefaultReadOnly(false); + + // poolSize parsing, the poolsize string passed in the following format + // initialSize:maxActive:maxIdle:minIdle + String[] sizeStrs = poolSize.split(":"); + dataSource.setInitialSize(Integer.parseInt(sizeStrs[0])); + dataSource.setMaxActive(Integer.parseInt(sizeStrs[1])); + dataSource.setMaxIdle(Integer.parseInt(sizeStrs[2])); + dataSource.setMinIdle(Integer.parseInt(sizeStrs[3])); + + dataSource.setValidationQuery("SELECT 1"); + dataSource.setTestOnBorrow(true); + dataSource.setTestOnReturn(false); + dataSource.setTestWhileIdle(true); + dataSource.setMinEvictableIdleTimeMillis(5 * 60 * 1000); + dataSource.setTimeBetweenEvictionRunsMillis(3 * 60 * 1000); + // dataSource.setNumTestsPerEvictionRun(3); + // max wait in milliseconds for a connection. + dataSource.setMaxWait(maxWaitInMillis); + + if (connectionProperties != null) { + for (Map.Entry entry : connectionProperties.entrySet()) { + LOG.info( + String.format( + "Add connection properties %s=%s", + entry.getKey(), entry.getValue())); + dataSource.addConnectionProperty(entry.getKey(), entry.getValue()); + } + } + + // force connection pool initialization. + Connection conn = null; + try { + // Here not getting the connection from ThreadLocal no need to worry about that. + conn = dataSource.getConnection(); + LOG.info("mysql conn: {}", conn); + } catch (SQLException e) { + LOG.error( + String.format( + "Failed to get a db connection when creating DataSource, url = %s", + url), + e); + } finally { + DbUtils.closeQuietly(conn); + } + LOG.info("mysql dataSource: {}", dataSource); + return dataSource; } - LOG.info("mysql dataSource: {}", dataSource); - return dataSource; - } - public static void transactionalUpdate(BasicDataSource dataSource, - List updateStatements) throws Exception { - QueryRunner queryRunner = new QueryRunner(); - Connection connection = dataSource.getConnection(); - boolean autoStatus = connection.getAutoCommit(); - connection.setAutoCommit(false); - try { - for (UpdateStatement updateStatement : updateStatements) { - queryRunner - .update(connection, updateStatement.getStatement(), updateStatement.getValueArray()); - } - connection.commit(); - } catch (SQLException e) { - connection.rollback(); - throw e; - } finally { - connection.setAutoCommit(autoStatus); - DbUtils.closeQuietly(connection); + public static void transactionalUpdate( + BasicDataSource dataSource, List updateStatements) throws Exception { + QueryRunner queryRunner = new QueryRunner(); + Connection connection = dataSource.getConnection(); + boolean autoStatus = connection.getAutoCommit(); + connection.setAutoCommit(false); + try { + for (UpdateStatement updateStatement : updateStatements) { + queryRunner.update( + connection, + updateStatement.getStatement(), + updateStatement.getValueArray()); + } + connection.commit(); + } catch (SQLException e) { + connection.rollback(); + throw e; + } finally { + connection.setAutoCommit(autoStatus); + DbUtils.closeQuietly(connection); + } } - } } diff --git a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java index 4a005294d0..d4241deae4 100644 --- a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java +++ b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2016 Pinterest, Inc. + * Copyright (c) 2016-2024 Pinterest, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,12 @@ */ package com.pinterest.deployservice.db; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.pinterest.deployservice.bean.AcceptanceStatus; import com.pinterest.deployservice.bean.AcceptanceType; import com.pinterest.deployservice.bean.AgentBean; @@ -69,15 +75,6 @@ import com.pinterest.deployservice.dao.TokenRolesDAO; import com.pinterest.deployservice.dao.UserRolesDAO; import com.pinterest.deployservice.dao.UtilDAO; -import org.apache.commons.dbcp.BasicDataSource; -import org.apache.commons.lang.builder.EqualsBuilder; -import org.joda.time.Interval; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.testcontainers.containers.MySQLContainer; - import java.sql.Connection; import java.util.ArrayList; import java.util.Arrays; @@ -86,13 +83,13 @@ import java.util.HashSet; import java.util.List; import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - +import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.joda.time.Interval; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MySQLContainer; public class DBDAOTest { private static BuildDAO buildDAO; @@ -113,17 +110,12 @@ public class DBDAOTest { private static ScheduleDAO scheduleDAO; private static UtilDAO utilDAO; - @ClassRule - public static final MySQLContainer mysql = new MySQLContainer().withDatabaseName("deploy"); - + public static final MySQLContainer mysql = DBUtils.getContainer(); - @BeforeClass + @BeforeAll public static void setUpClass() throws Exception { - BasicDataSource DATASOURCE = new BasicDataSource(); - DATASOURCE.setDriverClassName("com.mysql.jdbc.Driver"); - DATASOURCE.setUrl(mysql.getJdbcUrl()); - DATASOURCE.setUsername(mysql.getUsername()); - DATASOURCE.setPassword(mysql.getPassword()); + mysql.start(); + BasicDataSource DATASOURCE = DatabaseUtil.createLocalDataSource(mysql.getJdbcUrl()); DBUtils.runMigrations(DATASOURCE); buildDAO = new DBBuildDAOImpl(DATASOURCE); @@ -145,18 +137,19 @@ public static void setUpClass() throws Exception { utilDAO = new DBUtilDAOImpl(DATASOURCE); } + @AfterAll + public static void tearDownClass() throws Exception { + mysql.stop(); + } + @Test public void testDeploymentQueries() throws Exception { long now = System.currentTimeMillis(); - BuildBean buildBean1 = - genDefaultBuildInfoBean("bbb-1", "s-1", "ccc-1", "r-1", now); - BuildBean buildBean2 = - genDefaultBuildInfoBean("bbb-2", "s-1", "ccc-1", "r-1", now + 1000); - BuildBean buildBean3 = - genDefaultBuildInfoBean("bbb-3", "s-1", "ccc-1", "r-1", now + 2000); - BuildBean buildBean4 = - genDefaultBuildInfoBean("bbb-4", "s-2", "ccc-2", "r-1", now + 3000); + BuildBean buildBean1 = genDefaultBuildInfoBean("bbb-1", "s-1", "ccc-1", "r-1", now); + BuildBean buildBean2 = genDefaultBuildInfoBean("bbb-2", "s-1", "ccc-1", "r-1", now + 1000); + BuildBean buildBean3 = genDefaultBuildInfoBean("bbb-3", "s-1", "ccc-1", "r-1", now + 2000); + BuildBean buildBean4 = genDefaultBuildInfoBean("bbb-4", "s-2", "ccc-2", "r-1", now + 3000); buildDAO.insert(buildBean1); buildDAO.insert(buildBean2); @@ -164,21 +157,20 @@ public void testDeploymentQueries() throws Exception { buildDAO.insert(buildBean4); DeployBean deployBean1 = - genDefaultDeployBean("d-1", "env-1", "bbb-1", now, DeployState.SUCCEEDED); + genDefaultDeployBean("d-1", "env-1", "bbb-1", now, DeployState.SUCCEEDED); DeployBean deployBean2 = - genDefaultDeployBean("d-2", "env-1", "bbb-1", now + 1000, DeployState.SUCCEEDED); + genDefaultDeployBean("d-2", "env-1", "bbb-1", now + 1000, DeployState.SUCCEEDED); DeployBean deployBean3 = - genDefaultDeployBean("d-3", "env-1", "bbb-1", now + 2000, DeployState.RUNNING); + genDefaultDeployBean("d-3", "env-1", "bbb-1", now + 2000, DeployState.RUNNING); DeployBean deployBean4 = - genDefaultDeployBean("d-4", "env-2", "bbb-2", now, DeployState.FAILING); + genDefaultDeployBean("d-4", "env-2", "bbb-2", now, DeployState.FAILING); // just so we have the build - BuildBean buildBeanx = - genDefaultBuildInfoBean("d-x", "s-1", "ccc-x", "r-1", now); + BuildBean buildBeanx = genDefaultBuildInfoBean("d-x", "s-1", "ccc-x", "r-1", now); buildDAO.insert(buildBeanx); DeployBean deployBean5 = - genDefaultDeployBean("d-5", "env-3", "bcc-x", now, DeployState.SUCCEEDING); + genDefaultDeployBean("d-5", "env-3", "bcc-x", now, DeployState.SUCCEEDING); DeployBean deployBean6 = - genDefaultDeployBean("d-6", "env-2", "bbb-4", now, DeployState.SUCCEEDED); + genDefaultDeployBean("d-6", "env-2", "bbb-4", now, DeployState.SUCCEEDED); deployDAO.insert(deployBean1); deployDAO.insert(deployBean2); @@ -285,9 +277,8 @@ public void testDeploymentQueries() throws Exception { deployBean5.setAcc_status(AcceptanceStatus.ACCEPTED); deployDAO.update("d-5", deployBean5); - List - beans = - deployDAO.getAcceptedDeploys("env-3", new Interval(0, Long.MAX_VALUE), 100); + List beans = + deployDAO.getAcceptedDeploys("env-3", new Interval(0, Long.MAX_VALUE), 100); assertEquals(beans.size(), 1); assertEquals(beans.get(0).getDeploy_id(), "d-5"); @@ -310,14 +301,10 @@ public void testDeploymentQueries() throws Exception { @Test public void testBuildDAO() throws Exception { long now = System.currentTimeMillis(); - BuildBean buildBean1 = - genDefaultBuildInfoBean("b-1", "sss-1", "c-1", "r-1", now); - BuildBean buildBean2 = - genDefaultBuildInfoBean("b-2", "sss-1", "c-1", "r-1", now + 1000); - BuildBean buildBean22 = - genDefaultBuildInfoBean("b-22", "sss-2", "c-1", "r-1", now + 1000); - BuildBean buildBean3 = - genDefaultBuildInfoBean("b-3", "sss-1", "c-1", "r-1", now + 2000); + BuildBean buildBean1 = genDefaultBuildInfoBean("b-1", "sss-1", "c-1", "r-1", now); + BuildBean buildBean2 = genDefaultBuildInfoBean("b-2", "sss-1", "c-1", "r-1", now + 1000); + BuildBean buildBean22 = genDefaultBuildInfoBean("b-22", "sss-2", "c-1", "r-1", now + 1000); + BuildBean buildBean3 = genDefaultBuildInfoBean("b-3", "sss-1", "c-1", "r-1", now + 2000); buildDAO.insert(buildBean1); buildDAO.insert(buildBean2); @@ -330,9 +317,8 @@ public void testBuildDAO() throws Exception { assertEquals(buildDAO.getByCommit7("c-1", "sss-2", 1, 10).size(), 1); assertEquals(buildDAO.getBuildNames("sss-", 1, 100).size(), 2); - List - buildBeans = - buildDAO.getAcceptedBuilds("sss-1", null, new Interval(now, Long.MAX_VALUE), 100); + List buildBeans = + buildDAO.getAcceptedBuilds("sss-1", null, new Interval(now, Long.MAX_VALUE), 100); assertEquals(buildBeans.size(), 2); BuildBean bean1 = buildBeans.get(0); assertEquals(bean1.getBuild_id(), "b-3"); @@ -351,9 +337,8 @@ public void testBuildDAO() throws Exception { List buildBeans4 = buildDAO.getByName("sss-1", "branch-1", 1, 2); assertEquals(buildBeans4.size(), 2); - List - allBuildBeans = - buildDAO.getBuildsFromIds(Arrays.asList("b-1", "b-2", "b-22")); + List allBuildBeans = + buildDAO.getBuildsFromIds(Arrays.asList("b-1", "b-2", "b-22")); assertEquals(3, allBuildBeans.size()); allBuildBeans = buildDAO.getBuildsFromIds(Arrays.asList("b-1", "b-2", "Not There")); @@ -374,14 +359,12 @@ public void testBuildDAO() throws Exception { @Test public void testAgentUpdate() throws Exception { - AgentBean - agentBean1 = - genDefaultAgentBean("h1", "id-1", "e-1", "d-1", DeployStage.PRE_DOWNLOAD); + AgentBean agentBean1 = + genDefaultAgentBean("h1", "id-1", "e-1", "d-1", DeployStage.PRE_DOWNLOAD); agentDAO.insertOrUpdate(agentBean1); - AgentBean - updateBean1 = - genDefaultAgentBean("h1", "id-1", "e-1", "d-1", DeployStage.POST_DOWNLOAD); + AgentBean updateBean1 = + genDefaultAgentBean("h1", "id-1", "e-1", "d-1", DeployStage.POST_DOWNLOAD); updateBean1.setFirst_deploy_time(10L); agentDAO.insertOrUpdate(updateBean1); @@ -399,15 +382,12 @@ public void testAgentUpdate() throws Exception { @Test public void testAgentUpdateMultiple() throws Exception { - AgentBean - agentBean1 = - genDefaultAgentBean("h5", "id-5", "e-2", "d-1", DeployStage.PRE_DOWNLOAD); - AgentBean - agentBean2 = - genDefaultAgentBean("h6", "id-6", "e-2", "d-1", DeployStage.PRE_DOWNLOAD); - AgentBean - agentBean3 = - genDefaultAgentBean("h7", "id-7", "e-2", "d-1", DeployStage.PRE_DOWNLOAD); + AgentBean agentBean1 = + genDefaultAgentBean("h5", "id-5", "e-2", "d-1", DeployStage.PRE_DOWNLOAD); + AgentBean agentBean2 = + genDefaultAgentBean("h6", "id-6", "e-2", "d-1", DeployStage.PRE_DOWNLOAD); + AgentBean agentBean3 = + genDefaultAgentBean("h7", "id-7", "e-2", "d-1", DeployStage.PRE_DOWNLOAD); agentDAO.insertOrUpdate(agentBean1); agentDAO.insertOrUpdate(agentBean2); @@ -431,15 +411,13 @@ public void testAgentUpdateMultiple() throws Exception { @Test public void testFirstDeployCount() throws Exception { - AgentBean - agentBean1 = - genDefaultAgentBean("h12", "id-123", "e-12", "d-12", DeployStage.POST_RESTART); + AgentBean agentBean1 = + genDefaultAgentBean("h12", "id-123", "e-12", "d-12", DeployStage.POST_RESTART); agentBean1.setFirst_deploy(true); agentBean1.setStatus(AgentStatus.ABORTED_BY_SERVICE); - AgentBean - agentBean2 = - genDefaultAgentBean("h22", "id-124", "e-12", "d-12", DeployStage.POST_RESTART); + AgentBean agentBean2 = + genDefaultAgentBean("h22", "id-124", "e-12", "d-12", DeployStage.POST_RESTART); agentBean2.setFirst_deploy(true); agentDAO.insertOrUpdate(agentBean1); @@ -453,14 +431,14 @@ public void testFirstDeployCount() throws Exception { @Test public void testAgentQueries() throws Exception { - AgentBean agentBean1 = genDefaultAgentBean( - "h-1", "id-1", "e-1", "d-1", DeployStage.POST_RESTART); - AgentBean agentBean11 = genDefaultAgentBean( - "h-1", "id-1", "e-2", "d-1", DeployStage.SERVING_BUILD); - AgentBean agentBean2 = genDefaultAgentBean( - "h-2", "id-3", "e-1", "d-2", DeployStage.RESTARTING); - AgentBean agentBean3 = genDefaultAgentBean( - "h-3", "id-4", "e-1", "d-1", DeployStage.SERVING_BUILD); + AgentBean agentBean1 = + genDefaultAgentBean("h-1", "id-1", "e-1", "d-1", DeployStage.POST_RESTART); + AgentBean agentBean11 = + genDefaultAgentBean("h-1", "id-1", "e-2", "d-1", DeployStage.SERVING_BUILD); + AgentBean agentBean2 = + genDefaultAgentBean("h-2", "id-3", "e-1", "d-2", DeployStage.RESTARTING); + AgentBean agentBean3 = + genDefaultAgentBean("h-3", "id-4", "e-1", "d-1", DeployStage.SERVING_BUILD); agentBean3.setFirst_deploy_time(System.currentTimeMillis()); agentDAO.insertOrUpdate(agentBean1); @@ -584,18 +562,17 @@ public void testEnvDAO() throws Exception { // Added 2 hosts to group1 and group2 Set groups = new HashSet<>(Arrays.asList("group1", "group2")); - hostDAO - .insertOrUpdate("host-1", "1.1.1.1", "id-123434", HostState.ACTIVE.toString(), groups, "test"); - hostDAO - .insertOrUpdate("host-2", "1.1.1.2", "id-123435", HostState.TERMINATING.toString(), - groups, "test"); - hostDAO - .insertOrUpdate("host-2", "1.1.1.2", "id-123435", HostState.ACTIVE.toString(), groups, "test"); + hostDAO.insertOrUpdate( + "host-1", "1.1.1.1", "id-123434", HostState.ACTIVE.toString(), groups, "test"); + hostDAO.insertOrUpdate( + "host-2", "1.1.1.2", "id-123435", HostState.TERMINATING.toString(), groups, "test"); + hostDAO.insertOrUpdate( + "host-2", "1.1.1.2", "id-123435", HostState.ACTIVE.toString(), groups, "test"); List hostBeans = hostDAO.getHostsByHostId("id-123435"); assertEquals(hostBeans.get(0).getState(), HostState.TERMINATING); // Total capacity for env-1 should be 2, host-1(group1), host-2(group2) and one missing - // host1 + // host1 assertEquals(environDAO.getOverrideHosts("env-1", "s-1", "prod").size(), 0); assertEquals(environDAO.countTotalCapacity("env-1", "s-1", "prod"), 2); assertEquals(environDAO.getMissingHosts("env-1").size(), 1); @@ -643,10 +620,13 @@ public void testEnvDAO() throws Exception { @Test public void testHostDAO() throws Exception { Set groups = new HashSet<>(Arrays.asList("group1", "group2")); - hostDAO.insertOrUpdate("host-1", "1.1.1.1", "id-1", HostState.ACTIVE.toString(), groups, "test"); + hostDAO.insertOrUpdate( + "host-1", "1.1.1.1", "id-1", HostState.ACTIVE.toString(), groups, "test"); groups = new HashSet<>(Arrays.asList("group1")); - hostDAO.insertOrUpdate("host-2", "1.1.1.2", "id-2", HostState.ACTIVE.toString(), groups, "test"); - hostDAO.insertOrUpdate("host-3", "1.1.1.3", "id-3", HostState.ACTIVE.toString(), groups, "test"); + hostDAO.insertOrUpdate( + "host-2", "1.1.1.2", "id-2", HostState.ACTIVE.toString(), groups, "test"); + hostDAO.insertOrUpdate( + "host-3", "1.1.1.3", "id-3", HostState.ACTIVE.toString(), groups, "test"); /* host-1 : group1, group2 host-2 : group1 @@ -674,9 +654,10 @@ public void testHostDAO() throws Exception { hostDAO.deleteById("id-2"); // test host transactional delete - hostDAO.insertOrUpdate("host-1", "1.1.1.1", "id-1", HostState.ACTIVE.toString(), groups, "test"); - AgentBean agentBean = genDefaultAgentBean( - "host-1", "id-1", "e-1", "d-1", DeployStage.SERVING_BUILD); + hostDAO.insertOrUpdate( + "host-1", "1.1.1.1", "id-1", HostState.ACTIVE.toString(), groups, "test"); + AgentBean agentBean = + genDefaultAgentBean("host-1", "id-1", "e-1", "d-1", DeployStage.SERVING_BUILD); agentDAO.insertOrUpdate(agentBean); AgentErrorBean agentErrorBean = new AgentErrorBean(); agentErrorBean.setHost_name("host-1"); @@ -698,8 +679,8 @@ public void testHostDAO() throws Exception { assertEquals(environDAO.getMissingHosts("e-3").size(), 1); Set groups2 = new HashSet<>(Arrays.asList("new_group")); - hostDAO - .insertOrUpdate("host-3", "3.3.3.3", "id-3", HostState.TERMINATING.toString(), groups2, "test"); + hostDAO.insertOrUpdate( + "host-3", "3.3.3.3", "id-3", HostState.TERMINATING.toString(), groups2, "test"); assertEquals(environDAO.getMissingHosts("e-3").size(), 0); Collection hostBean3 = hostDAO.getByEnvIdAndHostName("e-3", "host-3"); @@ -726,9 +707,17 @@ public void testHostDAO() throws Exception { assertEquals(hostBeans3.get(0).getHost_name(), "i-9"); HashSet groups9 = new HashSet<>(Arrays.asList("test_dup")); - hostDAO.insertOrUpdate("h-8", "9.9.9.9", "i-8", HostState.TERMINATING.toString(), groups9, "test"); - hostDAO.insertOrUpdate("h-9", "9.9.9.9", "i-9", HostState.PENDING_TERMINATE.toString(), groups9, "test"); - hostDAO.insertOrUpdate("h-10", "9.9.9.9", "i-10", HostState.PENDING_TERMINATE_NO_REPLACE.toString(), groups9, "test"); + hostDAO.insertOrUpdate( + "h-8", "9.9.9.9", "i-8", HostState.TERMINATING.toString(), groups9, "test"); + hostDAO.insertOrUpdate( + "h-9", "9.9.9.9", "i-9", HostState.PENDING_TERMINATE.toString(), groups9, "test"); + hostDAO.insertOrUpdate( + "h-10", + "9.9.9.9", + "i-10", + HostState.PENDING_TERMINATE_NO_REPLACE.toString(), + groups9, + "test"); List hostBeans4 = hostDAO.getHosts("h-9"); assertEquals(hostBeans4.size(), 1); @@ -738,10 +727,14 @@ public void testHostDAO() throws Exception { List hostBeans5 = hostDAO.getTerminatingHosts(); assertEquals(4, hostBeans5.size()); - // If state is PENDING_TERMINATE, PENDING_TERMINATE_NO_REPLACE or TERMINATING, cannot overwrite its state - hostDAO.insertOrUpdate("h-8", "9.9.9.8", "i-8", HostState.PROVISIONED.toString(), groups9, "test"); - hostDAO.insertOrUpdate("h-9", "9.9.9.8", "i-9", HostState.PROVISIONED.toString(), groups9, "test"); - hostDAO.insertOrUpdate("h-10", "9.9.9.8", "i-10", HostState.PROVISIONED.toString(), groups9, "test"); + // If state is PENDING_TERMINATE, PENDING_TERMINATE_NO_REPLACE or TERMINATING, cannot + // overwrite its state + hostDAO.insertOrUpdate( + "h-8", "9.9.9.8", "i-8", HostState.PROVISIONED.toString(), groups9, "test"); + hostDAO.insertOrUpdate( + "h-9", "9.9.9.8", "i-9", HostState.PROVISIONED.toString(), groups9, "test"); + hostDAO.insertOrUpdate( + "h-10", "9.9.9.8", "i-10", HostState.PROVISIONED.toString(), groups9, "test"); hostBeans5 = hostDAO.getTerminatingHosts(); assertEquals(4, hostBeans5.size()); @@ -769,24 +762,28 @@ public void testHostDAO() throws Exception { hostBean7.setState(HostState.ACTIVE); hostDAO.insert(hostBean7); - AgentBean - agentBean1 = - genDefaultAgentBean("i-11", "i-11", "e-1", "d-1", DeployStage.RESTARTING); + AgentBean agentBean1 = + genDefaultAgentBean("i-11", "i-11", "e-1", "d-1", DeployStage.RESTARTING); agentBean1.setStatus(AgentStatus.AGENT_FAILED); agentDAO.insertOrUpdate(agentBean1); - HostState[] nonRetirableStates = { HostState.TERMINATING, HostState.PENDING_TERMINATE_NO_REPLACE, - HostState.PENDING_TERMINATE }; + HostState[] nonRetirableStates = { + HostState.TERMINATING, + HostState.PENDING_TERMINATE_NO_REPLACE, + HostState.PENDING_TERMINATE + }; - for (HostState state: nonRetirableStates) { + for (HostState state : nonRetirableStates) { hostBean7.setCan_retire(1); hostBean7.setState(state); hostDAO.updateHostById("i-13", hostBean7); - Collection retiredHostBeanIds = hostDAO.getToBeRetiredHostIdsByGroup("retire-group"); + Collection retiredHostBeanIds = + hostDAO.getToBeRetiredHostIdsByGroup("retire-group"); assertEquals(2, retiredHostBeanIds.size()); - Collection retiredAndFailedHostIds = hostDAO.getToBeRetiredAndFailedHostIdsByGroup("retire-group"); + Collection retiredAndFailedHostIds = + hostDAO.getToBeRetiredAndFailedHostIdsByGroup("retire-group"); assertEquals(1, retiredAndFailedHostIds.size()); Collection failedHostIds = hostDAO.getFailedHostIdsByGroup("retire-group"); @@ -908,9 +905,8 @@ public void testUserRolesDAO() throws Exception { bean.setResource_type(Resource.Type.ENV); bean.setRole(Role.ADMIN); userRolesDAO.insert(bean); - UserRolesBean - bean2 = - userRolesDAO.getByNameAndResource("test", "envTest", Resource.Type.ENV); + UserRolesBean bean2 = + userRolesDAO.getByNameAndResource("test", "envTest", Resource.Type.ENV); assertEquals(bean2.getRole(), Role.ADMIN); } @@ -922,9 +918,8 @@ public void testGroupRolesDAO() throws Exception { bean.setResource_type(Resource.Type.ENV); bean.setRole(Role.ADMIN); groupRolesDAO.insert(bean); - GroupRolesBean - bean2 = - groupRolesDAO.getByNameAndResource("group", "123", Resource.Type.ENV); + GroupRolesBean bean2 = + groupRolesDAO.getByNameAndResource("group", "123", Resource.Type.ENV); assertEquals(bean2.getRole(), Role.ADMIN); } @@ -938,13 +933,11 @@ public void testTokenRolesDAO() throws Exception { bean.setRole(Role.ADMIN); bean.setExpire_date(System.currentTimeMillis()); tokenRolesDAO.insert(bean); - TokenRolesBean - bean2 = - tokenRolesDAO.getByNameAndResource("test", "envTest", Resource.Type.ENV); + TokenRolesBean bean2 = + tokenRolesDAO.getByNameAndResource("test", "envTest", Resource.Type.ENV); assertEquals(bean2.getRole(), Role.ADMIN); } - @Test public void testConfigHistoryDAO() throws Exception { ConfigHistoryBean bean = new ConfigHistoryBean(); @@ -973,8 +966,13 @@ public void testConfigHistoryDAO() throws Exception { @Test public void testTagDAO() throws Exception { - TagBean tag = genTagBean(TagValue.BAD_BUILD, "TestEnv", "BUILD", - genDefaultBuildInfoBean("b-3", "sss-1", "c-1", "r-1", System.currentTimeMillis())); + TagBean tag = + genTagBean( + TagValue.BAD_BUILD, + "TestEnv", + "BUILD", + genDefaultBuildInfoBean( + "b-3", "sss-1", "c-1", "r-1", System.currentTimeMillis())); tagDAO.insert(tag); TagBean tag2 = tagDAO.getById(tag.getId()); assertNotNull(tag2); @@ -995,20 +993,19 @@ public void testTagDAO() throws Exception { targetList = tagDAO.getByTargetIdAndType(tag.getTarget_id(), TagTargetType.BUILD); assertEquals(0, targetList.size()); - tagDAO - .insert(genTagBean(TagValue.BAD_BUILD, "env1", "BUILD", new HashMap())); - tagDAO - .insert(genTagBean(TagValue.BAD_BUILD, "env1", "BUILD", new HashMap())); - tagDAO - .insert(genTagBean(TagValue.BAD_BUILD, "env1", "BUILD", new HashMap())); - tagDAO - .insert(genTagBean(TagValue.BAD_BUILD, "env1", "BUILD", new HashMap())); + tagDAO.insert( + genTagBean(TagValue.BAD_BUILD, "env1", "BUILD", new HashMap())); + tagDAO.insert( + genTagBean(TagValue.BAD_BUILD, "env1", "BUILD", new HashMap())); + tagDAO.insert( + genTagBean(TagValue.BAD_BUILD, "env1", "BUILD", new HashMap())); + tagDAO.insert( + genTagBean(TagValue.BAD_BUILD, "env1", "BUILD", new HashMap())); assertEquals(4, tagDAO.getByValue(TagValue.BAD_BUILD).size()); assertEquals(0, tagDAO.getByValue(TagValue.GOOD_BUILD).size()); } - @Test public void testScheduleDAO() throws Exception { Long time = System.currentTimeMillis(); @@ -1041,9 +1038,8 @@ public void testScheduleDAO() throws Exception { assertEquals(updatedBean.getCurrent_session(), (Integer) 1); assertEquals(updatedBean.getState(), ScheduleState.RUNNING); assertEquals(updatedBean.getHost_numbers(), "50,60,500"); - } - + @Test public void testUtilDAO() throws Exception { StringBuilder lockNameBuilder = new StringBuilder(); @@ -1056,8 +1052,8 @@ public void testUtilDAO() throws Exception { } } - private EnvironBean genDefaultEnvBean(String envId, String envName, String envStage, - String deployId) { + private EnvironBean genDefaultEnvBean( + String envId, String envName, String envStage, String deployId) { EnvironBean envBean = new EnvironBean(); envBean.setEnv_id(envId); envBean.setEnv_name(envName); @@ -1067,7 +1063,7 @@ private EnvironBean genDefaultEnvBean(String envId, String envName, String envSt envBean.setPriority(DeployPriority.NORMAL); envBean.setStuck_th(100); - //To keep the precision, the default success_th value should be 10000 in DB. + // To keep the precision, the default success_th value should be 10000 in DB. envBean.setSuccess_th(10000); envBean.setDescription("foo"); envBean.setDeploy_id(deployId); @@ -1090,8 +1086,8 @@ private EnvironBean genDefaultEnvBean(String envId, String envName, String envSt return envBean; } - private DeployBean genDefaultDeployBean(String id, String envId, String buildId, - long startDate, DeployState state) { + private DeployBean genDefaultDeployBean( + String id, String envId, String buildId, long startDate, DeployState state) { DeployBean deployBean = new DeployBean(); deployBean.setDeploy_id(id); deployBean.setEnv_id(envId); @@ -1119,8 +1115,8 @@ private RatingBean genDefaultRatingsBean(String id, String author, long timestam return ratingBean; } - private BuildBean genDefaultBuildInfoBean(String id, String buildName, - String commitId, String repoUrl, long buildDate) { + private BuildBean genDefaultBuildInfoBean( + String id, String buildName, String commitId, String repoUrl, long buildDate) { BuildBean buildBean = new BuildBean(); buildBean.setBuild_id(id); buildBean.setBuild_name(buildName); @@ -1135,8 +1131,12 @@ private BuildBean genDefaultBuildInfoBean(String id, String buildName, return buildBean; } - private AgentBean genDefaultAgentBean(String hostName, String hostId, String envId, - String deployId, DeployStage deployStage) { + private AgentBean genDefaultAgentBean( + String hostName, + String hostId, + String envId, + String deployId, + DeployStage deployStage) { AgentBean agentBean = new AgentBean(); agentBean.setHost_name(hostName); agentBean.setHost_id(hostId); @@ -1165,7 +1165,7 @@ private DataBean genDefaultDataBean(String id, String data) { } private TagBean genTagBean(TagValue val, String target_id, String target_type, Object meta_info) - throws Exception { + throws Exception { TagBean bean = new TagBean(); bean.setId(CommonUtils.getBase64UUID()); bean.setCreated_date(System.currentTimeMillis()); diff --git a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java index e97d8905bc..0f62a71894 100644 --- a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java +++ b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java @@ -1,25 +1,53 @@ +/** + * Copyright (c) 2024 Pinterest, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.pinterest.deployservice.db; import com.ibatis.common.jdbc.ScriptRunner; -import org.apache.commons.dbcp.BasicDataSource; - import java.io.BufferedReader; import java.io.InputStreamReader; import java.sql.Connection; - +import org.apache.commons.dbcp.BasicDataSource; +import org.testcontainers.containers.MySQLContainer; public class DBUtils { - public static BasicDataSource runMigrations(BasicDataSource dataSource) throws Exception { - Connection conn = dataSource.getConnection(); - ScriptRunner runner = new ScriptRunner(conn, false, true); - runner.runScript( - new BufferedReader( - new InputStreamReader(DBUtils.class.getResourceAsStream("/sql/cleanup.sql")))); - runner.runScript( - new BufferedReader( - new InputStreamReader(DBUtils.class.getResourceAsStream("/sql/deploy.sql")))); - conn.prepareStatement("SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));").execute(); - conn.close(); - return dataSource; - } + + public static String DATABASE_NAME = "deploy"; + public static String DATABASE_USER = "root"; + public static String DATABASE_PASSWORD = ""; + + public static MySQLContainer getContainer() { + return new MySQLContainer<>() + .withDatabaseName(DATABASE_NAME) + .withUsername(DATABASE_USER) + .withPassword(DATABASE_PASSWORD); + } + + public static void runMigrations(BasicDataSource dataSource) throws Exception { + Connection conn = dataSource.getConnection(); + ScriptRunner runner = new ScriptRunner(conn, false, true); + runner.runScript( + new BufferedReader( + new InputStreamReader( + DBUtils.class.getResourceAsStream("/sql/cleanup.sql")))); + runner.runScript( + new BufferedReader( + new InputStreamReader( + DBUtils.class.getResourceAsStream("/sql/deploy.sql")))); + conn.prepareStatement("SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));") + .execute(); + conn.close(); + } } diff --git a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java index d433814e3b..4225a3c47d 100644 --- a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java +++ b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java @@ -1,12 +1,12 @@ /** - * Copyright 2016 Pinterest, Inc. + * Copyright (c) 2016-2024 Pinterest, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From b49650c1bc56a3f6b87f22ed0cc9fab07d3c06b8 Mon Sep 17 00:00:00 2001 From: andrerodrigues Date: Thu, 21 Mar 2024 20:05:01 +0000 Subject: [PATCH 4/8] Add H2 Database to the EmbeddedDataSourceFactory --- deploy-service/teletraanservice/pom.xml | 5 +++++ .../teletraan/config/EmbeddedDataSourceFactory.java | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/deploy-service/teletraanservice/pom.xml b/deploy-service/teletraanservice/pom.xml index 59c37e572e..7127dc8c70 100644 --- a/deploy-service/teletraanservice/pom.xml +++ b/deploy-service/teletraanservice/pom.xml @@ -71,6 +71,11 @@ ibatis2-common 2.1.7.597 + + com.h2database + h2 + runtime + org.mockito mockito-core diff --git a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java index 4225a3c47d..0bcc094053 100644 --- a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java +++ b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java @@ -16,12 +16,13 @@ package com.pinterest.teletraan.config; import com.fasterxml.jackson.annotation.JsonTypeName; +import com.pinterest.deployservice.db.DatabaseUtil; import org.apache.commons.dbcp.BasicDataSource; @JsonTypeName("embedded") public class EmbeddedDataSourceFactory implements DataSourceFactory { - public BasicDataSource build() throws Exception { - return new BasicDataSource(); + return DatabaseUtil.createDataSource( + "org.h2.Driver", "sa", "", "jdbc:h2:mem:deploy", "0:8:8:0", 200); } } From 02dd50ec2af1c369352a1c5c20a7a631779ca924 Mon Sep 17 00:00:00 2001 From: andrerodrigues Date: Fri, 22 Mar 2024 00:24:25 +0000 Subject: [PATCH 5/8] Add back MXJ for EmbeddedDataSourceFactory but commented --- deploy-service/common/pom.xml | 5 ++ .../deployservice/db/DatabaseUtil.java | 10 ++++ deploy-service/teletraanservice/pom.xml | 10 ++-- .../config/EmbeddedDataSourceFactory.java | 53 ++++++++++++++++++- 4 files changed, 71 insertions(+), 7 deletions(-) diff --git a/deploy-service/common/pom.xml b/deploy-service/common/pom.xml index d9d0429e2c..c02026ac0e 100644 --- a/deploy-service/common/pom.xml +++ b/deploy-service/common/pom.xml @@ -154,6 +154,11 @@ 1.10.8 test + + mysql + mysql-connector-mxj + 5.0.12 + org.testcontainers testcontainers diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java index 5d923e2188..3145f89513 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java @@ -73,6 +73,16 @@ public static BasicDataSource createMysqlDataSource( } // Embedded mysql source, for unit test only + public static BasicDataSource createMXJDataSource(String dbName, String baseDir, int port) { + String + url = + String.format( + "jdbc:mysql:mxj://localhost:%d/%s?server" + + ".basedir=%s&createDatabaseIfNotExist=true&server.initialize-user=true", + port, dbName, baseDir); + return createDataSource(MYSQL_JDBC_DRIVER, url, "tester", "passwd", "0:8:8:0", + MAX_WAIT_TIME_FOR_CONN_IN_MS); + } public static BasicDataSource createLocalDataSource(String url) { return createDataSource( MYSQL_JDBC_DRIVER, url, "root", "", "0:8:8:0", MAX_WAIT_TIME_FOR_CONN_IN_MS, null); diff --git a/deploy-service/teletraanservice/pom.xml b/deploy-service/teletraanservice/pom.xml index 7127dc8c70..3e41f7871b 100644 --- a/deploy-service/teletraanservice/pom.xml +++ b/deploy-service/teletraanservice/pom.xml @@ -66,16 +66,16 @@ quartz + + mysql + mysql-connector-mxj + 5.0.12 + com.ibatis ibatis2-common 2.1.7.597 - - com.h2database - h2 - runtime - org.mockito mockito-core diff --git a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java index 0bcc094053..ed9a32c418 100644 --- a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java +++ b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java @@ -16,13 +16,62 @@ package com.pinterest.teletraan.config; import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.commons.dbcp.BasicDataSource; + +/* +import com.fasterxml.jackson.annotation.JsonProperty; import com.pinterest.deployservice.db.DatabaseUtil; import org.apache.commons.dbcp.BasicDataSource; +import com.ibatis.common.jdbc.ScriptRunner; +import com.mysql.management.driverlaunched.ServerLauncherSocketFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.sql.Connection; +*/ + @JsonTypeName("embedded") public class EmbeddedDataSourceFactory implements DataSourceFactory { + /* + * + * Uncomment this block and the imports if you want to use embedded mysql + * + private final static String DEFAULT_BASE_DIR = "/tmp/teletraan/db"; + private final static String DEFAULT_DB_NAME = "deploy"; + private final static int DEFAULT_PORT = 3305; + + @JsonProperty + private String workDir = DEFAULT_BASE_DIR; + + public String getWorkDir() { + return workDir; + } + + public void setWorkDir(String workDir) { + this.workDir = workDir; + } + public BasicDataSource build() throws Exception { - return DatabaseUtil.createDataSource( - "org.h2.Driver", "sa", "", "jdbc:h2:mem:deploy", "0:8:8:0", 200); + try { + // making sure we do not have anything running + ServerLauncherSocketFactory.shutdown(new File(workDir), null); + } catch (Exception e) { + // ignore + } + + BasicDataSource + DATASOURCE = + DatabaseUtil.createMXJDataSource(DEFAULT_DB_NAME, workDir, DEFAULT_PORT); + Connection conn = DATASOURCE.getConnection(); + ScriptRunner runner = new ScriptRunner(conn, false, false); + runner.runScript(new BufferedReader(new InputStreamReader( + DatabaseUtil.class.getResourceAsStream("/sql/deploy.sql")))); + return DATASOURCE; + } + */ + public BasicDataSource build() { + return new BasicDataSource(); } } From 28c26b907d698d040f1cfe2e653b9221f0c544e0 Mon Sep 17 00:00:00 2001 From: andrerodrigues Date: Fri, 22 Mar 2024 00:58:38 +0000 Subject: [PATCH 6/8] Format code --- .../deployservice/db/DBHostTagDAOImpl.java | 98 ++++++++++++------- .../deployservice/db/DatabaseUtil.java | 11 ++- 2 files changed, 70 insertions(+), 39 deletions(-) diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DBHostTagDAOImpl.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DBHostTagDAOImpl.java index 9e2282d538..94c6995603 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DBHostTagDAOImpl.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DBHostTagDAOImpl.java @@ -1,5 +1,5 @@ -/* - * Copyright 2016 Pinterest, Inc. +/** + * Copyright (c) 2016-2024 Pinterest, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,12 @@ */ package com.pinterest.deployservice.db; -import com.google.common.collect.ImmutableList; import com.pinterest.deployservice.bean.HostTagBean; import com.pinterest.deployservice.bean.HostTagInfo; import com.pinterest.deployservice.bean.SetClause; import com.pinterest.deployservice.bean.UpdateStatement; import com.pinterest.deployservice.dao.HostTagDAO; +import java.util.List; import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.dbutils.QueryRunner; import org.apache.commons.dbutils.ResultSetHandler; @@ -28,24 +28,30 @@ import org.apache.commons.dbutils.handlers.BeanHandler; import org.apache.commons.dbutils.handlers.BeanListHandler; -import java.util.List; - - public class DBHostTagDAOImpl implements HostTagDAO { - private static final String INSERT_HOST_TAG_TEMPLATE = "INSERT INTO host_tags SET %s ON DUPLICATE KEY UPDATE %s"; - private static final String DELETE_HOST_TAG_BY_ENV_ID_AND_HOST_ID = "DELETE FROM host_tags WHERE env_id = ? AND host_id IN ( %s ) "; - private static final String DELETE_HOST_TAG_BY_ENV_ID_AND_TAG_NAME = "DELETE FROM host_tags WHERE env_id = ? AND tag_name = ? "; + private static final String INSERT_HOST_TAG_TEMPLATE = + "INSERT INTO host_tags SET %s ON DUPLICATE KEY UPDATE %s"; + private static final String DELETE_HOST_TAG_BY_ENV_ID_AND_HOST_ID = + "DELETE FROM host_tags WHERE env_id = ? AND host_id IN ( %s ) "; + private static final String DELETE_HOST_TAG_BY_ENV_ID_AND_TAG_NAME = + "DELETE FROM host_tags WHERE env_id = ? AND tag_name = ? "; private static final String DELETE_BY_HOST_ID = "DELETE FROM host_tags WHERE host_id = ?"; - private static final String GET_HOST_TAG_BY_HOST_ID_AND_TAG_NAME = "SELECT * FROM host_tags WHERE host_id = ? AND tag_name = ? "; - private static final String GET_PREREQUISITES_TAG_VALUES_BY_HOST_ID_AND_TAG_NAME = "SELECT DISTINCT(tag_value) AS tag_value FROM host_tags WHERE env_id = ? AND tag_name = ? AND tag_value < ?"; - private static final String GET_HOSTS_BY_ENV_ID_AND_TAG_NAME = "SELECT DISTINCT(host_tags.host_id) AS host_id, host_tags.tag_value AS tag_value, host_tags.tag_name AS tag_name, agents.host_name AS host_name FROM agents " + - "INNER JOIN host_tags ON agents.host_id = host_tags.host_id WHERE host_tags.tag_name = ? AND agents.env_id = ?"; - private static final String GET_HOSTS_BY_ENV_ID = "SELECT DISTINCT(host_tags.host_id) AS host_id, host_tags.tag_value AS tag_value, host_tags.tag_name AS tag_name, hosts.host_name AS host_name FROM hosts " + - "INNER JOIN host_tags ON hosts.host_id = host_tags.host_id " + - "WHERE host_tags.env_id = ?"; - private static final String COUNT_HOSTS_BY_ENV_ID_AND_TAGS = "SELECT count(DISTINCT(host_id)) FROM host_tags WHERE env_id = ? AND tag_name = ? AND tag_value IN (%s) "; - private static final String GET_ALL_BY_ENV_ID_AND_TAG_NAME = "SELECT * FROM host_tags WHERE env_id = ? AND tag_name = ? "; + private static final String GET_HOST_TAG_BY_HOST_ID_AND_TAG_NAME = + "SELECT * FROM host_tags WHERE host_id = ? AND tag_name = ? "; + private static final String GET_PREREQUISITES_TAG_VALUES_BY_HOST_ID_AND_TAG_NAME = + "SELECT DISTINCT(tag_value) AS tag_value FROM host_tags WHERE env_id = ? AND tag_name = ? AND tag_value < ?"; + private static final String GET_HOSTS_BY_ENV_ID_AND_TAG_NAME = + "SELECT DISTINCT(host_tags.host_id) AS host_id, host_tags.tag_value AS tag_value, host_tags.tag_name AS tag_name, agents.host_name AS host_name FROM agents " + + "INNER JOIN host_tags ON agents.host_id = host_tags.host_id WHERE host_tags.tag_name = ? AND agents.env_id = ?"; + private static final String GET_HOSTS_BY_ENV_ID = + "SELECT DISTINCT(host_tags.host_id) AS host_id, host_tags.tag_value AS tag_value, host_tags.tag_name AS tag_name, hosts.host_name AS host_name FROM hosts " + + "INNER JOIN host_tags ON hosts.host_id = host_tags.host_id " + + "WHERE host_tags.env_id = ?"; + private static final String COUNT_HOSTS_BY_ENV_ID_AND_TAGS = + "SELECT count(DISTINCT(host_id)) FROM host_tags WHERE env_id = ? AND tag_name = ? AND tag_value IN (%s) "; + private static final String GET_ALL_BY_ENV_ID_AND_TAG_NAME = + "SELECT * FROM host_tags WHERE env_id = ? AND tag_name = ? "; private static final RowProcessor ROW_PROCESSOR = new HostTagBeanRowProcessor(); private BasicDataSource dataSource; @@ -53,29 +59,31 @@ public DBHostTagDAOImpl(BasicDataSource dataSource) { this.dataSource = dataSource; } - @Override public void insertOrUpdate(HostTagBean hostTagBean) throws Exception { SetClause setClause = hostTagBean.genSetClause(); - String clause = String.format(INSERT_HOST_TAG_TEMPLATE, setClause.getClause(), HostTagBean.UPDATE_CLAUSE); + String clause = + String.format( + INSERT_HOST_TAG_TEMPLATE, setClause.getClause(), HostTagBean.UPDATE_CLAUSE); new QueryRunner(dataSource).update(clause, setClause.getValueArray()); } @Override public UpdateStatement genInsertOrUpdate(HostTagBean hostTagBean) { SetClause setClause = hostTagBean.genSetClause(); - String clause = String.format(INSERT_HOST_TAG_TEMPLATE, setClause.getClause(), HostTagBean.UPDATE_CLAUSE); + String clause = + String.format( + INSERT_HOST_TAG_TEMPLATE, setClause.getClause(), HostTagBean.UPDATE_CLAUSE); return new UpdateStatement(clause, setClause.getValueArray()); } - @Override public HostTagBean get(String hostId, String tagName) throws Exception { ResultSetHandler h = new BeanHandler(HostTagBean.class); - return new QueryRunner(dataSource).query(GET_HOST_TAG_BY_HOST_ID_AND_TAG_NAME, h, hostId, tagName); + return new QueryRunner(dataSource) + .query(GET_HOST_TAG_BY_HOST_ID_AND_TAG_NAME, h, hostId, tagName); } - @Override public void deleteAllByEnvId(String envId, String tagName) throws Exception { new QueryRunner(dataSource).update(DELETE_HOST_TAG_BY_ENV_ID_AND_TAG_NAME, envId, tagName); @@ -89,38 +97,56 @@ public void deleteByHostId(String hostId) throws Exception { @Override public void deleteAllByEnvIdAndHostIds(String envId, List hostIds) throws Exception { String hostStr = QueryUtils.genStringGroupClause(hostIds); - new QueryRunner(dataSource).update(String.format(DELETE_HOST_TAG_BY_ENV_ID_AND_HOST_ID, hostStr), envId); + new QueryRunner(dataSource) + .update(String.format(DELETE_HOST_TAG_BY_ENV_ID_AND_HOST_ID, hostStr), envId); } @Override - public List getAllByEnvIdAndTagName(String envId, String tagName) throws Exception { + public List getAllByEnvIdAndTagName(String envId, String tagName) + throws Exception { ResultSetHandler> h = new BeanListHandler<>(HostTagBean.class); return new QueryRunner(dataSource).query(GET_ALL_BY_ENV_ID_AND_TAG_NAME, h, envId, tagName); } @Override - public List getHostsByEnvIdAndTagName(String envId, String tagName) throws Exception { - ResultSetHandler> h = new BeanListHandler<>(HostTagInfo.class, ROW_PROCESSOR); - return new QueryRunner(dataSource).query(GET_HOSTS_BY_ENV_ID_AND_TAG_NAME, h, tagName, envId); + public List getHostsByEnvIdAndTagName(String envId, String tagName) + throws Exception { + ResultSetHandler> h = + new BeanListHandler<>(HostTagInfo.class, ROW_PROCESSOR); + return new QueryRunner(dataSource) + .query(GET_HOSTS_BY_ENV_ID_AND_TAG_NAME, h, tagName, envId); } @Override public List getHostsByEnvId(String envId) throws Exception { - ResultSetHandler> h = new BeanListHandler<>(HostTagInfo.class, ROW_PROCESSOR); + ResultSetHandler> h = + new BeanListHandler<>(HostTagInfo.class, ROW_PROCESSOR); return new QueryRunner(dataSource).query(GET_HOSTS_BY_ENV_ID, h, envId); } @Override - public long countHostsByEnvIdAndTags(String envId, String tagName, List tagValues) throws Exception { + public long countHostsByEnvIdAndTags(String envId, String tagName, List tagValues) + throws Exception { String tagValuesStr = QueryUtils.genStringGroupClause(tagValues); - Long n = new QueryRunner(dataSource).query(String.format(COUNT_HOSTS_BY_ENV_ID_AND_TAGS, tagValuesStr), - SingleResultSetHandlerFactory.newObjectHandler(), envId, tagName); + Long n = + new QueryRunner(dataSource) + .query( + String.format(COUNT_HOSTS_BY_ENV_ID_AND_TAGS, tagValuesStr), + SingleResultSetHandlerFactory.newObjectHandler(), + envId, + tagName); return n == null ? 0 : n; } @Override - public List getAllPrerequisiteTagValuesByEnvIdAndTagName(String envId, String tagName, String tagValue) throws Exception { - return new QueryRunner(dataSource).query(GET_PREREQUISITES_TAG_VALUES_BY_HOST_ID_AND_TAG_NAME, - SingleResultSetHandlerFactory.newListObjectHandler(), envId, tagName, tagValue); + public List getAllPrerequisiteTagValuesByEnvIdAndTagName( + String envId, String tagName, String tagValue) throws Exception { + return new QueryRunner(dataSource) + .query( + GET_PREREQUISITES_TAG_VALUES_BY_HOST_ID_AND_TAG_NAME, + SingleResultSetHandlerFactory.newListObjectHandler(), + envId, + tagName, + tagValue); } } diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java index 3145f89513..6f8e79cebd 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DatabaseUtil.java @@ -74,15 +74,20 @@ public static BasicDataSource createMysqlDataSource( // Embedded mysql source, for unit test only public static BasicDataSource createMXJDataSource(String dbName, String baseDir, int port) { - String - url = + String url = String.format( "jdbc:mysql:mxj://localhost:%d/%s?server" + ".basedir=%s&createDatabaseIfNotExist=true&server.initialize-user=true", port, dbName, baseDir); - return createDataSource(MYSQL_JDBC_DRIVER, url, "tester", "passwd", "0:8:8:0", + return createDataSource( + MYSQL_JDBC_DRIVER, + url, + "tester", + "passwd", + "0:8:8:0", MAX_WAIT_TIME_FOR_CONN_IN_MS); } + public static BasicDataSource createLocalDataSource(String url) { return createDataSource( MYSQL_JDBC_DRIVER, url, "root", "", "0:8:8:0", MAX_WAIT_TIME_FOR_CONN_IN_MS, null); From f251f21680cd3de226d95652b19a880b53a5e065 Mon Sep 17 00:00:00 2001 From: Tyler Ouyang Date: Fri, 22 Mar 2024 12:03:45 -0700 Subject: [PATCH 7/8] Remove embedded data source --- deploy-service/common/pom.xml | 5 - .../teletraanservice/bin/server-no-db.yaml | 91 ------------------- deploy-service/teletraanservice/pom.xml | 6 -- .../TeletraanServiceConfiguration.java | 4 - .../config/EmbeddedDataSourceFactory.java | 77 ---------------- ...nterest.teletraan.config.DataSourceFactory | 1 - 6 files changed, 184 deletions(-) delete mode 100644 deploy-service/teletraanservice/bin/server-no-db.yaml delete mode 100644 deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java diff --git a/deploy-service/common/pom.xml b/deploy-service/common/pom.xml index c02026ac0e..d9d0429e2c 100644 --- a/deploy-service/common/pom.xml +++ b/deploy-service/common/pom.xml @@ -154,11 +154,6 @@ 1.10.8 test - - mysql - mysql-connector-mxj - 5.0.12 - org.testcontainers testcontainers diff --git a/deploy-service/teletraanservice/bin/server-no-db.yaml b/deploy-service/teletraanservice/bin/server-no-db.yaml deleted file mode 100644 index c5e04699a8..0000000000 --- a/deploy-service/teletraanservice/bin/server-no-db.yaml +++ /dev/null @@ -1,91 +0,0 @@ -server: - applicationConnectors: - - type: http - port: 8080 - adminConnectors: - - type: http - port: 8081 - requestLog: - appenders: - - type: file - archive: true - archivedLogFilenamePattern: /tmp/teletraan/access-%d.log - archivedFileCount: 5 - currentLogFilename: /tmp/teletraan/access.log - layout: - type: access-json - timestampFormat: "yyyy-MM-dd'T'HH:mm:ss.SSSZ" - requestAttributes: [name] - customFieldNames: - requestAttributes: principal - includes: [timestamp, remoteAddress, protocol, method, requestUri, statusCode] - registerDefaultExceptionMappers: false - -logging: - level: INFO - loggers: - "com.pinterest.teletraan": DEBUG - "com.pinterest.deployservice": DEBUG - appenders: - - type: file - timeZone: UTC - threshold: DEBUG - archive: false - currentLogFilename: /tmp/teletraan/service.log - - type: console - threshold: ALL - timeZone: UTC - target: stdout - -db: - type: embedded - -system: - dashboardUrl: ${TELETRAAN_DASHBOARD_URL:-http://localhost:8888} - -workers: - # StateTransitioner looks for active deploys, set deploy status and - # transition deploys to their next states. - - name: StateTransitioner - properties: - initialDelay: 10 - period: 30 - - # AutoPromoter provides the Auto Deploy support. It finds the deploy - # candidates from preceding stage and promote them to next stages. - - name: AutoPromoter - properties: - initialDelay: 20 - period: 30 - bufferTimeMinutes: 2 - - # AgentJanitor cleans up any obsoleted hosts or agents records - - name: SimpleAgentJanitor - properties: - initialDelay: 30 - period: 300 - minStaleHostThreshold: 150 - maxStaleHostThreshold: 600 - - - name: MetricsEmitter - properties: - initialDelay: 2 - period: 5 - -# -# Health settings -# -health: - delayedShutdownHandlerEnabled: true - shutdownWaitPeriod: 30s - healthCheckUrlPaths: - - /healthcheck - - /health-check - healthChecks: - - name: generic - critical: true - schedule: - checkInterval: 5s - downtimeInterval: 10s - failureAttempts: 2 - successAttempts: 1 diff --git a/deploy-service/teletraanservice/pom.xml b/deploy-service/teletraanservice/pom.xml index 3e41f7871b..88a37fa9a7 100644 --- a/deploy-service/teletraanservice/pom.xml +++ b/deploy-service/teletraanservice/pom.xml @@ -65,12 +65,6 @@ org.quartz-scheduler quartz - - - mysql - mysql-connector-mxj - 5.0.12 - com.ibatis ibatis2-common diff --git a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/TeletraanServiceConfiguration.java b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/TeletraanServiceConfiguration.java index 8343b085d2..cb843678af 100644 --- a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/TeletraanServiceConfiguration.java +++ b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/TeletraanServiceConfiguration.java @@ -33,7 +33,6 @@ import com.pinterest.teletraan.config.DefaultEmailFactory; import com.pinterest.teletraan.config.DefaultHostGroupFactory; import com.pinterest.teletraan.config.EmailFactory; -import com.pinterest.teletraan.config.EmbeddedDataSourceFactory; import com.pinterest.teletraan.config.ExternalAlertsConfigFactory; import com.pinterest.teletraan.config.HostGroupFactory; import com.pinterest.teletraan.config.JenkinsFactory; @@ -124,9 +123,6 @@ public class TeletraanServiceConfiguration extends Configuration { private List accountAllowList; public DataSourceFactory getDataSourceFactory() { - if (dataSourceFactory == null) { - return new EmbeddedDataSourceFactory(); - } return dataSourceFactory; } diff --git a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java deleted file mode 100644 index ed9a32c418..0000000000 --- a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/config/EmbeddedDataSourceFactory.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright (c) 2016-2024 Pinterest, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.pinterest.teletraan.config; - -import com.fasterxml.jackson.annotation.JsonTypeName; -import org.apache.commons.dbcp.BasicDataSource; - -/* -import com.fasterxml.jackson.annotation.JsonProperty; -import com.pinterest.deployservice.db.DatabaseUtil; -import org.apache.commons.dbcp.BasicDataSource; - -import com.ibatis.common.jdbc.ScriptRunner; -import com.mysql.management.driverlaunched.ServerLauncherSocketFactory; - -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; -import java.sql.Connection; -*/ - -@JsonTypeName("embedded") -public class EmbeddedDataSourceFactory implements DataSourceFactory { - /* - * - * Uncomment this block and the imports if you want to use embedded mysql - * - private final static String DEFAULT_BASE_DIR = "/tmp/teletraan/db"; - private final static String DEFAULT_DB_NAME = "deploy"; - private final static int DEFAULT_PORT = 3305; - - @JsonProperty - private String workDir = DEFAULT_BASE_DIR; - - public String getWorkDir() { - return workDir; - } - - public void setWorkDir(String workDir) { - this.workDir = workDir; - } - - public BasicDataSource build() throws Exception { - try { - // making sure we do not have anything running - ServerLauncherSocketFactory.shutdown(new File(workDir), null); - } catch (Exception e) { - // ignore - } - - BasicDataSource - DATASOURCE = - DatabaseUtil.createMXJDataSource(DEFAULT_DB_NAME, workDir, DEFAULT_PORT); - Connection conn = DATASOURCE.getConnection(); - ScriptRunner runner = new ScriptRunner(conn, false, false); - runner.runScript(new BufferedReader(new InputStreamReader( - DatabaseUtil.class.getResourceAsStream("/sql/deploy.sql")))); - return DATASOURCE; - } - */ - public BasicDataSource build() { - return new BasicDataSource(); - } -} diff --git a/deploy-service/teletraanservice/src/main/resources/META-INF/services/com.pinterest.teletraan.config.DataSourceFactory b/deploy-service/teletraanservice/src/main/resources/META-INF/services/com.pinterest.teletraan.config.DataSourceFactory index b3193843a8..88f3f19683 100644 --- a/deploy-service/teletraanservice/src/main/resources/META-INF/services/com.pinterest.teletraan.config.DataSourceFactory +++ b/deploy-service/teletraanservice/src/main/resources/META-INF/services/com.pinterest.teletraan.config.DataSourceFactory @@ -1,3 +1,2 @@ com.pinterest.teletraan.config.ZKMysqlDataSourceFactory com.pinterest.teletraan.config.MysqlDataSourceFactory -com.pinterest.teletraan.config.EmbeddedDataSourceFactory From b15a3dafd158ce71566bac0c7b86f337268d1455 Mon Sep 17 00:00:00 2001 From: andrerodrigues Date: Fri, 22 Mar 2024 20:43:07 +0000 Subject: [PATCH 8/8] Revert changes on DBHostTagDAOImpl --- .../deployservice/db/DBHostTagDAOImpl.java | 98 +++++++------------ 1 file changed, 36 insertions(+), 62 deletions(-) diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DBHostTagDAOImpl.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DBHostTagDAOImpl.java index 94c6995603..9e2282d538 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DBHostTagDAOImpl.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/db/DBHostTagDAOImpl.java @@ -1,5 +1,5 @@ -/** - * Copyright (c) 2016-2024 Pinterest, Inc. +/* + * Copyright 2016 Pinterest, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,12 @@ */ package com.pinterest.deployservice.db; +import com.google.common.collect.ImmutableList; import com.pinterest.deployservice.bean.HostTagBean; import com.pinterest.deployservice.bean.HostTagInfo; import com.pinterest.deployservice.bean.SetClause; import com.pinterest.deployservice.bean.UpdateStatement; import com.pinterest.deployservice.dao.HostTagDAO; -import java.util.List; import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.dbutils.QueryRunner; import org.apache.commons.dbutils.ResultSetHandler; @@ -28,30 +28,24 @@ import org.apache.commons.dbutils.handlers.BeanHandler; import org.apache.commons.dbutils.handlers.BeanListHandler; +import java.util.List; + + public class DBHostTagDAOImpl implements HostTagDAO { - private static final String INSERT_HOST_TAG_TEMPLATE = - "INSERT INTO host_tags SET %s ON DUPLICATE KEY UPDATE %s"; - private static final String DELETE_HOST_TAG_BY_ENV_ID_AND_HOST_ID = - "DELETE FROM host_tags WHERE env_id = ? AND host_id IN ( %s ) "; - private static final String DELETE_HOST_TAG_BY_ENV_ID_AND_TAG_NAME = - "DELETE FROM host_tags WHERE env_id = ? AND tag_name = ? "; + private static final String INSERT_HOST_TAG_TEMPLATE = "INSERT INTO host_tags SET %s ON DUPLICATE KEY UPDATE %s"; + private static final String DELETE_HOST_TAG_BY_ENV_ID_AND_HOST_ID = "DELETE FROM host_tags WHERE env_id = ? AND host_id IN ( %s ) "; + private static final String DELETE_HOST_TAG_BY_ENV_ID_AND_TAG_NAME = "DELETE FROM host_tags WHERE env_id = ? AND tag_name = ? "; private static final String DELETE_BY_HOST_ID = "DELETE FROM host_tags WHERE host_id = ?"; - private static final String GET_HOST_TAG_BY_HOST_ID_AND_TAG_NAME = - "SELECT * FROM host_tags WHERE host_id = ? AND tag_name = ? "; - private static final String GET_PREREQUISITES_TAG_VALUES_BY_HOST_ID_AND_TAG_NAME = - "SELECT DISTINCT(tag_value) AS tag_value FROM host_tags WHERE env_id = ? AND tag_name = ? AND tag_value < ?"; - private static final String GET_HOSTS_BY_ENV_ID_AND_TAG_NAME = - "SELECT DISTINCT(host_tags.host_id) AS host_id, host_tags.tag_value AS tag_value, host_tags.tag_name AS tag_name, agents.host_name AS host_name FROM agents " - + "INNER JOIN host_tags ON agents.host_id = host_tags.host_id WHERE host_tags.tag_name = ? AND agents.env_id = ?"; - private static final String GET_HOSTS_BY_ENV_ID = - "SELECT DISTINCT(host_tags.host_id) AS host_id, host_tags.tag_value AS tag_value, host_tags.tag_name AS tag_name, hosts.host_name AS host_name FROM hosts " - + "INNER JOIN host_tags ON hosts.host_id = host_tags.host_id " - + "WHERE host_tags.env_id = ?"; - private static final String COUNT_HOSTS_BY_ENV_ID_AND_TAGS = - "SELECT count(DISTINCT(host_id)) FROM host_tags WHERE env_id = ? AND tag_name = ? AND tag_value IN (%s) "; - private static final String GET_ALL_BY_ENV_ID_AND_TAG_NAME = - "SELECT * FROM host_tags WHERE env_id = ? AND tag_name = ? "; + private static final String GET_HOST_TAG_BY_HOST_ID_AND_TAG_NAME = "SELECT * FROM host_tags WHERE host_id = ? AND tag_name = ? "; + private static final String GET_PREREQUISITES_TAG_VALUES_BY_HOST_ID_AND_TAG_NAME = "SELECT DISTINCT(tag_value) AS tag_value FROM host_tags WHERE env_id = ? AND tag_name = ? AND tag_value < ?"; + private static final String GET_HOSTS_BY_ENV_ID_AND_TAG_NAME = "SELECT DISTINCT(host_tags.host_id) AS host_id, host_tags.tag_value AS tag_value, host_tags.tag_name AS tag_name, agents.host_name AS host_name FROM agents " + + "INNER JOIN host_tags ON agents.host_id = host_tags.host_id WHERE host_tags.tag_name = ? AND agents.env_id = ?"; + private static final String GET_HOSTS_BY_ENV_ID = "SELECT DISTINCT(host_tags.host_id) AS host_id, host_tags.tag_value AS tag_value, host_tags.tag_name AS tag_name, hosts.host_name AS host_name FROM hosts " + + "INNER JOIN host_tags ON hosts.host_id = host_tags.host_id " + + "WHERE host_tags.env_id = ?"; + private static final String COUNT_HOSTS_BY_ENV_ID_AND_TAGS = "SELECT count(DISTINCT(host_id)) FROM host_tags WHERE env_id = ? AND tag_name = ? AND tag_value IN (%s) "; + private static final String GET_ALL_BY_ENV_ID_AND_TAG_NAME = "SELECT * FROM host_tags WHERE env_id = ? AND tag_name = ? "; private static final RowProcessor ROW_PROCESSOR = new HostTagBeanRowProcessor(); private BasicDataSource dataSource; @@ -59,31 +53,29 @@ public DBHostTagDAOImpl(BasicDataSource dataSource) { this.dataSource = dataSource; } + @Override public void insertOrUpdate(HostTagBean hostTagBean) throws Exception { SetClause setClause = hostTagBean.genSetClause(); - String clause = - String.format( - INSERT_HOST_TAG_TEMPLATE, setClause.getClause(), HostTagBean.UPDATE_CLAUSE); + String clause = String.format(INSERT_HOST_TAG_TEMPLATE, setClause.getClause(), HostTagBean.UPDATE_CLAUSE); new QueryRunner(dataSource).update(clause, setClause.getValueArray()); } @Override public UpdateStatement genInsertOrUpdate(HostTagBean hostTagBean) { SetClause setClause = hostTagBean.genSetClause(); - String clause = - String.format( - INSERT_HOST_TAG_TEMPLATE, setClause.getClause(), HostTagBean.UPDATE_CLAUSE); + String clause = String.format(INSERT_HOST_TAG_TEMPLATE, setClause.getClause(), HostTagBean.UPDATE_CLAUSE); return new UpdateStatement(clause, setClause.getValueArray()); } + @Override public HostTagBean get(String hostId, String tagName) throws Exception { ResultSetHandler h = new BeanHandler(HostTagBean.class); - return new QueryRunner(dataSource) - .query(GET_HOST_TAG_BY_HOST_ID_AND_TAG_NAME, h, hostId, tagName); + return new QueryRunner(dataSource).query(GET_HOST_TAG_BY_HOST_ID_AND_TAG_NAME, h, hostId, tagName); } + @Override public void deleteAllByEnvId(String envId, String tagName) throws Exception { new QueryRunner(dataSource).update(DELETE_HOST_TAG_BY_ENV_ID_AND_TAG_NAME, envId, tagName); @@ -97,56 +89,38 @@ public void deleteByHostId(String hostId) throws Exception { @Override public void deleteAllByEnvIdAndHostIds(String envId, List hostIds) throws Exception { String hostStr = QueryUtils.genStringGroupClause(hostIds); - new QueryRunner(dataSource) - .update(String.format(DELETE_HOST_TAG_BY_ENV_ID_AND_HOST_ID, hostStr), envId); + new QueryRunner(dataSource).update(String.format(DELETE_HOST_TAG_BY_ENV_ID_AND_HOST_ID, hostStr), envId); } @Override - public List getAllByEnvIdAndTagName(String envId, String tagName) - throws Exception { + public List getAllByEnvIdAndTagName(String envId, String tagName) throws Exception { ResultSetHandler> h = new BeanListHandler<>(HostTagBean.class); return new QueryRunner(dataSource).query(GET_ALL_BY_ENV_ID_AND_TAG_NAME, h, envId, tagName); } @Override - public List getHostsByEnvIdAndTagName(String envId, String tagName) - throws Exception { - ResultSetHandler> h = - new BeanListHandler<>(HostTagInfo.class, ROW_PROCESSOR); - return new QueryRunner(dataSource) - .query(GET_HOSTS_BY_ENV_ID_AND_TAG_NAME, h, tagName, envId); + public List getHostsByEnvIdAndTagName(String envId, String tagName) throws Exception { + ResultSetHandler> h = new BeanListHandler<>(HostTagInfo.class, ROW_PROCESSOR); + return new QueryRunner(dataSource).query(GET_HOSTS_BY_ENV_ID_AND_TAG_NAME, h, tagName, envId); } @Override public List getHostsByEnvId(String envId) throws Exception { - ResultSetHandler> h = - new BeanListHandler<>(HostTagInfo.class, ROW_PROCESSOR); + ResultSetHandler> h = new BeanListHandler<>(HostTagInfo.class, ROW_PROCESSOR); return new QueryRunner(dataSource).query(GET_HOSTS_BY_ENV_ID, h, envId); } @Override - public long countHostsByEnvIdAndTags(String envId, String tagName, List tagValues) - throws Exception { + public long countHostsByEnvIdAndTags(String envId, String tagName, List tagValues) throws Exception { String tagValuesStr = QueryUtils.genStringGroupClause(tagValues); - Long n = - new QueryRunner(dataSource) - .query( - String.format(COUNT_HOSTS_BY_ENV_ID_AND_TAGS, tagValuesStr), - SingleResultSetHandlerFactory.newObjectHandler(), - envId, - tagName); + Long n = new QueryRunner(dataSource).query(String.format(COUNT_HOSTS_BY_ENV_ID_AND_TAGS, tagValuesStr), + SingleResultSetHandlerFactory.newObjectHandler(), envId, tagName); return n == null ? 0 : n; } @Override - public List getAllPrerequisiteTagValuesByEnvIdAndTagName( - String envId, String tagName, String tagValue) throws Exception { - return new QueryRunner(dataSource) - .query( - GET_PREREQUISITES_TAG_VALUES_BY_HOST_ID_AND_TAG_NAME, - SingleResultSetHandlerFactory.newListObjectHandler(), - envId, - tagName, - tagValue); + public List getAllPrerequisiteTagValuesByEnvIdAndTagName(String envId, String tagName, String tagValue) throws Exception { + return new QueryRunner(dataSource).query(GET_PREREQUISITES_TAG_VALUES_BY_HOST_ID_AND_TAG_NAME, + SingleResultSetHandlerFactory.newListObjectHandler(), envId, tagName, tagValue); } }