From 3240264f25df0c9f57eb614266ea6a54de455bd1 Mon Sep 17 00:00:00 2001 From: Yuto Kawamura Date: Wed, 21 Oct 2020 18:38:33 +0900 Subject: [PATCH] Add javadocs and unit tests --- src/main/java/kmql/Engine.java | 28 +++++++ src/main/java/kmql/OutputFormat.java | 10 +++ src/main/java/kmql/OutputFormatRegistry.java | 18 +++++ src/main/java/kmql/Table.java | 21 +++++ src/main/java/kmql/TableRegistry.java | 26 ++++++ src/main/java/kmql/format/JsonFormat.java | 3 + src/main/java/kmql/format/SsvFormat.java | 11 ++- src/main/java/kmql/format/TableFormat.java | 4 +- src/test/java/kmql/DatabaseTest.java | 18 +---- src/test/java/kmql/EngineTest.java | 79 +++++++++++++++++++ .../java/kmql/OutputFormatRegistryTest.java | 24 ++++++ src/test/java/kmql/SqlAnalyzerTest.java | 20 +++++ src/test/java/kmql/SqlUtils.java | 14 ++++ src/test/java/kmql/TableRegistryTest.java | 24 ++++++ src/test/java/kmql/format/JsonFormatTest.java | 36 +++++++++ .../java/kmql/format/TableFormatTest.java | 42 ++++++++++ 16 files changed, 360 insertions(+), 18 deletions(-) create mode 100644 src/test/java/kmql/EngineTest.java create mode 100644 src/test/java/kmql/OutputFormatRegistryTest.java create mode 100644 src/test/java/kmql/SqlAnalyzerTest.java create mode 100644 src/test/java/kmql/TableRegistryTest.java create mode 100644 src/test/java/kmql/format/JsonFormatTest.java create mode 100644 src/test/java/kmql/format/TableFormatTest.java diff --git a/src/main/java/kmql/Engine.java b/src/main/java/kmql/Engine.java index 2e5c4a3..ab37b27 100644 --- a/src/main/java/kmql/Engine.java +++ b/src/main/java/kmql/Engine.java @@ -10,6 +10,9 @@ import lombok.AllArgsConstructor; import lombok.NonNull; +/** + * A core runtime of kmql. + */ @AllArgsConstructor public class Engine implements AutoCloseable { private final AdminClient adminClient; @@ -18,6 +21,13 @@ public class Engine implements AutoCloseable { @NonNull private OutputFormat outputFormat; + /** + * Creates a new {@link Engine} from the given {@link AdminClient} and the name of the output format. + * Default instances are used for both of {@link OutputFormatRegistry} and {@link TableRegistry}. + * @param adminClient an {@link AdminClient} to access Kafka cluster's metadata. + * @param outputFormatName the name of output format to use. + * @return an {@link Engine}. + */ public static Engine from(AdminClient adminClient, String outputFormatName) { OutputFormatRegistry outputFormatRegistry = OutputFormatRegistry.DEFAULT; OutputFormat outputFormat = lookupOutputFormat(outputFormatRegistry, outputFormatName); @@ -25,6 +35,13 @@ public static Engine from(AdminClient adminClient, String outputFormatName) { return new Engine(adminClient, db, outputFormatRegistry, outputFormat); } + /** + * Execute the given command. + * Command should be a valid SQL for now. + * @param command command to execute. + * @param output the output stream to write the formatted query result. + * @throws SQLException when SQL fails. + */ public void execute(String command, BufferedOutputStream output) throws SQLException { prepareRequiredTables(command); db.executeQuery(command, results -> { @@ -50,6 +67,9 @@ private void prepareRequiredTables(String sql) { } } + /** + * Initialize all tables that this engine supports. + */ public void initAllTables() { try { db.prepareAllTables(adminClient); @@ -58,10 +78,18 @@ public void initAllTables() { } } + /** + * Set the output format to the given instance of {@link OutputFormat}. + * @param outputFormat a new output format to use. + */ public void setOutputFormat(@NonNull OutputFormat outputFormat) { this.outputFormat = outputFormat; } + /** + * Set the output format to the specified instance by the given name. + * @param name name of the output format. + */ public void setOutputFormat(String name) { OutputFormat newFormat = lookupOutputFormat(outputFormatRegistry, name); setOutputFormat(newFormat); diff --git a/src/main/java/kmql/OutputFormat.java b/src/main/java/kmql/OutputFormat.java index 2ab7f82..07448be 100644 --- a/src/main/java/kmql/OutputFormat.java +++ b/src/main/java/kmql/OutputFormat.java @@ -5,6 +5,16 @@ import java.sql.ResultSet; import java.sql.SQLException; +/** + * An interface of the kmql output format implementation. + */ public interface OutputFormat { + /** + * Format the given {@link ResultSet} and write it into {@link BufferedOutputStream}. + * @param results the SQL query result. + * @param out the output stream to write the output. + * @throws SQLException if {@link ResultSet} throws. + * @throws IOException if {@link BufferedOutputStream} throws. + */ void formatTo(ResultSet results, BufferedOutputStream out) throws SQLException, IOException; } diff --git a/src/main/java/kmql/OutputFormatRegistry.java b/src/main/java/kmql/OutputFormatRegistry.java index f6ac696..c046b2c 100644 --- a/src/main/java/kmql/OutputFormatRegistry.java +++ b/src/main/java/kmql/OutputFormatRegistry.java @@ -8,6 +8,9 @@ import kmql.format.SsvFormat; import kmql.format.TableFormat; +/** + * Registry of outputs formats. + */ public class OutputFormatRegistry { public static final OutputFormatRegistry DEFAULT = new OutputFormatRegistry(); @@ -23,16 +26,31 @@ public OutputFormatRegistry() { formats = new ConcurrentHashMap<>(); } + /** + * Register the given format under the given name to the default registry. + * @param name the name of the format. + * @param format the format instance. + */ public static void registerDefault(String name, OutputFormat format) { DEFAULT.register(name, format); } + /** + * Register the given format under the given name. + * @param name the name of the format. + * @param format the format instance. + */ public void register(String name, OutputFormat format) { if (formats.putIfAbsent(name, format) != null) { throw new IllegalArgumentException("conflicting format name: " + name); } } + /** + * Lookup a format by the name. + * @param name the name of the format. + * @return an {@link OutputFormat} if presents. + */ public Optional lookup(String name) { return Optional.ofNullable(formats.get(name)); } diff --git a/src/main/java/kmql/Table.java b/src/main/java/kmql/Table.java index 915d114..8bb83f4 100644 --- a/src/main/java/kmql/Table.java +++ b/src/main/java/kmql/Table.java @@ -7,12 +7,33 @@ import org.apache.kafka.clients.admin.AdminClient; +/** + * An interface of a kmql table implementation. + */ public interface Table { + /** + * Name of the table. + * @return name of the table. + */ String name(); + /** + * Optionally declared list of table names that this table is depending on to construct the table. + * The returned tables are prepared before the {@link #prepare(Connection, AdminClient)} method of this + * table is called. + * @return list of table names to depend on. + */ default Collection dependencyTables() { return emptyList(); } + /** + * Prepare this table on the given {@link Connection}. This process includes both of creating a table and + * feeding in its data by obtaining it from the given {@link AdminClient}. + * At the time of this method call, absence of the target table is guaranteed. + * @param connection a JDBC {@link Connection}. + * @param adminClient a Kafka {@link AdminClient}. + * @throws Exception at any errors. + */ void prepare(Connection connection, AdminClient adminClient) throws Exception; } diff --git a/src/main/java/kmql/TableRegistry.java b/src/main/java/kmql/TableRegistry.java index f44be7b..ae4b2d3 100644 --- a/src/main/java/kmql/TableRegistry.java +++ b/src/main/java/kmql/TableRegistry.java @@ -12,6 +12,9 @@ import kmql.table.LogdirsTable; import kmql.table.ReplicasTable; +/** + * Registry of kmql tables. + */ public class TableRegistry implements Iterable> { public static final TableRegistry DEFAULT = new TableRegistry(); @@ -24,10 +27,19 @@ public class TableRegistry implements Iterable> { private final ConcurrentMap tables; + /** + * Register the given table under the {@link Table#name()} to the default registry. + * @param table the table instance. + */ public static void registerDefault(Table table) { DEFAULT.register(table.name(), table); } + /** + * Register the given format under the given name to the default registry. + * @param name the name of the table. + * @param table the table instance. + */ public static void registerDefault(String name, Table table) { DEFAULT.register(name, table); } @@ -36,16 +48,30 @@ public TableRegistry() { tables = new ConcurrentHashMap<>(); } + /** + * Register the given table under the given name. + * @param name the name of the table. + * @param table the table instance. + */ public void register(String name, Table table) { if (tables.putIfAbsent(name, table) != null) { throw new IllegalArgumentException("conflicting table name: " + name); } } + /** + * Lookup a table by the name. + * @param name the name of the table. + * @return an {@link Table} if presents. + */ public Optional lookup(String name) { return Optional.ofNullable(tables.get(name)); } + /** + * Returns the iterator over table entries registered in this registry. + * @return an iterator for table name and table instances. + */ @Override public Iterator> iterator() { return tables.entrySet().iterator(); diff --git a/src/main/java/kmql/format/JsonFormat.java b/src/main/java/kmql/format/JsonFormat.java index 0dddac0..d82f650 100644 --- a/src/main/java/kmql/format/JsonFormat.java +++ b/src/main/java/kmql/format/JsonFormat.java @@ -14,6 +14,9 @@ import kmql.OutputFormat; +/** + * JSON format. + */ public class JsonFormat implements OutputFormat { private final ObjectMapper mapper = new ObjectMapper() .disable(Feature.AUTO_CLOSE_TARGET); diff --git a/src/main/java/kmql/format/SsvFormat.java b/src/main/java/kmql/format/SsvFormat.java index d5df20e..7cde7a9 100644 --- a/src/main/java/kmql/format/SsvFormat.java +++ b/src/main/java/kmql/format/SsvFormat.java @@ -1,6 +1,15 @@ package kmql.format; -public class SsvFormat extends AbstractCsvFormat { +/** + * Space-Separated Values format. + * Example: + * {@code + * # HEADER1, HEADER2, HEADER3 + * foo 1234 true + * bar 5678 false + * } + */ +public class SsvFormat extends AbstractCsvFormat { public SsvFormat() { super(" "); } diff --git a/src/main/java/kmql/format/TableFormat.java b/src/main/java/kmql/format/TableFormat.java index 054ef36..ef66676 100644 --- a/src/main/java/kmql/format/TableFormat.java +++ b/src/main/java/kmql/format/TableFormat.java @@ -12,6 +12,9 @@ import kmql.OutputFormat; +/** + * Pretty-printed table format. + */ public class TableFormat implements OutputFormat { public static final String[][] ARRAY_PROTO = new String[0][]; @@ -36,7 +39,6 @@ public void formatTo(ResultSet results, BufferedOutputStream out) throws SQLExce PrintWriter pw = new PrintWriter(out); pw.write(FlipTable.of(headers, rows.toArray(ARRAY_PROTO))); - pw.println(); pw.flush(); } } diff --git a/src/test/java/kmql/DatabaseTest.java b/src/test/java/kmql/DatabaseTest.java index 72c7942..2c5b5dc 100644 --- a/src/test/java/kmql/DatabaseTest.java +++ b/src/test/java/kmql/DatabaseTest.java @@ -10,7 +10,6 @@ import static org.mockito.Mockito.verify; import java.sql.Connection; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.concurrent.atomic.AtomicReference; @@ -60,24 +59,11 @@ public void tearDown() throws Exception { db.close(); } - private boolean tableExists(String name) throws SQLException { - try (Statement stmt = connection.createStatement(); - ResultSet results = stmt.executeQuery("SHOW TABLES")) { - while (results.next()) { - String table = results.getString(1); - if (table.toLowerCase().equals(name.toLowerCase())) { - return true; - } - } - } - return false; - } - @Test public void prepareTable() throws Exception { db.prepareTable("xyz", adminClient); verify(xyzTable, times(1)).prepare(connection, adminClient); - assertTrue(tableExists("xyz")); + assertTrue(SqlUtils.tableExists(connection, "xyz")); // This should be no-op because it's already initialized db.prepareTable("xyz", adminClient); verify(xyzTable, times(1)).prepare(connection, adminClient); @@ -99,7 +85,7 @@ public void prepareAllTables() throws Exception { public void dropTable() throws Exception { db.prepareTable("xyz", adminClient); db.dropTable("xyz"); - assertFalse(tableExists("xyz")); + assertFalse(SqlUtils.tableExists(connection, "xyz")); } @Test(expected = IllegalStateException.class) diff --git a/src/test/java/kmql/EngineTest.java b/src/test/java/kmql/EngineTest.java new file mode 100644 index 0000000..82036a0 --- /dev/null +++ b/src/test/java/kmql/EngineTest.java @@ -0,0 +1,79 @@ +package kmql; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +import java.io.BufferedOutputStream; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.kafka.clients.admin.AdminClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +public class EngineTest { + private Engine engine; + + @Mock + private AdminClient adminClient; + private Connection connection; + private final List outputs = new ArrayList<>(); + + @Before + public void setUp() { + TableRegistry tableRegistry = new TableRegistry(); + tableRegistry.register("xyz", new Table() { + @Override + public String name() { + return "xyz"; + } + + @Override + public void prepare(Connection connection, AdminClient adminClient) throws Exception { + try (Statement stmt = connection.createStatement()) { + stmt.execute("CREATE TABLE xyz (id VARCHAR(255) NOT NULL)"); + stmt.executeUpdate("INSERT INTO xyz VALUES ('foo'), ('bar'), ('baz')"); + } + } + }); + connection = SqlUtils.connection(); + Database db = new Database(connection, tableRegistry); + OutputFormatRegistry outputFormatRegistry = new OutputFormatRegistry(); + OutputFormat rawFormat = (results, out) -> { + while (results.next()) { + String id = results.getString(1); + outputs.add(id); + } + }; + outputFormatRegistry.register("raw", rawFormat); + engine = new Engine(adminClient, db, outputFormatRegistry, rawFormat); + } + + @After + public void tearDown() throws Exception { + engine.close(); + } + + @Test + public void execute() throws SQLException { + // This call should initialize the table internally + engine.execute("SELECT id FROM xyz", mock(BufferedOutputStream.class)); + assertEquals(Arrays.asList("foo", "bar", "baz"), outputs); + outputs.clear(); + // This call should use existing table + engine.execute("SELECT id FROM xyz", mock(BufferedOutputStream.class)); + assertEquals(Arrays.asList("foo", "bar", "baz"), outputs); + } + + @Test + public void initAllTables() throws SQLException { + engine.initAllTables(); + SqlUtils.tableExists(connection, "xyz"); + } +} diff --git a/src/test/java/kmql/OutputFormatRegistryTest.java b/src/test/java/kmql/OutputFormatRegistryTest.java new file mode 100644 index 0000000..7d7ec00 --- /dev/null +++ b/src/test/java/kmql/OutputFormatRegistryTest.java @@ -0,0 +1,24 @@ +package kmql; + +import static org.junit.Assert.assertSame; +import static org.mockito.Mockito.mock; + +import org.junit.Test; + +public class OutputFormatRegistryTest { + private final OutputFormatRegistry registry = new OutputFormatRegistry(); + + @Test + public void register() { + OutputFormat format = mock(OutputFormat.class); + registry.register("foo", format); + assertSame(format, registry.lookup("foo").get()); + } + + @Test(expected = IllegalArgumentException.class) + public void registerConflict() { + OutputFormat format = mock(OutputFormat.class); + registry.register("foo", format); + registry.register("foo", format); + } +} diff --git a/src/test/java/kmql/SqlAnalyzerTest.java b/src/test/java/kmql/SqlAnalyzerTest.java new file mode 100644 index 0000000..175e533 --- /dev/null +++ b/src/test/java/kmql/SqlAnalyzerTest.java @@ -0,0 +1,20 @@ +package kmql; + +import static java.util.Collections.singletonList; +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; + +import org.junit.Test; + +public class SqlAnalyzerTest { + @Test + public void requiredTables() { + assertEquals(singletonList("xyz"), SqlAnalyzer.requiredTables("SELECT * FROM xyz WHERE x = 10")); + assertEquals(Arrays.asList("xyz", "foo"), + SqlAnalyzer.requiredTables("SELECT * FROM xyz LEFT JOIN foo")); + assertEquals(singletonList("xyz"), SqlAnalyzer.requiredTables("select * from xyz where x = 10")); + assertEquals(Arrays.asList("xyz", "foo"), + SqlAnalyzer.requiredTables("select * from xyz left join foo")); + } +} diff --git a/src/test/java/kmql/SqlUtils.java b/src/test/java/kmql/SqlUtils.java index 85129c6..0b74c87 100644 --- a/src/test/java/kmql/SqlUtils.java +++ b/src/test/java/kmql/SqlUtils.java @@ -4,6 +4,7 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import org.h2.tools.SimpleResultSet; @@ -27,6 +28,19 @@ public static Connection connection() { } } + public static boolean tableExists(Connection connection, String name) throws SQLException { + try (Statement stmt = connection.createStatement(); + ResultSet results = stmt.executeQuery("SHOW TABLES")) { + while (results.next()) { + String table = results.getString(1); + if (table.toLowerCase().equals(name.toLowerCase())) { + return true; + } + } + } + return false; + } + public static ResultSet resultSet(ColumnInfo[] columns, Object[]... rows) { SimpleResultSet results = new SimpleResultSet(); for (ColumnInfo column : columns) { diff --git a/src/test/java/kmql/TableRegistryTest.java b/src/test/java/kmql/TableRegistryTest.java new file mode 100644 index 0000000..b15eac8 --- /dev/null +++ b/src/test/java/kmql/TableRegistryTest.java @@ -0,0 +1,24 @@ +package kmql; + +import static org.junit.Assert.assertSame; +import static org.mockito.Mockito.mock; + +import org.junit.Test; + +public class TableRegistryTest { + private final TableRegistry registry = new TableRegistry(); + + @Test + public void register() { + Table table = mock(Table.class); + registry.register("xyz", table); + assertSame(table, registry.lookup("xyz").get()); + } + + @Test(expected = IllegalArgumentException.class) + public void registerTwice() { + Table table = mock(Table.class); + registry.register("xyz", table); + registry.register("xyz", table); + } +} diff --git a/src/test/java/kmql/format/JsonFormatTest.java b/src/test/java/kmql/format/JsonFormatTest.java new file mode 100644 index 0000000..134029c --- /dev/null +++ b/src/test/java/kmql/format/JsonFormatTest.java @@ -0,0 +1,36 @@ +package kmql.format; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.sql.ResultSet; +import java.sql.Types; + +import org.junit.Test; + +import kmql.SqlUtils; +import kmql.SqlUtils.ColumnInfo; + +public class JsonFormatTest { + + @Test + public void formatTo() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + BufferedOutputStream bout = new BufferedOutputStream(out); + ResultSet results = SqlUtils.resultSet(new ColumnInfo[] { + new ColumnInfo("ID", Types.INTEGER), + new ColumnInfo("HOST", Types.VARCHAR), + new ColumnInfo("IS_CONTROLLER", Types.BOOLEAN), + }, + new Object[] { 1, "host1.com", true }, + new Object[] { 2, "host2.com", false }, + new Object[] { 3, "host3.com", false }); + new JsonFormat().formatTo(results, bout); + bout.flush(); + String expected = "[{\"ID\":1,\"HOST\":\"host1.com\",\"IS_CONTROLLER\":true}," + + "{\"ID\":2,\"HOST\":\"host2.com\",\"IS_CONTROLLER\":false}," + + "{\"ID\":3,\"HOST\":\"host3.com\",\"IS_CONTROLLER\":false}]\n"; + assertEquals(expected, new String(out.toByteArray())); + } +} diff --git a/src/test/java/kmql/format/TableFormatTest.java b/src/test/java/kmql/format/TableFormatTest.java new file mode 100644 index 0000000..865cff6 --- /dev/null +++ b/src/test/java/kmql/format/TableFormatTest.java @@ -0,0 +1,42 @@ +package kmql.format; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.sql.ResultSet; +import java.sql.Types; + +import org.junit.Test; + +import kmql.SqlUtils; +import kmql.SqlUtils.ColumnInfo; + +public class TableFormatTest { + + @Test + public void formatTo() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + BufferedOutputStream bout = new BufferedOutputStream(out); + ResultSet results = SqlUtils.resultSet(new ColumnInfo[] { + new ColumnInfo("ID", Types.INTEGER), + new ColumnInfo("HOST", Types.VARCHAR), + new ColumnInfo("IS_CONTROLLER", Types.BOOLEAN), + }, + new Object[] { 1, "host1.com", true }, + new Object[] { 2, "host2.com", false }, + new Object[] { 3, "host3.com", false }); + new TableFormat().formatTo(results, bout); + bout.flush(); + String expected = "╔════╤═══════════╤═══════════════╗\n" + + "║ ID │ HOST │ IS_CONTROLLER ║\n" + + "╠════╪═══════════╪═══════════════╣\n" + + "║ 1 │ host1.com │ true ║\n" + + "╟────┼───────────┼───────────────╢\n" + + "║ 2 │ host2.com │ false ║\n" + + "╟────┼───────────┼───────────────╢\n" + + "║ 3 │ host3.com │ false ║\n" + + "╚════╧═══════════╧═══════════════╝\n"; + assertEquals(expected, new String(out.toByteArray())); + } +}