Merge pull request #9 from zendesk/ben/multiple_rows

change JSON output to what the binlog looks like.

src/main/java/com/zendesk/maxwell/MaxwellAbstractRowsEvent.java

@@ -154,24 +154,22 @@ public String toSQL() {
 
 	class RowMap extends HashMap<String, Object> {
 		public RowMap() {
-			this.put("data", new HashMap<String, Object>());
+			this.put("data", new ArrayList<HashMap<String, Object>>());
 		}
 
 		public void setRowType(String type) {
 			this.put("type", type);
 		}
 
 		@SuppressWarnings("unchecked")
-		public Map<String, Object> data() {
-			return (Map<String, Object>) this.get("data");
+		public List<Map<String, Object>> data() {
+			return (List<Map<String, Object>>) this.get("data");
 		}
 
-		@SuppressWarnings("unchecked")
-		private HashMap<String, Object> getDataHash() {
-			return (HashMap<String, Object>) this.get("data");
-		}
-		public void setColumn(String key, Object value) {
-			getDataHash().put(key, value);
+		public Map<String, Object> addRow() {
+			Map<String, Object> row = new HashMap<String, Object>();
+			this.data().add(row);
+			return row;
 		}
 
 		public void setTable(String name) {
@@ -180,16 +178,17 @@ public void setTable(String name) {
 	}
 	// the equivalent of "asJSON" -- convert the row to an array of
 	// hashes
-	public List<RowMap> jsonMaps() {
-		List<RowMap> list = new ArrayList<>();
+	public RowMap jsonMap() {
+		RowMap rowMap = new RowMap();
+
+		rowMap.setRowType(getType());
+		rowMap.setTable(getTable().getName());
 
 		for ( Row r : filteredRows()) {
-			RowMap rowMap = new RowMap();
 			Iterator<Column> colIter = r.getColumns().iterator();
 			Iterator<ColumnDef> defIter = table.getColumnList().iterator();
 
-			rowMap.setRowType(getType());
-			rowMap.setTable(getTable().getName());
+			Map<String, Object> jsonRow = rowMap.addRow();
 			while ( colIter.hasNext() && defIter.hasNext() ) {
 				Column c = colIter.next();
 				ColumnDef d = defIter.next();
@@ -198,28 +197,19 @@ public List<RowMap> jsonMaps() {
 				if ( value != null )
 					value = d.asJSON(value);
 
-				rowMap.setColumn(d.getName(), value);
+				jsonRow.put(d.getName(), value);
 			}
-			list.add(rowMap);
 		}
 
-		return list;
+		return rowMap;
 	}
 
-	public List<JSONObject> toJSONObjectList() {
-		ArrayList<JSONObject> a = new ArrayList<>();
-		for ( Map<String, Object> row : jsonMaps() ) {
-			a.add(new MaxwellJSONObject(row));
-		}
-		return a;
+	public JSONObject toJSONObject() {
+		return new MaxwellJSONObject(jsonMap());
 	}
 
 	// tbd -- is an array of rows really best?
 	public String toJSON() throws IOException {
-		JSONArray a = new JSONArray();
-		for ( Map<String, Object> row : jsonMaps() ) {
-			a.put(new MaxwellJSONObject(row));
-		}
-		return a.toString();
+		return toJSONObject().toString();
 	}
 }

src/main/java/com/zendesk/maxwell/MaxwellJSONObject.java

@@ -7,8 +7,10 @@
 
 import org.json.JSONObject;
 
+import com.zendesk.maxwell.MaxwellAbstractRowsEvent.RowMap;
+
 public class MaxwellJSONObject extends JSONObject {
-	public MaxwellJSONObject(Map<String, Object> map) {
+	public MaxwellJSONObject(RowMap map) {
 		super(map);
 	}
 

src/test/java/com/zendesk/maxwell/MysqlParserTest.java

@@ -39,10 +39,10 @@ public void testRowFilter() throws Exception {
 		list = getRowsForSQL(filter, input);
 		assertThat(list.size(), is(1));
 
-		RowMap jsonMap = list.get(0).jsonMaps().get(0);
+		RowMap jsonMap = list.get(0).jsonMap();
 
-		assertThat((Long) jsonMap.data().get("account_id"), is(2L));
-		assertThat((String) jsonMap.data().get("text_field"), is("goodbye"));
+		assertThat((Long) jsonMap.data().get(0).get("account_id"), is(2L));
+		assertThat((String) jsonMap.data().get(0).get("text_field"), is("goodbye"));
 	}
 
 	@Test
@@ -151,29 +151,29 @@ public void testAlterTable() throws Exception {
 		assertThat(e.getTable().getName(), is("minimal"));
 	}
 
-	private void runJSONTest(List<String> sql, List<JSONObject> json) throws Exception {
+	private void runJSONTest(List<String> sql, List<JSONObject> assertJSON) throws Exception {
 		List<JSONObject> eventJSON = new ArrayList<>();
 		List<JSONObject> matched = new ArrayList<>();
 		List<MaxwellAbstractRowsEvent> events = getRowsForSQL(null, sql.toArray(new String[0]));
 
 		for ( MaxwellAbstractRowsEvent e : events ) {
-			for ( JSONObject a : e.toJSONObjectList() ) {
-				eventJSON.add(a);
+			JSONObject a = e.toJSONObject();
 
-				for ( JSONObject b : json ) {
-					if ( JSONCompare.compare(a.toString(), b.toString()) )
-						matched.add(b);
-				}
+			eventJSON.add(a);
+
+			for ( JSONObject b : assertJSON ) {
+				if ( JSONCompare.compare(a.toString(), b.toString()) )
+					matched.add(b);
 			}
 		}
 
 		for ( JSONObject j : matched ) {
-			json.remove(j);
+			assertJSON.remove(j);
 		}
 
-		if ( json.size() > 0 ) {
+		if ( assertJSON.size() > 0 ) {
 			String msg = "Did not find: \n" +
-						 StringUtils.join(json.iterator(), "\n") +
+						 StringUtils.join(assertJSON.iterator(), "\n") +
 						 "\n\n in : " +
 						 StringUtils.join(eventJSON.iterator(), "\n");
 			assertThat(msg, false, is(true));

src/test/resources/sql/json/test_1j

@@ -4,24 +4,24 @@ CREATE TABLE `test_db_1`.`test_table` ( id bigint(20) not null auto_increment pr
 use test_db_1
 insert into test_table set textcol='test_col_1'
 
-->  {"table": "test_table", "type": "insert", "data": { "id": 1, "textcol": "test_col_1"} }
+->  {"table": "test_table", "type": "insert", "data": [ {"id": 1, "textcol": "test_col_1"} ] }
 
 insert into test_table set textcol='test_col_2'
 
-->  {"table": "test_table", "type": "insert", "data": { "id": 2, "textcol": "test_col_2"} }
+->  {"table": "test_table", "type": "insert", "data": [ { "id": 2, "textcol": "test_col_2"} ] }
 
 alter table test_table add column `datecol` datetime NULL default null AFTER id
 
 insert into test_table set textcol='test_col_2', datecol='1979-10-01 00:00:00'
 
--> {"table": "test_table", "type": "insert", "data": { "id": 3, "textcol": "test_col_2", "datecol": "1979-10-01 00:00:00"} }
+-> {"table": "test_table", "type": "insert", "data": [ { "id": 3, "textcol": "test_col_2", "datecol": "1979-10-01 00:00:00"} ] }
 
 update test_table set textcol='test_col_5' where id = 1
 
--> {"table": "test_table", "type": "update", "data": {"id": 1, "textcol": "test_col_5"} }
+-> {"table": "test_table", "type": "update", "data": [ {"id": 1, "textcol": "test_col_5"} ] }
 
-delete from test_table where id = 2;
+delete from test_table;
 
--> {"table": "test_table", "type": "delete", "data": {"id": 2, "textcol": "test_col_2"} }
+-> {"table": "test_table", "type": "delete", "data": [  {"id": 1, "textcol": "test_col_5"}, {"id": 2, "textcol": "test_col_2"}, { "id": 3, "textcol": "test_col_2", "datecol": "1979-10-01 00:00:00"} ] }
 
 DROP DATABASE `test_db_1`;

src/test/resources/sql/json/test_create_like

@@ -3,6 +3,6 @@ CREATE TABLE `test_db`.`test_alike` ( id int(11) auto_increment not null primary
 CREATE TABLE `test_db`.`test_alike2` LIKE `test_db`.`test_alike`
 
 insert into test_db.test_alike2 set flt = 1.5;
--> {"table": "test_alike2", "type": "insert", "data": { "id" : 1, "flt": 1.5 } }
+-> {"table": "test_alike2", "type": "insert", "data": [ { "id" : 1, "flt": 1.5 } ] }
 
 

src/test/resources/sql/json/test_enum

@@ -2,9 +2,9 @@ create database enum_db;
 use enum_db;
 create table test_enums(id int(10) unsigned not null auto_increment primary key, enum_field enum('foo', 'bar', 'baz'));
 insert into test_enums set enum_field='bar';
-  -> {table: "test_enums", type: "insert", data: {"id": 1, enum_field: "bar"} }
+  -> {table: "test_enums", type: "insert", data: [ {"id": 1, enum_field: "bar"} ] }
 insert into test_enums set enum_field='baz';
-  -> {table: "test_enums", type: "insert", data: {"id": 2, enum_field: "baz"} }
+  -> {table: "test_enums", type: "insert", data: [ {"id": 2, enum_field: "baz"} ] }
 insert into test_enums set enum_field='foo';
-  -> {table: "test_enums", type: "insert", data: {"id": 3, enum_field: "foo"} }
+  -> {table: "test_enums", type: "insert", data: [ {"id": 3, enum_field: "foo"} ] }
 

src/test/resources/sql/json/test_latin1

@@ -2,4 +2,4 @@ CREATE DATABASE `test_latin1` default charset 'latin1'
 create table `test_latin1`.`latin1` ( textcol text )
 use test_latin1
 insert into `latin1` set `textcol` = 'áááááááá';
--> {"table": "latin1", "type": "insert", "data": {"textcol": "áááááááá"}}
+-> {"table": "latin1", "type": "insert", "data": [ {"textcol": "áááááááá"} ] }

src/test/resources/sql/json/test_set

@@ -2,13 +2,13 @@ create database set_db;
 use set_db;
 create table test_sets(id int(10) unsigned not null auto_increment primary key, set_field set('foo', 'bar', 'baz') NOT NULL);
 insert into test_sets set set_field='bar,baz';
-  -> {table: "test_sets", type: "insert", data: {"id": 1, set_field: ["bar", "baz"]} }
+  -> {table: "test_sets", type: "insert", data: [{"id": 1, set_field: ["bar", "baz"]}] }
 insert into test_sets set set_field='baz,foo';
-  -> {table: "test_sets", type: "insert", data: {"id": 2, set_field: ["foo", "baz"]} }
+  -> {table: "test_sets", type: "insert", data: [{"id": 2, set_field: ["foo", "baz"]}] }
 insert into test_sets set set_field='foo';
-  -> {table: "test_sets", type: "insert", data: {"id": 3, set_field: ["foo"]} }
+  -> {table: "test_sets", type: "insert", data: [{"id": 3, set_field: ["foo"]}] }
 insert into test_sets set set_field='';
-  -> {table: "test_sets", type: "insert", data: {"id": 4, set_field: []} }
+  -> {table: "test_sets", type: "insert", data: [{"id": 4, set_field: []}] }
 alter table test_sets modify column set_field set('foo', 'bar', 'baz') NULL DEFAULT NULL;
 insert into test_sets set id=NULL;
-  -> {table: "test_sets", type: "insert", data: {"id": 5} }
+  -> {table: "test_sets", type: "insert", data: [{"id": 5}] }