Skip to content

Commit

Permalink
change JSON output to what the binlog looks like.
Browse files Browse the repository at this point in the history
this changes the data format output from

```
[
 {table: 'foo', data: {'bar': 'baz'},
 {table: 'foo', data: {'bar': 'zab'}
]
```

to

```
{
  table: 'foo',
  data: [ {'bar': 'baz'},
          {'bar': 'zab'} ]
}
```
  • Loading branch information
Ben Osheroff committed Mar 5, 2015
1 parent 0318ba5 commit 9d18b39
Show file tree
Hide file tree
Showing 8 changed files with 50 additions and 58 deletions.
46 changes: 18 additions & 28 deletions src/main/java/com/zendesk/maxwell/MaxwellAbstractRowsEvent.java
Original file line number Diff line number Diff line change
Expand Up @@ -154,24 +154,22 @@ public String toSQL() {

class RowMap extends HashMap<String, Object> {
public RowMap() {
this.put("data", new HashMap<String, Object>());
this.put("data", new ArrayList<HashMap<String, Object>>());
}

public void setRowType(String type) {
this.put("type", type);
}

@SuppressWarnings("unchecked")
public Map<String, Object> data() {
return (Map<String, Object>) this.get("data");
public List<Map<String, Object>> data() {
return (List<Map<String, Object>>) this.get("data");
}

@SuppressWarnings("unchecked")
private HashMap<String, Object> getDataHash() {
return (HashMap<String, Object>) this.get("data");
}
public void setColumn(String key, Object value) {
getDataHash().put(key, value);
public Map<String, Object> addRow() {
Map<String, Object> row = new HashMap<String, Object>();
this.data().add(row);
return row;
}

public void setTable(String name) {
Expand All @@ -180,16 +178,17 @@ public void setTable(String name) {
}
// the equivalent of "asJSON" -- convert the row to an array of
// hashes
public List<RowMap> jsonMaps() {
List<RowMap> list = new ArrayList<>();
public RowMap jsonMap() {
RowMap rowMap = new RowMap();

rowMap.setRowType(getType());
rowMap.setTable(getTable().getName());

for ( Row r : filteredRows()) {
RowMap rowMap = new RowMap();
Iterator<Column> colIter = r.getColumns().iterator();
Iterator<ColumnDef> defIter = table.getColumnList().iterator();

rowMap.setRowType(getType());
rowMap.setTable(getTable().getName());
Map<String, Object> jsonRow = rowMap.addRow();
while ( colIter.hasNext() && defIter.hasNext() ) {
Column c = colIter.next();
ColumnDef d = defIter.next();
Expand All @@ -198,28 +197,19 @@ public List<RowMap> jsonMaps() {
if ( value != null )
value = d.asJSON(value);

rowMap.setColumn(d.getName(), value);
jsonRow.put(d.getName(), value);
}
list.add(rowMap);
}

return list;
return rowMap;
}

public List<JSONObject> toJSONObjectList() {
ArrayList<JSONObject> a = new ArrayList<>();
for ( Map<String, Object> row : jsonMaps() ) {
a.add(new MaxwellJSONObject(row));
}
return a;
public JSONObject toJSONObject() {
return new MaxwellJSONObject(jsonMap());
}

// tbd -- is an array of rows really best?
public String toJSON() throws IOException {
JSONArray a = new JSONArray();
for ( Map<String, Object> row : jsonMaps() ) {
a.put(new MaxwellJSONObject(row));
}
return a.toString();
return toJSONObject().toString();
}
}
4 changes: 3 additions & 1 deletion src/main/java/com/zendesk/maxwell/MaxwellJSONObject.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@

import org.json.JSONObject;

import com.zendesk.maxwell.MaxwellAbstractRowsEvent.RowMap;

public class MaxwellJSONObject extends JSONObject {
public MaxwellJSONObject(Map<String, Object> map) {
public MaxwellJSONObject(RowMap map) {
super(map);
}

Expand Down
26 changes: 13 additions & 13 deletions src/test/java/com/zendesk/maxwell/MysqlParserTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ public void testRowFilter() throws Exception {
list = getRowsForSQL(filter, input);
assertThat(list.size(), is(1));

RowMap jsonMap = list.get(0).jsonMaps().get(0);
RowMap jsonMap = list.get(0).jsonMap();

assertThat((Long) jsonMap.data().get("account_id"), is(2L));
assertThat((String) jsonMap.data().get("text_field"), is("goodbye"));
assertThat((Long) jsonMap.data().get(0).get("account_id"), is(2L));
assertThat((String) jsonMap.data().get(0).get("text_field"), is("goodbye"));
}

@Test
Expand Down Expand Up @@ -151,29 +151,29 @@ public void testAlterTable() throws Exception {
assertThat(e.getTable().getName(), is("minimal"));
}

private void runJSONTest(List<String> sql, List<JSONObject> json) throws Exception {
private void runJSONTest(List<String> sql, List<JSONObject> assertJSON) throws Exception {
List<JSONObject> eventJSON = new ArrayList<>();
List<JSONObject> matched = new ArrayList<>();
List<MaxwellAbstractRowsEvent> events = getRowsForSQL(null, sql.toArray(new String[0]));

for ( MaxwellAbstractRowsEvent e : events ) {
for ( JSONObject a : e.toJSONObjectList() ) {
eventJSON.add(a);
JSONObject a = e.toJSONObject();

for ( JSONObject b : json ) {
if ( JSONCompare.compare(a.toString(), b.toString()) )
matched.add(b);
}
eventJSON.add(a);

for ( JSONObject b : assertJSON ) {
if ( JSONCompare.compare(a.toString(), b.toString()) )
matched.add(b);
}
}

for ( JSONObject j : matched ) {
json.remove(j);
assertJSON.remove(j);
}

if ( json.size() > 0 ) {
if ( assertJSON.size() > 0 ) {
String msg = "Did not find: \n" +
StringUtils.join(json.iterator(), "\n") +
StringUtils.join(assertJSON.iterator(), "\n") +
"\n\n in : " +
StringUtils.join(eventJSON.iterator(), "\n");
assertThat(msg, false, is(true));
Expand Down
12 changes: 6 additions & 6 deletions src/test/resources/sql/json/test_1j
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,24 @@ CREATE TABLE `test_db_1`.`test_table` ( id bigint(20) not null auto_increment pr
use test_db_1
insert into test_table set textcol='test_col_1'

-> {"table": "test_table", "type": "insert", "data": { "id": 1, "textcol": "test_col_1"} }
-> {"table": "test_table", "type": "insert", "data": [ {"id": 1, "textcol": "test_col_1"} ] }

insert into test_table set textcol='test_col_2'

-> {"table": "test_table", "type": "insert", "data": { "id": 2, "textcol": "test_col_2"} }
-> {"table": "test_table", "type": "insert", "data": [ { "id": 2, "textcol": "test_col_2"} ] }

alter table test_table add column `datecol` datetime NULL default null AFTER id

insert into test_table set textcol='test_col_2', datecol='1979-10-01 00:00:00'

-> {"table": "test_table", "type": "insert", "data": { "id": 3, "textcol": "test_col_2", "datecol": "1979-10-01 00:00:00"} }
-> {"table": "test_table", "type": "insert", "data": [ { "id": 3, "textcol": "test_col_2", "datecol": "1979-10-01 00:00:00"} ] }

update test_table set textcol='test_col_5' where id = 1

-> {"table": "test_table", "type": "update", "data": {"id": 1, "textcol": "test_col_5"} }
-> {"table": "test_table", "type": "update", "data": [ {"id": 1, "textcol": "test_col_5"} ] }

delete from test_table where id = 2;
delete from test_table;

-> {"table": "test_table", "type": "delete", "data": {"id": 2, "textcol": "test_col_2"} }
-> {"table": "test_table", "type": "delete", "data": [ {"id": 1, "textcol": "test_col_5"}, {"id": 2, "textcol": "test_col_2"}, { "id": 3, "textcol": "test_col_2", "datecol": "1979-10-01 00:00:00"} ] }

DROP DATABASE `test_db_1`;
2 changes: 1 addition & 1 deletion src/test/resources/sql/json/test_create_like
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@ CREATE TABLE `test_db`.`test_alike` ( id int(11) auto_increment not null primary
CREATE TABLE `test_db`.`test_alike2` LIKE `test_db`.`test_alike`

insert into test_db.test_alike2 set flt = 1.5;
-> {"table": "test_alike2", "type": "insert", "data": { "id" : 1, "flt": 1.5 } }
-> {"table": "test_alike2", "type": "insert", "data": [ { "id" : 1, "flt": 1.5 } ] }


6 changes: 3 additions & 3 deletions src/test/resources/sql/json/test_enum
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ create database enum_db;
use enum_db;
create table test_enums(id int(10) unsigned not null auto_increment primary key, enum_field enum('foo', 'bar', 'baz'));
insert into test_enums set enum_field='bar';
-> {table: "test_enums", type: "insert", data: {"id": 1, enum_field: "bar"} }
-> {table: "test_enums", type: "insert", data: [ {"id": 1, enum_field: "bar"} ] }
insert into test_enums set enum_field='baz';
-> {table: "test_enums", type: "insert", data: {"id": 2, enum_field: "baz"} }
-> {table: "test_enums", type: "insert", data: [ {"id": 2, enum_field: "baz"} ] }
insert into test_enums set enum_field='foo';
-> {table: "test_enums", type: "insert", data: {"id": 3, enum_field: "foo"} }
-> {table: "test_enums", type: "insert", data: [ {"id": 3, enum_field: "foo"} ] }

2 changes: 1 addition & 1 deletion src/test/resources/sql/json/test_latin1
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ CREATE DATABASE `test_latin1` default charset 'latin1'
create table `test_latin1`.`latin1` ( textcol text )
use test_latin1
insert into `latin1` set `textcol` = 'áááááááá';
-> {"table": "latin1", "type": "insert", "data": {"textcol": "áááááááá"}}
-> {"table": "latin1", "type": "insert", "data": [ {"textcol": "áááááááá"} ] }
10 changes: 5 additions & 5 deletions src/test/resources/sql/json/test_set
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ create database set_db;
use set_db;
create table test_sets(id int(10) unsigned not null auto_increment primary key, set_field set('foo', 'bar', 'baz') NOT NULL);
insert into test_sets set set_field='bar,baz';
-> {table: "test_sets", type: "insert", data: {"id": 1, set_field: ["bar", "baz"]} }
-> {table: "test_sets", type: "insert", data: [{"id": 1, set_field: ["bar", "baz"]}] }
insert into test_sets set set_field='baz,foo';
-> {table: "test_sets", type: "insert", data: {"id": 2, set_field: ["foo", "baz"]} }
-> {table: "test_sets", type: "insert", data: [{"id": 2, set_field: ["foo", "baz"]}] }
insert into test_sets set set_field='foo';
-> {table: "test_sets", type: "insert", data: {"id": 3, set_field: ["foo"]} }
-> {table: "test_sets", type: "insert", data: [{"id": 3, set_field: ["foo"]}] }
insert into test_sets set set_field='';
-> {table: "test_sets", type: "insert", data: {"id": 4, set_field: []} }
-> {table: "test_sets", type: "insert", data: [{"id": 4, set_field: []}] }
alter table test_sets modify column set_field set('foo', 'bar', 'baz') NULL DEFAULT NULL;
insert into test_sets set id=NULL;
-> {table: "test_sets", type: "insert", data: {"id": 5} }
-> {table: "test_sets", type: "insert", data: [{"id": 5}] }

0 comments on commit 9d18b39

Please sign in to comment.