fix lockup on errs on every row (#151)

* fix lockup on errs on every row

* add default support and unit tests

* fix globreader file stats and misc errors

.gitignore

@@ -13,7 +13,7 @@ apps/workers/deduper/deduper
 apps/workers/sort2file/sort2file
 apps/utils/file-watcher/file-watcher
 */stats/stats
-*/sql-load/sql-load
+apps/workers/sql-load/sql-load
 build
 
 coverage

apps/workers/sql-load/main.go

@@ -19,7 +19,6 @@ type options struct {
 	MySQL    bootstrap.DBOptions `toml:"mysql"`
 
 	sqlDB *sql.DB
-	//	sqlxDB *sqlx.DB // used for running direct exec command
 
 	producer bus.Producer
 	fileOpts *file.Options
@@ -38,14 +37,15 @@ delete : allows insert into pre-existing data by deleting previous data.
     - "?delete=date:2020-07-01|id:7"
 truncate: allows insert into pre-existing table by truncating before insertion
 fields : allows mapping different json key values to different database column names
-    - provide a list of field name mapping {json key name}:{DB column name} to be mapped 
-    - ?fields=jsonKey:dbColumnName
-
+    - provide a list of field name mapping {DB column name}:{json key name} to be mapped 
+    - ?fields=dbColumnName:jsonkey
+cached_insert: improves insert times by caching data into a temp table
+batch_size: (default:1000) number of rows to insert at a time (higher number increases memory usage) 
 Example task:
  
 {"type":"sql_load","info":"gs://bucket/path/to/file.json?table=schema.table_name&delete=date:2020-07-01|id:7"}
 {"type":"sql_load","info":"gs://bucket/path/of/files/to/load/?table=schema.table_name"}
-{"type":"sql_load","info":"gs://bucket/path/to/file.json?table=schema.table_name&delete=date:2020-07-01|id:7&fields=jsonKeyValue:dbColumnName"}`
+{"type":"sql_load","info":"gs://bucket/path/to/file.json?table=schema.table_name&delete=date:2020-07-01|id:7&fields=dbColumnName:jsonKeyValue"}`
 )
 
 func init() {