diff --git a/pom.xml b/pom.xml index 384e991..5f7722c 100644 --- a/pom.xml +++ b/pom.xml @@ -11,9 +11,9 @@ https://github.com/alipay/rdf-file - 2.2.5 - 2.2.5 - 2.2.5 + 2.2.6 + 2.2.6 + 2.2.6 diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/AbstractColumnInfoCodec.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/AbstractColumnInfoCodec.java new file mode 100644 index 0000000..2a31074 --- /dev/null +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/AbstractColumnInfoCodec.java @@ -0,0 +1,55 @@ +/** + * Alipay.com Inc. + * Copyright (c) 2004-2019 All Rights Reserved. + */ +package com.alipay.rdf.file.codec; + +import com.alipay.rdf.file.exception.RdfErrorEnum; +import com.alipay.rdf.file.exception.RdfFileException; +import com.alipay.rdf.file.loader.TemplateLoader; +import com.alipay.rdf.file.meta.FileColumnMeta; +import com.alipay.rdf.file.meta.FileMeta; +import com.alipay.rdf.file.model.FileConfig; +import com.alipay.rdf.file.model.FileDataTypeEnum; + +import java.util.List; + +/** + * Copyright (C) 2013-2018 Ant Financial Services Group + * + * @author quhongwei + * @version : AbstractColumnInfoCodec.java, v 0.1 2019年11月29日 12:55 quhongwei Exp $ + */ +public abstract class AbstractColumnInfoCodec { + + protected static String getValue(FileColumnMeta colMeta, String method) { + String value; + if ("desc".equalsIgnoreCase(method)) { + value = colMeta.getDesc(); + } else if ("name".equalsIgnoreCase(method)) { + value = colMeta.getName(); + } else { + throw new RdfFileException( + "rdf-file#AbstractColumnInfoCodec.getValue 无效方法参数method=" + method, + RdfErrorEnum.UNSUPPORTED_OPERATION); + } + return value; + } + + protected static List getColumnMetas(FileConfig config, FileDataTypeEnum dataType) { + FileMeta fileMeta = TemplateLoader.load(config); + switch (dataType) { + case HEAD: + return fileMeta.getHeadColumns(); + case BODY: + return fileMeta.getBodyColumns(); + case TAIL: + return fileMeta.getTailColumns(); + default: + throw new RdfFileException( + "rdf-file#AbstractColumnInfoCodec.getColumnMetas dateType=" + dataType.name(), + RdfErrorEnum.UNSUPPORTED_OPERATION); + } + } + +} \ No newline at end of file diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnHorizontalCodec.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnHorizontalCodec.java index a1625b3..09bd64e 100644 --- a/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnHorizontalCodec.java +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnHorizontalCodec.java @@ -23,6 +23,7 @@ * @author hongwei.quhw * @version $Id: HeadColumnCodec.java, v 0.1 2017-1-3 下午5:50:00 hongwei.quhw Exp $ */ +@Deprecated public class BodyColumnHorizontalCodec { /** diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnVerticalCodec.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnVerticalCodec.java index 2ecb2b5..a94d67f 100644 --- a/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnVerticalCodec.java +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/BodyColumnVerticalCodec.java @@ -18,6 +18,7 @@ * @author hongwei.quhw * @version $Id: HeadColumnCodec.java, v 0.1 2017-1-3 下午5:50:00 hongwei.quhw Exp $ */ +@Deprecated public class BodyColumnVerticalCodec { public static final BodyColumnVerticalCodec instance = new BodyColumnVerticalCodec(); diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/ColumnInfoHorizontalCodec.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/ColumnInfoHorizontalCodec.java new file mode 100644 index 0000000..f9d27b9 --- /dev/null +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/ColumnInfoHorizontalCodec.java @@ -0,0 +1,83 @@ +package com.alipay.rdf.file.codec; + +import com.alipay.rdf.file.exception.RdfErrorEnum; +import com.alipay.rdf.file.exception.RdfFileException; +import com.alipay.rdf.file.interfaces.FileReader; +import com.alipay.rdf.file.interfaces.FileWriter; +import com.alipay.rdf.file.loader.ProtocolLoader; +import com.alipay.rdf.file.loader.TemplateLoader; +import com.alipay.rdf.file.meta.FileColumnMeta; +import com.alipay.rdf.file.meta.FileMeta; +import com.alipay.rdf.file.model.FileConfig; +import com.alipay.rdf.file.model.FileDataTypeEnum; +import com.alipay.rdf.file.spi.RdfFileRowSplitSpi.SplitContext; +import com.alipay.rdf.file.util.RdfFileUtil; + +import java.util.List; + +/** + * Copyright (C) 2013-2018 Ant Financial Services Group + * + * 字段信息水平编码解码 + * + * @author hongwei.quhw + * @version $Id: ColumnInfoHorizontalCodec.java, v 0.1 2017-1-3 下午5:50:00 hongwei.quhw Exp $ + */ +public class ColumnInfoHorizontalCodec extends AbstractColumnInfoCodec { + + public static void serialize(FileDataTypeEnum layoutType, FileDataTypeEnum dataType, FileConfig config, FileWriter writer, + String method) { + FileMeta fileMeta = TemplateLoader.load(config); + List colMetas = getColumnMetas(config, dataType); + StringBuilder colHead = new StringBuilder(); + + if (null != RdfFileUtil.getRowSplit(config) && fileMeta.isStartWithSplit(layoutType)) { + colHead.append(RdfFileUtil.getRowSplit(config)); + } + + for (int i = 0; i < colMetas.size(); i++) { + FileColumnMeta colMeta = colMetas.get(i); + colHead.append(getValue(colMeta, method)); + //添加字段分割符 + if (null != RdfFileUtil.getRowSplit(config) && (i < colMetas.size() - 1 || fileMeta.isEndWithSplit(layoutType))) { + colHead.append(RdfFileUtil.getRowSplit(config)); + } + } + writer.writeLine(colHead.toString()); + + } + + public static T deserialize(FileDataTypeEnum layoutType, FileDataTypeEnum dataType, FileConfig config, + FileReader reader, + String method) { + String line = reader.readLine(); + RdfFileUtil.assertNotBlank(line, "文件=" + config.getFilePath() + ", " + layoutType.name() + " 内容缺失"); + + FileMeta fileMeta = TemplateLoader.load(config); + String[] columns = ProtocolLoader.loadProtocol(fileMeta.getProtocol()).getRowSplit().split( + new SplitContext(line, config, FileDataTypeEnum.BODY)); + List colMetas = getColumnMetas(config, dataType); + + int splitLength = fileMeta.isStartWithSplit(layoutType) ? colMetas.size() + 1 : colMetas.size(); + splitLength = fileMeta.isEndWithSplit(layoutType) ? splitLength + 1 : splitLength; + + if (splitLength != columns.length) { + throw new RdfFileException("文件=" + config.getFilePath() + "， " + layoutType.name() + " line=" + line, + RdfErrorEnum.DESERIALIZE_ERROR); + } + + int statIndex = fileMeta.isStartWithSplit(layoutType) ? 1 : 0; + int endIndex = fileMeta.isEndWithSplit(layoutType) ? columns.length - 1 : columns.length; + + for (int i = statIndex; i < endIndex; i++) { + FileColumnMeta colMeta = colMetas.get(i - statIndex); + if (!getValue(colMeta, method).equals(columns[i])) { + throw new RdfFileException( + "文件" + layoutType.name() + "字段校验：文件模板定义的第" + i + "个column为[" + colMetas.get(i).getDesc() + "], 实际文件中为[" + + columns[i] + "]", RdfErrorEnum.DESERIALIZE_ERROR); + } + } + return null; + } + +} diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/ColumnInfoVerticalCodec.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/ColumnInfoVerticalCodec.java new file mode 100644 index 0000000..4c37114 --- /dev/null +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/ColumnInfoVerticalCodec.java @@ -0,0 +1,47 @@ +package com.alipay.rdf.file.codec; + +import com.alipay.rdf.file.exception.RdfErrorEnum; +import com.alipay.rdf.file.exception.RdfFileException; +import com.alipay.rdf.file.interfaces.FileReader; +import com.alipay.rdf.file.interfaces.FileWriter; +import com.alipay.rdf.file.meta.FileColumnMeta; +import com.alipay.rdf.file.model.FileConfig; +import com.alipay.rdf.file.model.FileDataTypeEnum; +import com.alipay.rdf.file.util.RdfFileUtil; + +/** + * Copyright (C) 2013-2018 Ant Financial Services Group + * + * 字段信息纵向编码解码 + * + * @author hongwei.quhw + * @version $Id: ColumnInfoVerticalCodec.java, v 0.1 2017-1-3 下午5:50:00 hongwei.quhw Exp $ + */ +public class ColumnInfoVerticalCodec extends AbstractColumnInfoCodec { + public static final ColumnInfoVerticalCodec instance = new ColumnInfoVerticalCodec(); + + public static void serialize(FileDataTypeEnum layoutType, FileDataTypeEnum dataType, FileConfig fileConfig, + FileWriter writer, String method) { + //按行写入column字段 + for (FileColumnMeta colMeta : getColumnMetas(fileConfig, dataType)) { + writer.writeLine(getValue(colMeta, method)); + } + } + + public static T deserialize(FileDataTypeEnum layoutType, FileDataTypeEnum dataType, FileConfig fileConfig, + FileReader reader, String method) { + for (FileColumnMeta colMeta : getColumnMetas(fileConfig, dataType)) { + String columName = RdfFileUtil.assertTrimNotBlank(reader.readLine()); + RdfFileUtil.assertNotBlank(columName, "文件=" + fileConfig.getFilePath() + ", " + layoutType.name() + " 内容缺失"); + + String tempalteValue = getValue(colMeta, method); + if (!tempalteValue.equals(columName)) { + throw new RdfFileException( + "rdf-file#" + layoutType.name() + "模板中定义的column为" + tempalteValue + ", 文件中读取的column为" + columName + " 不一致", + RdfErrorEnum.VALIDATE_ERROR); + + } + } + return null; + } +} diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/RowsCodec.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/RowsCodec.java index 7b1f8b8..b55a0c0 100644 --- a/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/RowsCodec.java +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/codec/RowsCodec.java @@ -141,6 +141,6 @@ public static FileColumnMeta getColumnMeta(FileColumnMeta colMeta, FileMeta file return new FileColumnMeta(colMeta.getColIndex(), colMeta.getName(), colMeta.getDesc(), colMeta.getType(), colMeta.isRequired(), colMeta.getRange(), colMeta.getDefaultValue(), - fileMeta); + fileMeta, colMeta.getDataType()); } } diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/function/BodyColumnFunction.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/function/BodyColumnFunction.java index fb41bdf..7c6dab2 100644 --- a/rdf-file-core/src/main/java/com/alipay/rdf/file/function/BodyColumnFunction.java +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/function/BodyColumnFunction.java @@ -19,10 +19,13 @@ * Copyright (C) 2013-2018 Ant Financial Services Group * * body 字段 - * + * + * 使用更通用的ColumnInfoFunction函数功能替换 + * * @author hongwei.quhw * @version $Id: BodyColumnFunction.java, v 0.1 2017年8月19日 下午1:21:38 hongwei.quhw Exp $ */ +@Deprecated @SuppressWarnings("rawtypes") public class BodyColumnFunction extends RdfFunction { @Override diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/function/ColumnFunctionWrapper.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/function/ColumnFunctionWrapper.java index 772b68f..3c36821 100644 --- a/rdf-file-core/src/main/java/com/alipay/rdf/file/function/ColumnFunctionWrapper.java +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/function/ColumnFunctionWrapper.java @@ -127,7 +127,7 @@ private String serialize(Object field, FileColumnMeta columnMeta, FileConfig fil } private ColumnRegEx getColumnRegEx(FileColumnMeta columnMeta) { - String key = columnMeta.getFileMeta().getTemplatePath() + "-" + columnMeta.getName(); + String key = columnMeta.getFileMeta().getTemplatePath() + "-" + columnMeta.getDataType().name() + "-" + columnMeta.getName(); ColumnRegEx columnRegEx = columnRegExs.get(key); if (null == columnRegEx) { columnRegEx = new ColumnRegEx(); diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/function/ColumnInfoFunction.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/function/ColumnInfoFunction.java new file mode 100644 index 0000000..3284a7b --- /dev/null +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/function/ColumnInfoFunction.java @@ -0,0 +1,139 @@ +package com.alipay.rdf.file.function; + +import com.alipay.rdf.file.codec.ColumnInfoHorizontalCodec; +import com.alipay.rdf.file.codec.ColumnInfoVerticalCodec; +import com.alipay.rdf.file.exception.RdfErrorEnum; +import com.alipay.rdf.file.exception.RdfFileException; +import com.alipay.rdf.file.loader.ExtensionLoader; +import com.alipay.rdf.file.loader.FormatLoader; +import com.alipay.rdf.file.loader.TemplateLoader; +import com.alipay.rdf.file.meta.FileMeta; +import com.alipay.rdf.file.model.FileDataTypeEnum; +import com.alipay.rdf.file.protocol.RowDefinition; +import com.alipay.rdf.file.spi.RdfFileColumnTypeSpi; +import com.alipay.rdf.file.spi.RdfFileFormatSpi; +import com.alipay.rdf.file.util.RdfFileUtil; + +/** + * Copyright (C) 2013-2018 Ant Financial Services Group + * + * 字段信息序列化反序列化函数 + * + * 参考fund.xml协议 + * + * 1. ${columnInfo.count(body)} 计算body的字段数 + * 2. ${columnInfo.vertical(body,name)} 每个body字段名作为一行 + * 3. ${columnInfo.horizontal(tail,name)} 所有文件尾字段名作为一行数据 + * + * 函数第一个参数代表对文件数据模板中哪个部分的字段进行信息处理 + * 函数第二个参数代表对文件数据模板中name或者desc字段处理 + * + * @author quhongwei + * @version : ColumnInfoFunction.java, v 0.1 2019年11月29日 15:50 quhongwei Exp $ + */ +public class ColumnInfoFunction extends RdfFunction { + @Override + public void checkParams() { + if (("horizontal".equals(expression) || "vertical".equals(expression)) && (null == params || params.length != 2)) { + throw new RdfFileException("rdf-file#ColumnInfoFunction.checkParams() 指定的参数应该为两个", RdfErrorEnum.FUNCTION_ERROR); + } else if ("count".equals(expression) && (null == params || params.length != 1)) { + throw new RdfFileException("rdf-file#ColumnInfoFunction.checkParams() 指定的参数应该为一个", RdfErrorEnum.FUNCTION_ERROR); + } + } + + @Override + public int rowsAffected(RowDefinition rd, FileMeta fileMeta) { + if ("horizontal".equals(expression)) { + return 1; + } else if ("vertical".equals(expression)) { + return getColumnSize(fileMeta); + } else if ("count".equals(expression)) { + return 1; + } else { + throw new RdfFileException( + "rdf-file#ColumnInfoFunction" + expression + ", 无法计算rowsAffected", + RdfErrorEnum.UNSUPPORTED_OPERATION); + } + } + + public void horizontal(FuncContext ctx) { + FileDataTypeEnum dataType = FileDataTypeEnum.valueOf(params[0].toUpperCase()); + if (CodecType.SERIALIZE.equals(ctx.codecType)) { + ColumnInfoHorizontalCodec.serialize(rowType, dataType, ctx.fileConfig, ctx.writer, params[1]); + + } else if (CodecType.DESERIALIZE.equals(ctx.codecType)) { + ColumnInfoHorizontalCodec.deserialize(rowType, dataType, ctx.fileConfig, ctx.reader, params[1]); + } + } + + public void vertical(FuncContext ctx) { + FileDataTypeEnum dataType = FileDataTypeEnum.valueOf(params[0].toUpperCase()); + + if (CodecType.SERIALIZE.equals(ctx.codecType)) { + ColumnInfoVerticalCodec.serialize(rowType, dataType, ctx.fileConfig, ctx.writer, params[1]); + } else if (CodecType.DESERIALIZE.equals(ctx.codecType)) { + ColumnInfoVerticalCodec.deserialize(rowType, dataType, ctx.fileConfig, ctx.reader, params[1]); + } + } + + public void count(FuncContext ctx) { + FileMeta fileMeta = TemplateLoader.load(ctx.fileConfig); + String typeName = ctx.columnMeta.getType().getName(); + + RdfFileFormatSpi columnFormat = FormatLoader.getColumnFormt(fileMeta.getProtocol(), + typeName); + RdfFileUtil.assertNotNull(columnFormat, "类型type=" + typeName + " 对应的format没有"); + RdfFileColumnTypeSpi columnTypeCodec = ExtensionLoader + .getExtensionLoader(RdfFileColumnTypeSpi.class).getExtension(typeName); + RdfFileUtil.assertNotNull(columnTypeCodec, "没有type=" + typeName + " 对应的类型codec"); + + boolean startWithSplit = null != RdfFileUtil.getRowSplit(ctx.fileConfig) && fileMeta.isStartWithSplit(rowType); + boolean endtWithSplit = null != RdfFileUtil.getRowSplit(ctx.fileConfig) && fileMeta.isEndWithSplit(rowType); + + switch (ctx.codecType) { + case SERIALIZE: + StringBuilder line = new StringBuilder(); + if (startWithSplit) { + line.append(RdfFileUtil.getRowSplit(ctx.fileConfig)); + } + line.append(getColumnSize(fileMeta)); + if (endtWithSplit) { + line.append(RdfFileUtil.getRowSplit(ctx.fileConfig)); + } + ctx.writer.writeLine(columnFormat.serialize(line.toString(), ctx.columnMeta, ctx.fileConfig)); + break; + case DESERIALIZE: + String value = ctx.reader.readLine(); + if (startWithSplit) { + value = value.substring(RdfFileUtil.getRowSplit(ctx.fileConfig).length()); + } + if (endtWithSplit) { + value = value.substring(0, value.length() - RdfFileUtil.getRowSplit(ctx.fileConfig).length()); + } + Object field = columnTypeCodec.deserialize(value, ctx.columnMeta); + RdfFileUtil.assertEquals(field.toString(), + String.valueOf(fileMeta.getBodyColumns().size())); + break; + default: + throw new RdfFileException("不支持序列化反序列化类型" + ctx.codecType.name(), + RdfErrorEnum.UNSUPPORTED_OPERATION); + } + } + + private int getColumnSize(FileMeta fileMeta) { + FileDataTypeEnum dataType = FileDataTypeEnum.valueOf(params[0].toUpperCase()); + switch (dataType) { + case HEAD: + return fileMeta.getHeadColumns().size(); + case BODY: + return fileMeta.getBodyColumns().size(); + case TAIL: + return fileMeta.getTailColumns().size(); + default: + throw new RdfFileException( + "rdf-file#ColumnInfoFunction.rowsAffected dateType=" + dataType.name(), + RdfErrorEnum.UNSUPPORTED_OPERATION); + } + } + +} \ No newline at end of file diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/loader/TemplateLoader.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/loader/TemplateLoader.java index 4aa37ec..44a0ac8 100644 --- a/rdf-file-core/src/main/java/com/alipay/rdf/file/loader/TemplateLoader.java +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/loader/TemplateLoader.java @@ -1,9 +1,5 @@ package com.alipay.rdf.file.loader; -import java.io.InputStream; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - import com.alibaba.fastjson.JSON; import com.alipay.rdf.file.condition.RowConditionType; import com.alipay.rdf.file.exception.RdfErrorEnum; @@ -23,6 +19,10 @@ import com.alipay.rdf.file.model.RowCondition; import com.alipay.rdf.file.util.RdfFileUtil; +import java.io.InputStream; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + /** * Copyright (C) 2013-2018 Ant Financial Services Group * @@ -41,7 +41,7 @@ public class TemplateLoader { /** * 计算行总长度 * - * @param fileMeta + * @param fileConfig * @return */ public static int getRowLength(FileConfig fileConfig) { @@ -72,9 +72,9 @@ public static FileMeta load(FileConfig fileConfig) { /** * 加载模板 * - * @param template + * @param templatePath + * @param templateEncoding * @param templateEncoding - * @param fileEncoding * @return */ public static FileMeta load(String templatePath, String templateEncoding) { @@ -110,7 +110,7 @@ public static FileMeta load(String templatePath, String templateEncoding) { int colIndex = 0; //head for (String head : templateConfig.getHead()) { - fileMeta.addHeadColumn(parseFileColumn(templatePath, head, colIndex++, fileMeta)); + fileMeta.addHeadColumn(parseFileColumn(templatePath, head, colIndex++, fileMeta, FileDataTypeEnum.HEAD)); } // body 配置多模板解析 @@ -127,7 +127,7 @@ public static FileMeta load(String templatePath, String templateEncoding) { colIndex = 0; for (String body : bodyConfig.getBodyColumns()) { bodyMeta.getColumns() - .add(parseFileColumn(templatePath, body, colIndex++, fileMeta)); + .add(parseFileColumn(templatePath, body, colIndex++, fileMeta, FileDataTypeEnum.BODY)); } fileMeta.addBodyColumn(bodyMeta); @@ -154,7 +154,7 @@ public static FileMeta load(String templatePath, String templateEncoding) { FileBodyMeta bodyMeta = new FileBodyMeta(); for (String body : templateConfig.getBody()) { bodyMeta.getColumns() - .add(parseFileColumn(templatePath, body, colIndex++, fileMeta)); + .add(parseFileColumn(templatePath, body, colIndex++, fileMeta, FileDataTypeEnum.BODY)); } fileMeta.addBodyColumn(bodyMeta); } @@ -162,7 +162,7 @@ public static FileMeta load(String templatePath, String templateEncoding) { colIndex = 0; //tail for (String tail : templateConfig.getTail()) { - fileMeta.addTailColumn(parseFileColumn(templatePath, tail, colIndex++, fileMeta)); + fileMeta.addTailColumn(parseFileColumn(templatePath, tail, colIndex++, fileMeta, FileDataTypeEnum.TAIL)); } //解析汇总字段 @@ -232,7 +232,7 @@ private static void parseEndWithSplit(FileMeta fileMeta, String startSplit) { } private static FileColumnMeta parseFileColumn(String tempaltePath, String colConfig, - int colIndex, FileMeta fileMeta) { + int colIndex, FileMeta fileMeta, FileDataTypeEnum dataType) { colConfig = RdfFileUtil.assertTrimNotBlank(colConfig, "字段为空 index=" + colIndex); String[] fields = colConfig.trim().split("\\|"); @@ -304,7 +304,7 @@ private static FileColumnMeta parseFileColumn(String tempaltePath, String colCon } FileColumnMeta column = new FileColumnMeta(colIndex, key, name, type, required, range, - defaultValue, fileMeta); + defaultValue, fileMeta, dataType); return column; } diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/meta/FileColumnMeta.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/meta/FileColumnMeta.java index b79d551..0d57081 100644 --- a/rdf-file-core/src/main/java/com/alipay/rdf/file/meta/FileColumnMeta.java +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/meta/FileColumnMeta.java @@ -1,5 +1,7 @@ package com.alipay.rdf.file.meta; +import com.alipay.rdf.file.model.FileDataTypeEnum; + /** * Copyright (C) 2013-2018 Ant Financial Services Group * @@ -24,20 +26,12 @@ public class FileColumnMeta { /**默认值配置*/ private final String defaultValue; private final FileMeta fileMeta; + /**数据字段在文件中部位*/ + private final FileDataTypeEnum dataType; - /** - * @param colIndex - * @param name - * @param desc - * @param type - * @param required - * @param range - * @param defaultValue - * @param fileMeta - */ public FileColumnMeta(int colIndex, String name, String desc, FileColumnTypeMeta type, boolean required, FileColumnRangeMeta range, String defaultValue, - FileMeta fileMeta) { + FileMeta fileMeta, FileDataTypeEnum dataType) { this.colIndex = colIndex; this.desc = desc; this.name = name; @@ -46,6 +40,7 @@ public FileColumnMeta(int colIndex, String name, String desc, FileColumnTypeMeta this.required = required; this.defaultValue = defaultValue; this.fileMeta = fileMeta; + this.dataType = dataType; } /** @@ -120,6 +115,10 @@ public FileMeta getFileMeta() { return fileMeta; } + public FileDataTypeEnum getDataType() { + return dataType; + } + @Override public String toString() { StringBuffer sb = new StringBuffer("FileColumnMeta["); @@ -129,6 +128,7 @@ public String toString() { sb.append(",type=").append(type.getName()); sb.append(",range=").append(range); sb.append(",required=").append(required); + sb.append(",dataType=").append(dataType.name()); sb.append("]"); return sb.toString(); } diff --git a/rdf-file-core/src/main/java/com/alipay/rdf/file/protocol/FileDefinitionParser.java b/rdf-file-core/src/main/java/com/alipay/rdf/file/protocol/FileDefinitionParser.java index fcfa2ff..c8f0ac6 100644 --- a/rdf-file-core/src/main/java/com/alipay/rdf/file/protocol/FileDefinitionParser.java +++ b/rdf-file-core/src/main/java/com/alipay/rdf/file/protocol/FileDefinitionParser.java @@ -144,7 +144,7 @@ private void parseRow(String xpath, Element node, List rowDefinit } FileColumnMeta columnMeta = new FileColumnMeta(-1, output, output, columnTypeMeta, - false, columnRangeMeta, null, null); + false, columnRangeMeta, null, null, rowType); rd.setColumnMeta(columnMeta); } } diff --git a/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/cm.xml b/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/cm.xml new file mode 100644 index 0000000..c63b615 --- /dev/null +++ b/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/cm.xml @@ -0,0 +1,26 @@ + + + + + + ${column.value()} + + + + + + + + ${column.value()} + + + + + + + + ${column.value()} + + + + \ No newline at end of file diff --git a/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/de.xml b/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/de.xml index eb07e90..21d6c58 100644 --- a/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/de.xml +++ b/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/de.xml @@ -7,7 +7,7 @@ ${column.value()} - + diff --git a/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/fund.xml b/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/fund.xml index a52e2f0..e65776a 100644 --- a/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/fund.xml +++ b/rdf-file-core/src/main/resources/META-INF/rdf-file/protocol/fund.xml @@ -5,8 +5,8 @@ ${column.value()} - - + + diff --git a/rdf-file-core/src/main/resources/META-INF/rdf-file/services/com.alipay.rdf.file.spi.RdfFileFunctionSpi b/rdf-file-core/src/main/resources/META-INF/rdf-file/services/com.alipay.rdf.file.spi.RdfFileFunctionSpi index 89dc104..06d7602 100644 --- a/rdf-file-core/src/main/resources/META-INF/rdf-file/services/com.alipay.rdf.file.spi.RdfFileFunctionSpi +++ b/rdf-file-core/src/main/resources/META-INF/rdf-file/services/com.alipay.rdf.file.spi.RdfFileFunctionSpi @@ -1,2 +1,3 @@ column=com.alipay.rdf.file.function.ColumnFunction -bodycolumn=com.alipay.rdf.file.function.BodyColumnFunction \ No newline at end of file +bodycolumn=com.alipay.rdf.file.function.BodyColumnFunction +columnInfo=com.alipay.rdf.file.function.ColumnInfoFunction \ No newline at end of file diff --git a/rdf-file-core/src/test/java/com/alipay/rdf/file/cache/CacheManagerTest.java b/rdf-file-core/src/test/java/com/alipay/rdf/file/cache/CacheManagerTest.java index 5e4b52c..9903838 100644 --- a/rdf-file-core/src/test/java/com/alipay/rdf/file/cache/CacheManagerTest.java +++ b/rdf-file-core/src/test/java/com/alipay/rdf/file/cache/CacheManagerTest.java @@ -5,6 +5,7 @@ import java.util.HashMap; import java.util.Map; +import org.junit.Ignore; import org.junit.Test; import com.alipay.rdf.file.common.CacheManager; @@ -23,6 +24,7 @@ public class CacheManagerTest { + @Ignore @Test public void testRemoveDataTempalteCache() { diff --git a/rdf-file-core/src/test/java/com/alipay/rdf/file/loader/ProtocolLoaderTest.java b/rdf-file-core/src/test/java/com/alipay/rdf/file/loader/ProtocolLoaderTest.java index 992cd16..edd0c4f 100644 --- a/rdf-file-core/src/test/java/com/alipay/rdf/file/loader/ProtocolLoaderTest.java +++ b/rdf-file-core/src/test/java/com/alipay/rdf/file/loader/ProtocolLoaderTest.java @@ -1,13 +1,7 @@ package com.alipay.rdf.file.loader; -import java.util.List; -import java.util.Map; - -import org.junit.Assert; -import org.junit.Test; - -import com.alipay.rdf.file.function.BodyColumnFunction; import com.alipay.rdf.file.function.ColumnFunctionWrapper; +import com.alipay.rdf.file.function.ColumnInfoFunction; import com.alipay.rdf.file.function.VariableFunction; import com.alipay.rdf.file.model.FileConfig; import com.alipay.rdf.file.model.StorageConfig; @@ -18,10 +12,15 @@ import com.alipay.rdf.file.spi.RdfFileProcessorSpi; import com.alipay.rdf.file.split.RowSplitByFixedlLength; import com.alipay.rdf.file.split.RowSplitBySeparator; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.Map; /** - * - * + * + * * @author hongwei.quhw * @version $Id: ProtocolLoaderTest.java, v 0.1 2017年4月7日 上午11:02:24 hongwei.quhw Exp $ */ @@ -32,35 +31,35 @@ public void testLoadFund() { ProtocolDefinition pd = ProtocolLoader.loadProtocol("fund"); Assert.assertEquals("fund", pd.getName()); Assert.assertEquals(RowSplitByFixedlLength.class.getName(), - pd.getRowSplit().getClass().getName()); + pd.getRowSplit().getClass().getName()); List headRDs = pd.getHeads(); Assert.assertTrue(headRDs.get(0).isColumnloop()); Assert.assertEquals(ColumnLayoutEnum.vertical, headRDs.get(0).getColumnLayout()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - headRDs.get(0).getOutput().getClass().getName()); + headRDs.get(0).getOutput().getClass().getName()); Assert.assertFalse(headRDs.get(1).isColumnloop()); Assert.assertNull(headRDs.get(1).getColumnLayout()); - Assert.assertEquals(BodyColumnFunction.class.getName(), - headRDs.get(1).getOutput().getClass().getName()); + Assert.assertEquals(ColumnInfoFunction.class.getName(), + headRDs.get(1).getOutput().getClass().getName()); Assert.assertEquals("count", headRDs.get(1).getOutput().getExpression()); Assert.assertFalse(headRDs.get(2).isColumnloop()); Assert.assertNull(headRDs.get(2).getColumnLayout()); - Assert.assertEquals(BodyColumnFunction.class.getName(), - headRDs.get(2).getOutput().getClass().getName()); + Assert.assertEquals(ColumnInfoFunction.class.getName(), + headRDs.get(2).getOutput().getClass().getName()); Assert.assertEquals("vertical", headRDs.get(2).getOutput().getExpression()); Assert.assertFalse(headRDs.get(3).isColumnloop()); Assert.assertNull(headRDs.get(3).getColumnLayout()); Assert.assertEquals(VariableFunction.class.getName(), - headRDs.get(3).getOutput().getClass().getName()); + headRDs.get(3).getOutput().getClass().getName()); List bodyRDs = pd.getBodys(); Assert.assertEquals(1, bodyRDs.size()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - bodyRDs.get(0).getOutput().getClass().getName()); + bodyRDs.get(0).getOutput().getClass().getName()); Assert.assertEquals(ColumnLayoutEnum.horizontal, bodyRDs.get(0).getColumnLayout()); @@ -68,7 +67,7 @@ public void testLoadFund() { Assert.assertTrue(tailRDs.get(0).isColumnloop()); Assert.assertEquals(ColumnLayoutEnum.horizontal, tailRDs.get(0).getColumnLayout()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - tailRDs.get(0).getOutput().getClass().getName()); + tailRDs.get(0).getOutput().getClass().getName()); } @Test @@ -77,23 +76,23 @@ public void testLoadDE() { Assert.assertEquals("de", pd.getName()); Assert.assertEquals(RowSplitBySeparator.class.getName(), - pd.getRowSplit().getClass().getName()); + pd.getRowSplit().getClass().getName()); List headRDs = pd.getHeads(); Assert.assertTrue(headRDs.get(0).isColumnloop()); Assert.assertEquals(ColumnLayoutEnum.horizontal, headRDs.get(0).getColumnLayout()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - headRDs.get(0).getOutput().getClass().getName()); + headRDs.get(0).getOutput().getClass().getName()); Assert.assertFalse(headRDs.get(1).isColumnloop()); Assert.assertNull(headRDs.get(1).getColumnLayout()); - Assert.assertEquals(BodyColumnFunction.class.getName(), - headRDs.get(1).getOutput().getClass().getName()); + Assert.assertEquals(ColumnInfoFunction.class.getName(), + headRDs.get(1).getOutput().getClass().getName()); List bodyRDs = pd.getBodys(); Assert.assertEquals(1, bodyRDs.size()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - bodyRDs.get(0).getOutput().getClass().getName()); + bodyRDs.get(0).getOutput().getClass().getName()); Assert.assertEquals(ColumnLayoutEnum.horizontal, bodyRDs.get(0).getColumnLayout()); @@ -101,18 +100,18 @@ public void testLoadDE() { Assert.assertTrue(tailRDs.get(0).isColumnloop()); Assert.assertEquals(ColumnLayoutEnum.horizontal, tailRDs.get(0).getColumnLayout()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - tailRDs.get(0).getOutput().getClass().getName()); + tailRDs.get(0).getOutput().getClass().getName()); } @Test public void testOrder() { FileConfig fileConfig = new FileConfig("filePaht", "/reader/de/template/template1.json", - new StorageConfig("nas")); + new StorageConfig("nas")); fileConfig.addProcessorKey("processor1"); fileConfig.addProcessorKey("processor2"); fileConfig.addProcessorKey("processor3"); Map> ret = ProcessorLoader - .loadByType(fileConfig, ProcessorTypeEnum.AFTER_CLOSE_WRITER); + .loadByType(fileConfig, ProcessorTypeEnum.AFTER_CLOSE_WRITER); List proces = ret.get(ProcessorTypeEnum.AFTER_CLOSE_WRITER); Assert.assertEquals(3, proces.size()); diff --git a/rdf-file-core/src/test/java/com/alipay/rdf/file/protocol/FileDefinitionParserTest.java b/rdf-file-core/src/test/java/com/alipay/rdf/file/protocol/FileDefinitionParserTest.java index a16bd45..14accab 100644 --- a/rdf-file-core/src/test/java/com/alipay/rdf/file/protocol/FileDefinitionParserTest.java +++ b/rdf-file-core/src/test/java/com/alipay/rdf/file/protocol/FileDefinitionParserTest.java @@ -1,17 +1,17 @@ package com.alipay.rdf.file.protocol; -import java.io.InputStream; -import java.util.List; - -import org.junit.Assert; -import org.junit.Test; - import com.alipay.rdf.file.function.BodyColumnFunction; import com.alipay.rdf.file.function.ColumnFunctionWrapper; +import com.alipay.rdf.file.function.ColumnInfoFunction; import com.alipay.rdf.file.function.VariableFunction; import com.alipay.rdf.file.loader.ResourceLoader; import com.alipay.rdf.file.split.RowSplitByFixedlLength; import com.alipay.rdf.file.split.RowSplitBySeparator; +import org.junit.Assert; +import org.junit.Test; + +import java.io.InputStream; +import java.util.List; public class FileDefinitionParserTest { @@ -22,35 +22,35 @@ public void testParseFund() { ProtocolDefinition pd = new FileDefinitionParser().parseFileDefinition(is); Assert.assertEquals("fund", pd.getName()); Assert.assertEquals(RowSplitByFixedlLength.class.getName(), - pd.getRowSplit().getClass().getName()); + pd.getRowSplit().getClass().getName()); List headRDs = pd.getHeads(); Assert.assertTrue(headRDs.get(0).isColumnloop()); Assert.assertEquals(ColumnLayoutEnum.vertical, headRDs.get(0).getColumnLayout()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - headRDs.get(0).getOutput().getClass().getName()); + headRDs.get(0).getOutput().getClass().getName()); Assert.assertFalse(headRDs.get(1).isColumnloop()); Assert.assertNull((headRDs.get(1).getColumnLayout())); - Assert.assertEquals(BodyColumnFunction.class.getName(), - headRDs.get(1).getOutput().getClass().getName()); + Assert.assertEquals(ColumnInfoFunction.class.getName(), + headRDs.get(1).getOutput().getClass().getName()); Assert.assertEquals("count", headRDs.get(1).getOutput().getExpression()); Assert.assertFalse(headRDs.get(2).isColumnloop()); Assert.assertNull(headRDs.get(2).getColumnLayout()); - Assert.assertEquals(BodyColumnFunction.class.getName(), - headRDs.get(2).getOutput().getClass().getName()); + Assert.assertEquals(ColumnInfoFunction.class.getName(), + headRDs.get(2).getOutput().getClass().getName()); Assert.assertEquals("vertical", headRDs.get(2).getOutput().getExpression()); Assert.assertFalse(headRDs.get(3).isColumnloop()); Assert.assertNull(headRDs.get(3).getColumnLayout()); Assert.assertEquals(VariableFunction.class.getName(), - headRDs.get(3).getOutput().getClass().getName()); + headRDs.get(3).getOutput().getClass().getName()); List bodyRDs = pd.getBodys(); Assert.assertEquals(1, bodyRDs.size()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - bodyRDs.get(0).getOutput().getClass().getName()); + bodyRDs.get(0).getOutput().getClass().getName()); Assert.assertEquals(ColumnLayoutEnum.horizontal, bodyRDs.get(0).getColumnLayout()); @@ -58,7 +58,7 @@ public void testParseFund() { Assert.assertTrue(tailRDs.get(0).isColumnloop()); Assert.assertEquals(ColumnLayoutEnum.horizontal, tailRDs.get(0).getColumnLayout()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - tailRDs.get(0).getOutput().getClass().getName()); + tailRDs.get(0).getOutput().getClass().getName()); } @Test @@ -69,23 +69,23 @@ public void testParseDE() { Assert.assertEquals("de", pd.getName()); Assert.assertEquals(RowSplitBySeparator.class.getName(), - pd.getRowSplit().getClass().getName()); + pd.getRowSplit().getClass().getName()); List headRDs = pd.getHeads(); Assert.assertTrue(headRDs.get(0).isColumnloop()); Assert.assertEquals(ColumnLayoutEnum.horizontal, headRDs.get(0).getColumnLayout()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - headRDs.get(0).getOutput().getClass().getName()); + headRDs.get(0).getOutput().getClass().getName()); Assert.assertFalse(headRDs.get(1).isColumnloop()); Assert.assertNull(headRDs.get(1).getColumnLayout()); - Assert.assertEquals(BodyColumnFunction.class.getName(), - headRDs.get(1).getOutput().getClass().getName()); + Assert.assertEquals(ColumnInfoFunction.class.getName(), + headRDs.get(1).getOutput().getClass().getName()); List bodyRDs = pd.getBodys(); Assert.assertEquals(1, bodyRDs.size()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - bodyRDs.get(0).getOutput().getClass().getName()); + bodyRDs.get(0).getOutput().getClass().getName()); Assert.assertEquals(ColumnLayoutEnum.horizontal, bodyRDs.get(0).getColumnLayout()); @@ -93,6 +93,6 @@ public void testParseDE() { Assert.assertTrue(tailRDs.get(0).isColumnloop()); Assert.assertEquals(ColumnLayoutEnum.horizontal, tailRDs.get(0).getColumnLayout()); Assert.assertEquals(ColumnFunctionWrapper.class.getName(), - tailRDs.get(0).getOutput().getClass().getName()); + tailRDs.get(0).getOutput().getClass().getName()); } } diff --git a/rdf-file-core/src/test/java/com/alipay/rdf/file/reader/FileReaderCmExtTest.java b/rdf-file-core/src/test/java/com/alipay/rdf/file/reader/FileReaderCmExtTest.java new file mode 100644 index 0000000..8904a4f --- /dev/null +++ b/rdf-file-core/src/test/java/com/alipay/rdf/file/reader/FileReaderCmExtTest.java @@ -0,0 +1,146 @@ +package com.alipay.rdf.file.reader; + +import com.alipay.rdf.file.interfaces.FileFactory; +import com.alipay.rdf.file.interfaces.FileReader; +import com.alipay.rdf.file.model.FileConfig; +import com.alipay.rdf.file.model.StorageConfig; +import com.alipay.rdf.file.util.DateUtil; +import com.alipay.rdf.file.util.TemporaryFolderUtil; +import junit.framework.Assert; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * @author hongwei.quhw + * @version $Id: FileReaderCmTest.java, v 0.1 2017年4月7日 下午5:38:15 hongwei.quhw Exp $ + */ +public class FileReaderCmExtTest { + TemporaryFolderUtil tf = new TemporaryFolderUtil(); + + @Before + public void setUp() throws IOException { + tf.create(); + } + + /** + * 测试行前后带分隔符场景 + */ + @Test + public void testReadCMExtFile() { + String filePath = File.class.getResource("/reader/cm/data/data4.txt").getPath(); + + FileConfig config = new FileConfig(filePath, "/reader/cm/template/template4.json", + new StorageConfig("nas")); + + FileReader fileReader = FileFactory.createReader(config); + + Map head = fileReader.readHead(HashMap.class); + Assert.assertEquals(new Long(100), head.get("totalCount")); + Assert.assertEquals(new BigDecimal("300.03"), head.get("totalAmount")); + + Map row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_0", row.get("seq")); + Assert.assertEquals("inst_seq_0", row.get("instSeq")); + Assert.assertEquals("2013-11-09 12:34:56", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131109", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131112 12:23:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.33"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("10.22"), row.get("amount")); + Assert.assertEquals(new Integer(22), row.get("age")); + Assert.assertEquals(new Long(12345), row.get("longN")); + Assert.assertEquals(Boolean.TRUE, row.get("bol")); + Assert.assertEquals("备注1", row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_1", row.get("seq")); + Assert.assertEquals("inst_seq_1", row.get("instSeq")); + Assert.assertEquals("2013-11-10 15:56:12", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131110", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131113 12:33:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.34"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("11.88"), row.get("amount")); + Assert.assertEquals(new Integer(33), row.get("age")); + Assert.assertEquals(new Long(56789), row.get("longN")); + Assert.assertEquals(Boolean.FALSE, row.get("bol")); + Assert.assertNull(row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertNull(row); + + Map tail = fileReader.readTail(HashMap.class); + Assert.assertEquals("endFile", tail.get("fileEnd")); + + fileReader.close(); + } + + + @Test + public void testReadCMExtFile2() { + String filePath = File.class.getResource("/reader/cm/data/data5.txt").getPath(); + + FileConfig config = new FileConfig(filePath, "/reader/cm/template/template5.json", + new StorageConfig("nas")); + + FileReader fileReader = FileFactory.createReader(config); + + Map head = fileReader.readHead(HashMap.class); + Assert.assertEquals(new Long(100), head.get("totalCount")); + Assert.assertEquals(new BigDecimal("300.03"), head.get("totalAmount")); + + Map row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_0", row.get("seq")); + Assert.assertEquals("inst_seq_0", row.get("instSeq")); + Assert.assertEquals("2013-11-09 12:34:56", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131109", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131112 12:23:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.33"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("10.22"), row.get("amount")); + Assert.assertEquals(new Integer(22), row.get("age")); + Assert.assertEquals(new Long(12345), row.get("longN")); + Assert.assertEquals(Boolean.TRUE, row.get("bol")); + Assert.assertEquals("备注1", row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_1", row.get("seq")); + Assert.assertEquals("inst_seq_1", row.get("instSeq")); + Assert.assertEquals("2013-11-10 15:56:12", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131110", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131113 12:33:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.34"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("11.88"), row.get("amount")); + Assert.assertEquals(new Integer(33), row.get("age")); + Assert.assertEquals(new Long(56789), row.get("longN")); + Assert.assertEquals(Boolean.FALSE, row.get("bol")); + Assert.assertNull(row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertNull(row); + + Map tail = fileReader.readTail(HashMap.class); + Assert.assertEquals("endFile", tail.get("fileEnd")); + + fileReader.close(); + } + + + @After + public void after() { + tf.delete(); + } +} diff --git a/rdf-file-core/src/test/java/com/alipay/rdf/file/reader/FileReaderCmTest.java b/rdf-file-core/src/test/java/com/alipay/rdf/file/reader/FileReaderCmTest.java new file mode 100644 index 0000000..7cd0515 --- /dev/null +++ b/rdf-file-core/src/test/java/com/alipay/rdf/file/reader/FileReaderCmTest.java @@ -0,0 +1,201 @@ +package com.alipay.rdf.file.reader; + +import com.alipay.rdf.file.interfaces.FileFactory; +import com.alipay.rdf.file.interfaces.FileReader; +import com.alipay.rdf.file.model.FileConfig; +import com.alipay.rdf.file.model.FileDefaultConfig; +import com.alipay.rdf.file.model.StorageConfig; +import com.alipay.rdf.file.util.DateUtil; +import com.alipay.rdf.file.util.TemporaryFolderUtil; +import com.alipay.rdf.file.util.TestLog; +import junit.framework.Assert; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * @author hongwei.quhw + * @version $Id: FileReaderCmTest.java, v 0.1 2017年4月7日 下午5:38:15 hongwei.quhw Exp $ + */ +public class FileReaderCmTest { + TemporaryFolderUtil tf = new TemporaryFolderUtil(); + + @Before + public void setUp() throws IOException { + tf.create(); + } + + /** + * 测试行前后带分隔符场景 + */ + @Test + public void testReadCMFile3() { + String filePath = File.class.getResource("/reader/cm/data/data3.txt").getPath(); + + FileConfig config = new FileConfig(filePath, "/reader/cm/template/template3.json", + new StorageConfig("nas")); + + FileReader fileReader = FileFactory.createReader(config); + + Map head = fileReader.readHead(HashMap.class); + Assert.assertEquals(new Long(100), head.get("totalCount")); + Assert.assertEquals(new BigDecimal("300.03"), head.get("totalAmount")); + + Map row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_0", row.get("seq")); + Assert.assertEquals("inst_seq_0", row.get("instSeq")); + Assert.assertEquals("2013-11-09 12:34:56", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131109", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131112 12:23:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.33"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("10.22"), row.get("amount")); + Assert.assertEquals(new Integer(22), row.get("age")); + Assert.assertEquals(new Long(12345), row.get("longN")); + Assert.assertEquals(Boolean.TRUE, row.get("bol")); + Assert.assertEquals("备注1", row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_1", row.get("seq")); + Assert.assertEquals("inst_seq_1", row.get("instSeq")); + Assert.assertEquals("2013-11-10 15:56:12", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131110", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131113 12:33:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.34"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("11.88"), row.get("amount")); + Assert.assertEquals(new Integer(33), row.get("age")); + Assert.assertEquals(new Long(56789), row.get("longN")); + Assert.assertEquals(Boolean.FALSE, row.get("bol")); + Assert.assertNull(row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertNull(row); + + Map tail = fileReader.readTail(HashMap.class); + Assert.assertEquals("endFile", tail.get("fileEnd")); + + fileReader.close(); + } + + /** + * 测试行前后带分隔符场景 + */ + @Test + public void testReadCMFile2() { + String filePath = File.class.getResource("/reader/cm/data/data2.txt").getPath(); + + FileConfig config = new FileConfig(filePath, "/reader/cm/template/template2.json", + new StorageConfig("nas")); + + FileReader fileReader = FileFactory.createReader(config); + + Map head = fileReader.readHead(HashMap.class); + Assert.assertEquals(new Long(100), head.get("totalCount")); + Assert.assertEquals(new BigDecimal("300.03"), head.get("totalAmount")); + + Map row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_0", row.get("seq")); + Assert.assertEquals("inst_seq_0", row.get("instSeq")); + Assert.assertEquals("2013-11-09 12:34:56", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131109", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131112 12:23:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.33"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("10.22"), row.get("amount")); + Assert.assertEquals(new Integer(22), row.get("age")); + Assert.assertEquals(new Long(12345), row.get("longN")); + Assert.assertEquals(Boolean.TRUE, row.get("bol")); + Assert.assertEquals("备注1", row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_1", row.get("seq")); + Assert.assertEquals("inst_seq_1", row.get("instSeq")); + Assert.assertEquals("2013-11-10 15:56:12", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131110", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131113 12:33:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.34"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("11.88"), row.get("amount")); + Assert.assertEquals(new Integer(33), row.get("age")); + Assert.assertEquals(new Long(56789), row.get("longN")); + Assert.assertEquals(Boolean.FALSE, row.get("bol")); + Assert.assertNull(row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertNull(row); + + Map tail = fileReader.readTail(HashMap.class); + Assert.assertEquals("endFile", tail.get("fileEnd")); + + fileReader.close(); + } + + @Test + public void testReadCMFile() throws Exception { + String filePath = File.class.getResource("/reader/cm/data/data1.txt").getPath(); + + FileConfig config = new FileConfig(filePath, "/reader/cm/template/template1.json", + new StorageConfig("nas")); + + FileReader fileReader = FileFactory.createReader(config); + + Map head = fileReader.readHead(HashMap.class); + Assert.assertEquals(new Long(100), head.get("totalCount")); + Assert.assertEquals(new BigDecimal("300.03"), head.get("totalAmount")); + + Map row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_0", row.get("seq")); + Assert.assertEquals("inst_seq_0", row.get("instSeq")); + Assert.assertEquals("2013-11-09 12:34:56", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131109", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131112 12:23:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.33"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("10.22"), row.get("amount")); + Assert.assertEquals(new Integer(22), row.get("age")); + Assert.assertEquals(new Long(12345), row.get("longN")); + Assert.assertEquals(Boolean.TRUE, row.get("bol")); + Assert.assertEquals("备注1", row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_1", row.get("seq")); + Assert.assertEquals("inst_seq_1", row.get("instSeq")); + Assert.assertEquals("2013-11-10 15:56:12", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131110", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131113 12:33:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.34"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("11.88"), row.get("amount")); + Assert.assertEquals(new Integer(33), row.get("age")); + Assert.assertEquals(new Long(56789), row.get("longN")); + Assert.assertEquals(Boolean.FALSE, row.get("bol")); + Assert.assertNull(row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertNull(row); + + Map tail = fileReader.readTail(HashMap.class); + Assert.assertEquals("endFile", tail.get("fileEnd")); + + fileReader.close(); + } + + @After + public void after() { + tf.delete(); + } +} diff --git a/rdf-file-core/src/test/java/com/alipay/rdf/file/reader/FileReaderDe2Test.java b/rdf-file-core/src/test/java/com/alipay/rdf/file/reader/FileReaderDe2Test.java new file mode 100644 index 0000000..721d69a --- /dev/null +++ b/rdf-file-core/src/test/java/com/alipay/rdf/file/reader/FileReaderDe2Test.java @@ -0,0 +1,92 @@ +package com.alipay.rdf.file.reader; + +import com.alipay.rdf.file.interfaces.FileFactory; +import com.alipay.rdf.file.interfaces.FileReader; +import com.alipay.rdf.file.model.FileConfig; +import com.alipay.rdf.file.model.StorageConfig; +import com.alipay.rdf.file.util.DateUtil; +import com.alipay.rdf.file.util.TemporaryFolderUtil; +import junit.framework.Assert; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * @author hongwei.quhw + * @version $Id: FileReaderDeTest.java, v 0.1 2017年4月7日 下午5:38:15 hongwei.quhw Exp $ + */ +public class FileReaderDe2Test { + TemporaryFolderUtil tf = new TemporaryFolderUtil(); + + @Before + public void setUp() throws IOException { + tf.create(); + } + + @Test + public void testReadDEFile() throws Exception { + String filePath = File.class.getResource("/reader/de/data/data2_1.txt").getPath(); + + FileConfig config = new FileConfig(filePath, "/reader/de/template/template2_1.json", + new StorageConfig("nas")); + + FileReader fileReader = FileFactory.createReader(config); + + Assert.assertNull(fileReader.readTail(HashMap.class)); + + Map head = fileReader.readHead(HashMap.class); + Assert.assertEquals(new Long(100), head.get("totalCount")); + Assert.assertEquals(new BigDecimal("300.03"), head.get("amount")); + + Map row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_0", row.get("seq")); + Assert.assertEquals("inst_seq_0", row.get("instSeq")); + Assert.assertEquals("2013-11-09 12:34:56", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131109", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131112 12:23:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.33"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("10.22"), row.get("amount")); + Assert.assertEquals(new Integer(22), row.get("age")); + Assert.assertEquals(new Long(12345), row.get("longN")); + Assert.assertEquals(Boolean.TRUE, row.get("bol")); + Assert.assertEquals("备注1", row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertEquals("seq_1", row.get("seq")); + Assert.assertEquals("inst_seq_1", row.get("instSeq")); + Assert.assertEquals("2013-11-10 15:56:12", + DateUtil.format((Date) row.get("gmtApply"), "yyyy-MM-dd HH:mm:ss")); + Assert.assertEquals("20131110", DateUtil.format((Date) row.get("date"), "yyyyMMdd")); + Assert.assertEquals("20131113 12:33:34", + DateUtil.format((Date) row.get("dateTime"), "yyyyMMdd HH:mm:ss")); + Assert.assertEquals(new BigDecimal("23.34"), row.get("applyNumber")); + Assert.assertEquals(new BigDecimal("11.88"), row.get("amount")); + Assert.assertEquals(new Integer(33), row.get("age")); + Assert.assertEquals(new Long(56789), row.get("longN")); + Assert.assertEquals(Boolean.FALSE, row.get("bol")); + Assert.assertNull(row.get("memo")); + + row = fileReader.readRow(HashMap.class); + Assert.assertNull(row); + + Assert.assertNull(fileReader.readTail(HashMap.class)); + + fileReader.close(); + } + + + + @After + public void after() { + tf.delete(); + } +} diff --git a/rdf-file-core/src/test/java/com/alipay/rdf/file/writer/CmFileWriterTest.java b/rdf-file-core/src/test/java/com/alipay/rdf/file/writer/CmFileWriterTest.java new file mode 100644 index 0000000..1931498 --- /dev/null +++ b/rdf-file-core/src/test/java/com/alipay/rdf/file/writer/CmFileWriterTest.java @@ -0,0 +1,335 @@ +package com.alipay.rdf.file.writer; + +import com.alipay.rdf.file.interfaces.FileFactory; +import com.alipay.rdf.file.interfaces.FileWriter; +import com.alipay.rdf.file.model.FileConfig; +import com.alipay.rdf.file.model.StorageConfig; +import com.alipay.rdf.file.util.DateUtil; +import com.alipay.rdf.file.util.TemporaryFolderUtil; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.math.BigDecimal; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * de 写测试 + * + * @author hongwei.quhw + * @version $Id: FileWriterDeTest.java, v 0.1 2017年8月10日 下午7:02:59 hongwei.quhw Exp $ + */ +public class CmFileWriterTest { + TemporaryFolderUtil tf = new TemporaryFolderUtil(); + + @Before + public void setUp() throws IOException { + tf.create(); + } + + @Test + public void testWriter() throws Exception { + String filePath = tf.getRoot().getAbsolutePath(); + System.out.println(filePath); + + FileConfig config = new FileConfig(new File(filePath, "test.txt").getAbsolutePath(), + "/writer/template/cm.json", new StorageConfig("nas")); + config.setLineBreak("\r"); + config.setFileEncoding("UTF-8"); + + FileWriter fileWriter = FileFactory.createWriter(config); + + Map head = new HashMap(); + head.put("totalCount", 2); + head.put("totalAmount", new BigDecimal("23.22")); + fileWriter.writeHead(head); + + Map body = new HashMap(); + + Date testDate = DateUtil.parse("2017-01-03 12:22:33", "yyyy-MM-dd HH:mm:ss"); + + body.put("seq", "seq12345"); + body.put("instSeq", "303"); + body.put("gmtApply", testDate); + body.put("date", testDate); + body.put("dateTime", testDate); + body.put("applyNumber", 12); + body.put("amount", new BigDecimal("1.22")); + body.put("age", new Integer(33)); + body.put("longN", new Long(33)); + body.put("bol", true); + body.put("memo", "memo1"); + fileWriter.writeRow(body); + + testDate = DateUtil.parse("2016-02-03 12:22:33", "yyyy-MM-dd HH:mm:ss"); + + body.put("seq", "seq234567"); + body.put("instSeq", "505"); + body.put("gmtApply", testDate); + body.put("date", testDate); + body.put("dateTime", testDate); + body.put("applyNumber", 12); + body.put("amount", new BigDecimal("1.09")); + body.put("age", 66); + body.put("longN", 125); + body.put("bol", false); + body.put("memo", "memo2"); + fileWriter.writeRow(body); + + fileWriter.close(); + + //校验文件 + BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream(new File(config.getFilePath())), "UTF-8")); + Assert.assertEquals("总笔数|总金额", reader.readLine()); + Assert.assertEquals("2|23.22", reader.readLine()); + Assert.assertEquals("流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注", + reader.readLine()); + Assert.assertEquals( + "seq12345|303|2017-01-03 12:22:33|20170103|20170103 12:22:33|12|1.22|33|33|true|memo1", + reader.readLine()); + Assert.assertEquals( + "seq234567|505|2016-02-03 12:22:33|20160203|20160203 12:22:33|12|1.09|66|125|false|memo2", + reader.readLine()); + + Assert.assertNull(reader.readLine()); + + reader.close(); + } + + @Test + public void testWriter2() throws Exception { + String filePath = tf.getRoot().getAbsolutePath(); + System.out.println(filePath); + + FileConfig config = new FileConfig(new File(filePath, "test.txt").getAbsolutePath(), + "/writer/template/cm2.json", new StorageConfig("nas")); + config.setLineBreak("\r"); + config.setFileEncoding("UTF-8"); + + FileWriter fileWriter = FileFactory.createWriter(config); + + Map head = new HashMap(); + head.put("totalCount", 2); + head.put("totalAmount", new BigDecimal("23.22")); + fileWriter.writeHead(head); + + Map body = new HashMap(); + + Date testDate = DateUtil.parse("2017-01-03 12:22:33", "yyyy-MM-dd HH:mm:ss"); + + body.put("seq", "seq12345"); + body.put("instSeq", "303"); + body.put("gmtApply", testDate); + body.put("date", testDate); + body.put("dateTime", testDate); + body.put("applyNumber", 12); + body.put("amount", new BigDecimal("1.22")); + body.put("age", new Integer(33)); + body.put("longN", new Long(33)); + body.put("bol", true); + body.put("memo", "memo1"); + fileWriter.writeRow(body); + + testDate = DateUtil.parse("2016-02-03 12:22:33", "yyyy-MM-dd HH:mm:ss"); + + body.put("seq", "seq234567"); + body.put("instSeq", "505"); + body.put("gmtApply", testDate); + body.put("date", testDate); + body.put("dateTime", testDate); + body.put("applyNumber", 12); + body.put("amount", new BigDecimal("1.09")); + body.put("age", 66); + body.put("longN", 125); + body.put("bol", false); + body.put("memo", "memo2"); + fileWriter.writeRow(body); + + fileWriter.writeTail(new HashMap()); + + fileWriter.close(); + + //校验文件 + BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream(new File(config.getFilePath())), "UTF-8")); + Assert.assertEquals("总笔数|总金额", reader.readLine()); + Assert.assertEquals("2|23.22", reader.readLine()); + Assert.assertEquals("流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注", + reader.readLine()); + Assert.assertEquals( + "seq12345|303|2017-01-03 12:22:33|20170103|20170103 12:22:33|12|1.22|33|33|true|memo1", + reader.readLine()); + Assert.assertEquals( + "seq234567|505|2016-02-03 12:22:33|20160203|20160203 12:22:33|12|1.09|66|125|false|memo2", + reader.readLine()); + + Assert.assertEquals("文件结束A|文件结束B", reader.readLine()); + Assert.assertEquals("AEnd|BEnd", reader.readLine()); + Assert.assertNull(reader.readLine()); + reader.close(); + } + + @Test + public void testWriter3() throws Exception { + String filePath = tf.getRoot().getAbsolutePath(); + System.out.println(filePath); + + FileConfig config = new FileConfig(new File(filePath, "test.txt").getAbsolutePath(), + "/writer/template/cm3.json", new StorageConfig("nas")); + config.setLineBreak("\r"); + config.setFileEncoding("UTF-8"); + + FileWriter fileWriter = FileFactory.createWriter(config); + + Map head = new HashMap(); + head.put("totalCount", 2); + head.put("totalAmount", new BigDecimal("23.22")); + fileWriter.writeHead(head); + + Map body = new HashMap(); + + Date testDate = DateUtil.parse("2017-01-03 12:22:33", "yyyy-MM-dd HH:mm:ss"); + + body.put("seq", "seq12345"); + body.put("instSeq", "303"); + body.put("gmtApply", testDate); + body.put("date", testDate); + body.put("dateTime", testDate); + body.put("applyNumber", 12); + body.put("amount", new BigDecimal("1.22")); + body.put("age", new Integer(33)); + body.put("longN", new Long(33)); + body.put("bol", true); + body.put("memo", "memo1"); + fileWriter.writeRow(body); + + testDate = DateUtil.parse("2016-02-03 12:22:33", "yyyy-MM-dd HH:mm:ss"); + + body.put("seq", "seq234567"); + body.put("instSeq", "505"); + body.put("gmtApply", testDate); + body.put("date", testDate); + body.put("dateTime", testDate); + body.put("applyNumber", 12); + body.put("amount", new BigDecimal("1.09")); + body.put("age", 66); + body.put("longN", 125); + body.put("bol", false); + body.put("memo", "memo2"); + fileWriter.writeRow(body); + + fileWriter.writeTail(new HashMap()); + + fileWriter.close(); + + //校验文件 + BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream(new File(config.getFilePath())), "UTF-8")); + Assert.assertEquals("|总笔数|总金额", reader.readLine()); + Assert.assertEquals("|2|23.22", reader.readLine()); + Assert.assertEquals("|流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注", + reader.readLine()); + Assert.assertEquals( + "|seq12345|303|2017-01-03 12:22:33|20170103|20170103 12:22:33|12|1.22|33|33|true|memo1|", + reader.readLine()); + Assert.assertEquals( + "|seq234567|505|2016-02-03 12:22:33|20160203|20160203 12:22:33|12|1.09|66|125|false|memo2|", + reader.readLine()); + + Assert.assertEquals("|文件结束A|文件结束B|", reader.readLine()); + Assert.assertEquals("|AEnd|BEnd|", reader.readLine()); + Assert.assertNull(reader.readLine()); + reader.close(); + } + + @Test + public void testWriter4() throws Exception { + String filePath = tf.getRoot().getAbsolutePath(); + System.out.println(filePath); + + FileConfig config = new FileConfig(new File(filePath, "test.txt").getAbsolutePath(), + "/writer/template/cm_ext.json", new StorageConfig("nas")); + config.setLineBreak("\r"); + config.setFileEncoding("UTF-8"); + + FileWriter fileWriter = FileFactory.createWriter(config); + + Map head = new HashMap(); + head.put("totalCount", 2); + head.put("totalAmount", new BigDecimal("23.22")); + fileWriter.writeHead(head); + + Map body = new HashMap(); + + Date testDate = DateUtil.parse("2017-01-03 12:22:33", "yyyy-MM-dd HH:mm:ss"); + + body.put("seq", "seq12345"); + body.put("instSeq", "303"); + body.put("gmtApply", testDate); + body.put("date", testDate); + body.put("dateTime", testDate); + body.put("applyNumber", 12); + body.put("amount", new BigDecimal("1.22")); + body.put("age", new Integer(33)); + body.put("longN", new Long(33)); + body.put("bol", true); + body.put("memo", "memo1"); + fileWriter.writeRow(body); + + testDate = DateUtil.parse("2016-02-03 12:22:33", "yyyy-MM-dd HH:mm:ss"); + + body.put("seq", "seq234567"); + body.put("instSeq", "505"); + body.put("gmtApply", testDate); + body.put("date", testDate); + body.put("dateTime", testDate); + body.put("applyNumber", 12); + body.put("amount", new BigDecimal("1.09")); + body.put("age", 66); + body.put("longN", 125); + body.put("bol", false); + body.put("memo", "memo2"); + fileWriter.writeRow(body); + + fileWriter.writeTail(new HashMap()); + + fileWriter.close(); + + //校验文件 + BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream(new File(config.getFilePath())), "UTF-8")); + Assert.assertEquals("总笔数", reader.readLine()); + Assert.assertEquals("总金额", reader.readLine()); + Assert.assertEquals("2", reader.readLine()); + Assert.assertEquals("23.22", reader.readLine()); + Assert.assertEquals("|流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注", + reader.readLine()); + Assert.assertEquals( + "|seq12345|303|2017-01-03 12:22:33|20170103|20170103 12:22:33|12|1.22|33|33|true|memo1|", + reader.readLine()); + Assert.assertEquals( + "|seq234567|505|2016-02-03 12:22:33|20160203|20160203 12:22:33|12|1.09|66|125|false|memo2|", + reader.readLine()); + + Assert.assertEquals("文件结束A", reader.readLine()); + Assert.assertEquals("文件结束B", reader.readLine()); + Assert.assertEquals("AEnd", reader.readLine()); + Assert.assertEquals("BEnd", reader.readLine()); + Assert.assertNull(reader.readLine()); + reader.close(); + } + + @After + public void after() { + tf.delete(); + } +} diff --git a/rdf-file-core/src/test/resources/META-INF/rdf-file/protocol/cm_ext.xml b/rdf-file-core/src/test/resources/META-INF/rdf-file/protocol/cm_ext.xml new file mode 100644 index 0000000..9040137 --- /dev/null +++ b/rdf-file-core/src/test/resources/META-INF/rdf-file/protocol/cm_ext.xml @@ -0,0 +1,26 @@ + + + + + + ${column.value()} + + + + + + + + ${column.value()} + + + + + + + + ${column.value()} + + + + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/data/data1.txt b/rdf-file-core/src/test/resources/reader/cm/data/data1.txt new file mode 100644 index 0000000..a59d9c6 --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/data/data1.txt @@ -0,0 +1,7 @@ +总笔数|总金额 +100|300.03 +流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注 +seq_0|inst_seq_0|2013-11-09 12:34:56|20131109|20131112 12:23:34|23.33|10.22|22|12345|true|备注1 +seq_1|inst_seq_1|2013-11-10 15:56:12|20131110|20131113 12:33:34|23.34|11.88|33|56789|false| +文件结束 +endFile \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/data/data2.txt b/rdf-file-core/src/test/resources/reader/cm/data/data2.txt new file mode 100644 index 0000000..c101371 --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/data/data2.txt @@ -0,0 +1,7 @@ +|总笔数|总金额 +|100|300.03 +|流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注 +seq_0|inst_seq_0|2013-11-09 12:34:56|20131109|20131112 12:23:34|23.33|10.22|22|12345|true|备注1 +seq_1|inst_seq_1|2013-11-10 15:56:12|20131110|20131113 12:33:34|23.34|11.88|33|56789|false| +文件结束| +endFile| \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/data/data3.txt b/rdf-file-core/src/test/resources/reader/cm/data/data3.txt new file mode 100644 index 0000000..0fbe0dc --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/data/data3.txt @@ -0,0 +1,7 @@ +|总笔数|总金额 +|100|300.03 +|流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注 +|seq_0|inst_seq_0|2013-11-09 12:34:56|20131109|20131112 12:23:34|23.33|10.22|22|12345|true|备注1| +|seq_1|inst_seq_1|2013-11-10 15:56:12|20131110|20131113 12:33:34|23.34|11.88|33|56789|false|| +|文件结束| +|endFile| \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/data/data4.txt b/rdf-file-core/src/test/resources/reader/cm/data/data4.txt new file mode 100644 index 0000000..029fda5 --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/data/data4.txt @@ -0,0 +1,9 @@ +总笔数 +总金额 +100 +300.03 +流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注 +seq_0|inst_seq_0|2013-11-09 12:34:56|20131109|20131112 12:23:34|23.33|10.22|22|12345|true|备注1 +seq_1|inst_seq_1|2013-11-10 15:56:12|20131110|20131113 12:33:34|23.34|11.88|33|56789|false| +文件结束 +endFile \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/data/data5.txt b/rdf-file-core/src/test/resources/reader/cm/data/data5.txt new file mode 100644 index 0000000..b45ccf3 --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/data/data5.txt @@ -0,0 +1,9 @@ +总笔数 +总金额 +100 +300.03 +|流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注 +|seq_0|inst_seq_0|2013-11-09 12:34:56|20131109|20131112 12:23:34|23.33|10.22|22|12345|true|备注1| +|seq_1|inst_seq_1|2013-11-10 15:56:12|20131110|20131113 12:33:34|23.34|11.88|33|56789|false|| +文件结束 +endFile \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/template/template1.json b/rdf-file-core/src/test/resources/reader/cm/template/template1.json new file mode 100644 index 0000000..d130ed4 --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/template/template1.json @@ -0,0 +1,24 @@ +{ + "head":[ + "totalCount|总笔数|Required|Long", + "totalAmount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|BigDecimal", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "tail":[ + "fileEnd|文件结束|default:endFile" + ], + "protocol":"CM" +} + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/template/template2.json b/rdf-file-core/src/test/resources/reader/cm/template/template2.json new file mode 100644 index 0000000..079555a --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/template/template2.json @@ -0,0 +1,26 @@ +{ + "head":[ + "totalCount|总笔数|Required|Long", + "totalAmount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|BigDecimal", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "tail":[ + "fileEnd|文件结束|default:endFile" + ], + "startWithSplit": "head", + "endWithSplit": "tail", + "protocol":"CM" +} + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/template/template3.json b/rdf-file-core/src/test/resources/reader/cm/template/template3.json new file mode 100644 index 0000000..1233b17 --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/template/template3.json @@ -0,0 +1,26 @@ +{ + "head":[ + "totalCount|总笔数|Required|Long", + "totalAmount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|BigDecimal", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "tail":[ + "fileEnd|文件结束|default:endFile" + ], + "startWithSplit": "head|body|tail", + "endWithSplit": "body|tail", + "protocol":"CM" +} + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/template/template4.json b/rdf-file-core/src/test/resources/reader/cm/template/template4.json new file mode 100644 index 0000000..cfc61cc --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/template/template4.json @@ -0,0 +1,24 @@ +{ + "head":[ + "totalCount|总笔数|Required|Long", + "totalAmount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|BigDecimal", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "tail":[ + "fileEnd|文件结束|default:endFile" + ], + "protocol":"CM_EXT" +} + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/cm/template/template5.json b/rdf-file-core/src/test/resources/reader/cm/template/template5.json new file mode 100644 index 0000000..7276625 --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/cm/template/template5.json @@ -0,0 +1,26 @@ +{ + "head":[ + "totalCount|总笔数|Required|Long", + "totalAmount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|BigDecimal", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "tail":[ + "fileEnd|文件结束|default:endFile" + ], + "startWithSplit": "head|body|tail", + "endWithSplit": "body|tail", + "protocol":"CM_EXT" +} + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/de/data/data2_1.txt b/rdf-file-core/src/test/resources/reader/de/data/data2_1.txt new file mode 100644 index 0000000..edf894f --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/de/data/data2_1.txt @@ -0,0 +1,4 @@ +总笔数:100|总金额:300.03 +流水号|基金公司订单号|订单申请时间|普通日期|普通日期时间|普通数字|金额|年龄|长整型|布尔值|备注 +seq_0|inst_seq_0|2013-11-09 12:34:56|20131109|20131112 12:23:34|23.33|10.22|22|12345|true|备注1 +seq_1|inst_seq_1|2013-11-10 15:56:12|20131110|20131113 12:33:34|23.34|11.88|33|56789|false| \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/reader/de/template/template2_1.json b/rdf-file-core/src/test/resources/reader/de/template/template2_1.json new file mode 100644 index 0000000..2b1b749 --- /dev/null +++ b/rdf-file-core/src/test/resources/reader/de/template/template2_1.json @@ -0,0 +1,21 @@ +{ + "head":[ + "totalCount|总笔数|Required|Long", + "amount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|BigDecimal", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "protocol":"DE" +} + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/writer/template/cm.json b/rdf-file-core/src/test/resources/writer/template/cm.json new file mode 100644 index 0000000..db38d41 --- /dev/null +++ b/rdf-file-core/src/test/resources/writer/template/cm.json @@ -0,0 +1,22 @@ +{ + "head":[ + "totalCount|总笔数|Required|Integer", + "totalAmount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|Long", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "protocol":"CM", + "lineBreak":"\r\n" +} + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/writer/template/cm2.json b/rdf-file-core/src/test/resources/writer/template/cm2.json new file mode 100644 index 0000000..2378be8 --- /dev/null +++ b/rdf-file-core/src/test/resources/writer/template/cm2.json @@ -0,0 +1,26 @@ +{ + "head":[ + "totalCount|总笔数|Required|Integer", + "totalAmount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|Long", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "tail":[ + "endA|文件结束A|default:AEnd", + "endB|文件结束B|default:BEnd" + ], + "protocol":"CM", + "lineBreak":"\r\n" +} + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/writer/template/cm3.json b/rdf-file-core/src/test/resources/writer/template/cm3.json new file mode 100644 index 0000000..30bffe0 --- /dev/null +++ b/rdf-file-core/src/test/resources/writer/template/cm3.json @@ -0,0 +1,28 @@ +{ + "head":[ + "totalCount|总笔数|Required|Integer", + "totalAmount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|Long", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "tail":[ + "endA|文件结束A|default:AEnd", + "endB|文件结束B|default:BEnd" + ], + "startWithSplit": "head|body|tail", + "endWithSplit": "body|tail", + "protocol":"CM", + "lineBreak":"\r\n" +} + \ No newline at end of file diff --git a/rdf-file-core/src/test/resources/writer/template/cm_ext.json b/rdf-file-core/src/test/resources/writer/template/cm_ext.json new file mode 100644 index 0000000..f1f93ec --- /dev/null +++ b/rdf-file-core/src/test/resources/writer/template/cm_ext.json @@ -0,0 +1,28 @@ +{ + "head":[ + "totalCount|总笔数|Required|Integer", + "totalAmount|总金额|BigDecimal|Required" + ], + "body":[ + "seq|流水号", + "instSeq|基金公司订单号|Required", + "gmtApply|订单申请时间|Date:yyyy-MM-dd HH:mm:ss", + "date|普通日期|Date:yyyyMMdd", + "dateTime|普通日期时间|Date:yyyyMMdd HH:mm:ss", + "applyNumber|普通数字|Long", + "amount|金额|BigDecimal", + "age|年龄|Integer", + "longN|长整型|Long", + "bol|布尔值|Boolean", + "memo|备注" + ], + "tail":[ + "endA|文件结束A|default:AEnd", + "endB|文件结束B|default:BEnd" + ], + "startWithSplit": "head|body|tail", + "endWithSplit": "body|tail", + "protocol":"CM_EXT", + "lineBreak":"\r\n" +} + \ No newline at end of file diff --git a/rdf-file-test/src/main/resources/pom/rdf-file-core.pom b/rdf-file-test/src/main/resources/pom/rdf-file-core.pom index 1e86c8d..1c1c1c9 100644 --- a/rdf-file-test/src/main/resources/pom/rdf-file-core.pom +++ b/rdf-file-test/src/main/resources/pom/rdf-file-core.pom @@ -4,7 +4,7 @@ 4.0.0 com.alipay.rdf.file rdf-file-core - 2.2.5 + 2.2.6 RDF FILE CORE RDF FILE CORE https://github.com/alipay/rdf-file diff --git a/rdf-file-test/src/main/resources/pom/rdf-file-oss.pom b/rdf-file-test/src/main/resources/pom/rdf-file-oss.pom index 592182d..d0115e4 100644 --- a/rdf-file-test/src/main/resources/pom/rdf-file-oss.pom +++ b/rdf-file-test/src/main/resources/pom/rdf-file-oss.pom @@ -4,7 +4,7 @@ 4.0.0 com.alipay.rdf.file rdf-file-oss - 2.2.5 + 2.2.6 RDF FILE OSS RDF FILE OSS https://github.com/alipay/rdf-file diff --git a/rdf-file-test/src/main/resources/pom/rdf-file-sftp.pom b/rdf-file-test/src/main/resources/pom/rdf-file-sftp.pom index 1babaa8..1cefe37 100644 --- a/rdf-file-test/src/main/resources/pom/rdf-file-sftp.pom +++ b/rdf-file-test/src/main/resources/pom/rdf-file-sftp.pom @@ -4,7 +4,7 @@ 4.0.0 com.alipay.rdf.file rdf-file-sftp - 2.2.5 + 2.2.6 RDF FILE SFTP RDF FILE SFTP https://github.com/alipay/rdf-file