From 3599933e5455256e554e1ef129e68cfa89b8de1a Mon Sep 17 00:00:00 2001 From: Patrick <33585596+pgmitche@users.noreply.github.com> Date: Sun, 11 Oct 2020 17:46:13 +1100 Subject: [PATCH] Fix - Various bugs (#14) * Update example to use explicit period * Reset start/end pos counter per source copybook * Allow detection of explicit periods * Allow detection of explicit periods * Revise handling of slice/occurs input * Regen example * Update test copybook template usage * Update type fallthrough for float treatment * Update template package name to copygen --- Makefile | 4 +++ cmd/pkg/cmd/root.go | 2 +- cmd/pkg/decoder/decode_test.go | 16 ++++++------ cmd/pkg/decoder/lines.go | 4 +-- cmd/pkg/decoder/lines_test.go | 4 +++ cmd/pkg/decoder/util.go | 23 +++++++++++++--- cmd/pkg/decoder/util_test.go | 6 ++++- cmd/pkg/template/template.go | 35 +++++++++++++++---------- decode.go | 2 +- decode_test.go | 48 +++++++++++++++++++++++++++++----- example/ExampleCopybook.txt | 2 +- example/example.go | 2 +- setfunc.go | 14 +++++----- tag.go | 2 +- 14 files changed, 117 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index bdb3ec5..8809cc7 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,10 @@ run: build chmod +x ./dist/gopic ./dist/gopic dir -i dummy -o dummyout +.PHONY: example +example: install + gopic file -o example/example.go -i example/ExampleCopybook.txt + define CHECK_COVERAGE awk \ -F '[ %]+' \ diff --git a/cmd/pkg/cmd/root.go b/cmd/pkg/cmd/root.go index 060b1ae..e8a2255 100644 --- a/cmd/pkg/cmd/root.go +++ b/cmd/pkg/cmd/root.go @@ -124,7 +124,7 @@ func run(r io.Reader, output string) error { name := strings.TrimSuffix(output, filepath.Ext(output)) n := name[strings.LastIndex(name, "/")+1:] - c := copybook.New(n, template.CopyBook) + c := copybook.New(n, template.Copybook()) b, err := ioutil.ReadAll(r) if err != nil { diff --git a/cmd/pkg/decoder/decode_test.go b/cmd/pkg/decoder/decode_test.go index 7c4a918..5af8276 100644 --- a/cmd/pkg/decoder/decode_test.go +++ b/cmd/pkg/decoder/decode_test.go @@ -20,7 +20,7 @@ func TestDecoder_Unmarshal(t *testing.T) { }{ { name: "SuccessfullyParseBasicDummyCopybook", - c: copybook.New("dummy", template.CopyBook), + c: copybook.New("dummy", template.Copybook()), in: `000600 10 DUMMY-1 PIC X. 00000167 000610 10 DUMMY-2 PIC X(3). 00000168 000620 10 DUMMY-3 PIC 9(7). 00000169 @@ -75,7 +75,7 @@ func Test_decoder_findDataRecord(t *testing.T) { { name: "BasicPICStringSingle", line: "000600 10 DUMMY-1 PIC X. 00000167", - c: copybook.New("dummy", template.CopyBook), + c: copybook.New("dummy", template.Copybook()), want: ©book.Record{ Num: 600, Level: 10, @@ -86,7 +86,7 @@ func Test_decoder_findDataRecord(t *testing.T) { }, { name: "BasicPICStringParentheses", line: "000610 10 DUMMY-2 PIC X(3). 00000168", - c: copybook.New("dummy", template.CopyBook), + c: copybook.New("dummy", template.Copybook()), want: ©book.Record{ Num: 610, Level: 10, @@ -97,7 +97,7 @@ func Test_decoder_findDataRecord(t *testing.T) { }, { name: "BasicPICUintParentheses", line: "000620 10 DUMMY-3 PIC 9(7). 00000169", - c: copybook.New("dummy", template.CopyBook), + c: copybook.New("dummy", template.Copybook()), want: ©book.Record{ Num: 620, Level: 10, @@ -108,7 +108,7 @@ func Test_decoder_findDataRecord(t *testing.T) { }, { name: "BasicPICStringMulti", line: "000640 10 DUMMY-5 PIC XX. 00000171", - c: copybook.New("dummy", template.CopyBook), + c: copybook.New("dummy", template.Copybook()), want: ©book.Record{ Num: 640, Level: 10, @@ -119,18 +119,18 @@ func Test_decoder_findDataRecord(t *testing.T) { }, { name: "MultiTypeMultiParenthesesNumber", line: "000640 10 DUMMY-5 PIC S9(9)V9(9). 00000171", - c: copybook.New("dummy", template.CopyBook), + c: copybook.New("dummy", template.Copybook()), want: ©book.Record{ Num: 640, Level: 10, Name: "DUMMY-5", - Picture: reflect.Int, + Picture: reflect.Float64, Length: 20, }, }, { name: "MultiTypeMultiParenthesesString", line: "000640 10 DUMMY-5 PIC A(9)V9(9). 00000171", - c: copybook.New("dummy", template.CopyBook), + c: copybook.New("dummy", template.Copybook()), want: ©book.Record{ Num: 640, Level: 10, diff --git a/cmd/pkg/decoder/lines.go b/cmd/pkg/decoder/lines.go index f4c45c4..2646dce 100644 --- a/cmd/pkg/decoder/lines.go +++ b/cmd/pkg/decoder/lines.go @@ -36,12 +36,12 @@ var ( // Defines picture clause // 000600 10 DUMMY-1 PIC X. 00000167 // 000620 10 DUMMY-2 PIC 9(7). 00000169 - generousPICLine = regexp.MustCompile(`^[0-9]+ +[0-9]{2} +[a-zA-Z0-9\-]+ +PIC [AXPVS()0-9]+\. +0+[0-9]+$`) + generousPICLine = regexp.MustCompile(`^[0-9]+ +[0-9]{2} +[a-zA-Z0-9\-]+ +PIC [.AXPVS()0-9]+\. +0+[0-9]+$`) // Defines picture clause that deviates from typical pattern // 10 DUMMY-1 PIC X. 00000167 // 10 DUMMY-2 PIC 9(7). 00000169 - generousIncompletePICLine = regexp.MustCompile(`[0-9]{2} +[a-zA-Z0-9\-]+ +PIC [AXPVS()0-9]+\.`) + generousIncompletePICLine = regexp.MustCompile(`[0-9]{2} +[a-zA-Z0-9\-]+ +PIC [.AXPVS()0-9]+\.`) // Matches same-line REDEFINES definitions // 000550 05 DUMMY-1 PIC X(340). 00000162 diff --git a/cmd/pkg/decoder/lines_test.go b/cmd/pkg/decoder/lines_test.go index e719e33..e8c50d7 100644 --- a/cmd/pkg/decoder/lines_test.go +++ b/cmd/pkg/decoder/lines_test.go @@ -36,6 +36,10 @@ func Test_getLineType(t *testing.T) { name: "BasicPICNumberNoPeriodPotentialOccursTarget", line: "12345 01 NAME PIC 9(9) 054321", want: occursMulti, + }, { + name: "RegularPICLiteralDot", + line: "12345 01 NAME PIC 9(9).9(9). 054321", + want: pic, }, } for _, test := range tests { diff --git a/cmd/pkg/decoder/util.go b/cmd/pkg/decoder/util.go index f7d9a36..f7bcee8 100644 --- a/cmd/pkg/decoder/util.go +++ b/cmd/pkg/decoder/util.go @@ -13,7 +13,13 @@ const ( unknown picType = iota unsigned signed + decimal alpha + + alphaIndicators = "XA" + decimalIndicators = ".VP" + signedIntIndicators = "S" + intIndicators = "9" ) var ( @@ -21,6 +27,7 @@ var ( unknown: reflect.Invalid, unsigned: reflect.Uint, signed: reflect.Int, + decimal: reflect.Float64, alpha: reflect.String, } ) @@ -29,21 +36,29 @@ var ( // that contains a PIC definition func parsePICType(s string) reflect.Kind { picType := unknown - if strings.ContainsAny(s, "XA") { + s = strings.TrimRight(s, ".") + if strings.ContainsAny(s, alphaIndicators) { if alpha > picType { picType = alpha return types[picType] } } - if strings.ContainsAny(s, "S") { + if strings.ContainsAny(s, decimalIndicators) { + if decimal > picType { + picType = decimal + return types[picType] + } + } + + if strings.ContainsAny(s, signedIntIndicators) { if signed > picType { picType = signed return types[picType] } } - if strings.ContainsAny(s, "VP9") { + if strings.ContainsAny(s, intIndicators) { picType = unsigned return types[picType] } @@ -55,7 +70,7 @@ func parsePICType(s string) reflect.Kind { // PIC definition such as: X(2)., XX., 9(9)., etc. func parsePICCount(s string) (int, error) { // prepare a slice of runes, representing the string - s = strings.Trim(s, ".") + s = strings.TrimRight(s, ".") c := []rune(s) size := 0 diff --git a/cmd/pkg/decoder/util_test.go b/cmd/pkg/decoder/util_test.go index f7337f3..3cb649c 100644 --- a/cmd/pkg/decoder/util_test.go +++ b/cmd/pkg/decoder/util_test.go @@ -41,6 +41,10 @@ func Test_parsePICCount(t *testing.T) { name: "SignedNumberWithMultiParentheses", in: "S9(9)V9(9).", want: 20, + }, { + name: "SignedNumberWithMultiParenthesesLiteralPeriod", + in: "S9(9).9(9).", + want: 20, }, } for _, test := range tests { @@ -86,7 +90,7 @@ func Test_parsePICType(t *testing.T) { }, { name: "SWithParenthesesSingleDigitDecimalPlace", in: "S9(9)V9(9).", - want: reflect.Int, + want: reflect.Float64, }, { name: "MultiParenthesesWithAlphanumeric", in: "S(9)VX(9).", diff --git a/cmd/pkg/template/template.go b/cmd/pkg/template/template.go index d4c2327..295d876 100644 --- a/cmd/pkg/template/template.go +++ b/cmd/pkg/template/template.go @@ -20,7 +20,7 @@ var ( } ) -var CopyBook = template.Must( +var copyBook = template.Must( template.New("struct"). Funcs(templateFuncs). Parse(` @@ -30,7 +30,7 @@ var CopyBook = template.Must( //////////////////////////////// // nolint -package tempcopybook +package copygen // Copybook{{.Name}} contains a representation of your provided Copybook type Copybook{{.Name}} struct { @@ -40,27 +40,34 @@ type Copybook{{.Name}} struct { } `)) +func Copybook() *template.Template { + startPos = 1 + endPos = 1 + + return copyBook +} + // goType translates a type into a go type func goType(t reflect.Kind, i int) string { + tag := "" switch t { case reflect.String: - if i > 0 { - return "[]string" - } - return "string" + tag = "string" case reflect.Int: - if i > 0 { - return "[]int" - } - return "int" + tag = "int" case reflect.Uint: - if i > 0 { - return "[]uint" - } - return "uint" + tag = "uint" + case reflect.Float64: + tag = "float64" default: panic(fmt.Sprintf("unrecognized type %v", t)) } + + if i > 0 { + tag = fmt.Sprintf("[]%s", tag) + } + + return tag } func picTag(l int, i int) string { diff --git a/decode.go b/decode.go index 1d5d03d..b9c172c 100644 --- a/decode.go +++ b/decode.go @@ -62,7 +62,7 @@ func (d *Decoder) scanLine(v reflect.Value) (bool, error) { l := string(d.s.Bytes()) t := v.Type() - set := newSetFunc(t, 0) + set := newSetFunc(t, 0, 0) return true, set(v, l) } diff --git a/decode_test.go b/decode_test.go index 12ae94f..d331555 100644 --- a/decode_test.go +++ b/decode_test.go @@ -1,7 +1,6 @@ package pic import ( - "reflect" "testing" "github.com/stretchr/testify/require" @@ -13,6 +12,18 @@ func TestUnmarshal(t *testing.T) { Int int `pic:"5"` Float float64 `pic:"5"` } + + type occursOffset struct { + A string `pic:"13"` // start:227 end:239 + B string `pic:"13"` // start:240 end:252 + C string `pic:"13"` // start:253 end:265 + D string `pic:"13"` // start:266 end:278 + E string `pic:"13"` // start:279 end:291 + F string `pic:"13"` // start:292 end:304 + G string `pic:"2"` // start:305 end:306 + H []string `pic:"13,12"` // start:307 end:462 + } + for _, test := range []struct { name string val []byte @@ -78,16 +89,41 @@ func TestUnmarshal(t *testing.T) { target: basicTypes{}, expected: basicTypes{}, shouldErr: true, + }, { + name: "offsetcheck", + val: []byte("000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 00000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 000000000.00 "), + target: &occursOffset{}, + expected: &occursOffset{ + A: "000000000.00", + B: "000000000.00", + C: "000000000.00", + D: "000000000.00", + E: "000000000.00", + F: "000000000.00", + G: "00", + H: []string{ + "000000000.00", + "000000000.00", + "000000000.00", + "000000000.00", + "000000000.00", + "000000000.00", + "000000000.00", + "000000000.00", + "000000000.00", + "000000000.00", + "000000000.00", + "000000000.00"}, + }, }, } { tt := test t.Run(tt.name, func(t *testing.T) { err := Unmarshal(tt.val, tt.target) - if tt.shouldErr != (err != nil) { - t.Errorf("Unmarshal() err want %v, have %v (%v)", tt.shouldErr, err != nil, err) - } - if !tt.shouldErr && !reflect.DeepEqual(tt.target, tt.expected) { - t.Errorf("Unmarshal() want %+v, have %+v", tt.expected, tt.target) + if tt.shouldErr { + require.Error(t, err) + } else { + require.Equal(t, tt.target, tt.expected) } }) } diff --git a/example/ExampleCopybook.txt b/example/ExampleCopybook.txt index 94a3935..65a82fa 100644 --- a/example/ExampleCopybook.txt +++ b/example/ExampleCopybook.txt @@ -21,7 +21,7 @@ 000430 15 DUMMY-GROUP-1-OBJECT-H PIC 9(4). 00000144 000550 05 DUMMY-GROUP-2 PIC X(201). 00000162 000830 05 DUMMY-GROUP-3 REDEFINES DUMMY-GROUP-2. 00000195 -000840 10 DUMMY-GROUP-2-OBJECT-A PIC X(14). 00000196 +000840 10 DUMMY-GROUP-2-OBJECT-A PIC 9(11).9(2). 00000196 000850 10 DUMMY-GROUP-2-OBJECT-B PIC 9(7). 00000197 001060 10 DUMMY-GROUP-2-OBJECT-C PIC XXXX. 00000218 001070 10 DUMMY-GROUP-2-OBJECT-D PIC X. 00000219 diff --git a/example/example.go b/example/example.go index 6f009cd..1fc8f00 100644 --- a/example/example.go +++ b/example/example.go @@ -15,7 +15,7 @@ type Copybookexample struct { DUMMYGROUP1OBJECTE string `pic:"8"` // start:50 end:57 DUMMYGROUP1OBJECTG string `pic:"2"` // start:58 end:59 DUMMYGROUP1OBJECTH uint `pic:"4"` // start:60 end:63 - DUMMYGROUP2OBJECTA string `pic:"14"` // start:64 end:77 + DUMMYGROUP2OBJECTA float64 `pic:"14"` // start:64 end:77 DUMMYGROUP2OBJECTB uint `pic:"7"` // start:78 end:84 DUMMYGROUP2OBJECTC string `pic:"4"` // start:85 end:88 DUMMYGROUP2OBJECTD string `pic:"1"` // start:89 end:89 diff --git a/setfunc.go b/setfunc.go index 89849a3..be4b44b 100644 --- a/setfunc.go +++ b/setfunc.go @@ -8,7 +8,7 @@ import ( type setFunc func(v reflect.Value, s string) error -func newSetFunc(t reflect.Type, occursSize int) setFunc { +func newSetFunc(t reflect.Type, picSize, occursSize int) setFunc { switch t.Kind() { case reflect.String: return strSetFunc @@ -19,7 +19,7 @@ func newSetFunc(t reflect.Type, occursSize int) setFunc { case reflect.Float64: return floatSetFunc(64) case reflect.Slice: - return arraySetFunc(occursSize) + return arraySetFunc(picSize, occursSize) case reflect.Ptr: return ptrSetFunc(t) case reflect.Interface: @@ -65,9 +65,9 @@ func floatSetFunc(size int) setFunc { } } -func arraySetFunc(count int) setFunc { +func arraySetFunc(l, count int) setFunc { return func(v reflect.Value, s string) error { - size := len(s) / count + size := l / count if len(s) == 0 { return nilSetFunc(v, s) } @@ -77,7 +77,7 @@ func arraySetFunc(count int) setFunc { } many := reflect.MakeSlice(v.Type(), count, count) - sf := newSetFunc(v.Type().Elem(), 0) + sf := newSetFunc(v.Type().Elem(), 0, 0) track := 1 for i := 0; i < count; i++ { @@ -95,7 +95,7 @@ func arraySetFunc(count int) setFunc { } func ptrSetFunc(t reflect.Type) setFunc { - innerSetter := newSetFunc(t.Elem(), 0) + innerSetter := newSetFunc(t.Elem(), 0, 0) return func(v reflect.Value, s string) error { if len(s) == 0 { return nilSetFunc(v, s) @@ -110,7 +110,7 @@ func ptrSetFunc(t reflect.Type) setFunc { } func ifaceSetFunc(v reflect.Value, s string) error { - return newSetFunc(v.Elem().Type(), 0)(v.Elem(), s) + return newSetFunc(v.Elem().Type(), 0, 0)(v.Elem(), s) } func structSetFunc(t reflect.Type) setFunc { diff --git a/tag.go b/tag.go index d3004a8..ed3bdfd 100644 --- a/tag.go +++ b/tag.go @@ -59,7 +59,7 @@ func makeStructRepresentation(t reflect.Type) structRepresentation { sr.fields[i].start = s sr.fields[i].end = e sr.fields[i].err = err - sr.fields[i].setFunc = newSetFunc(f.Type, occursSize) + sr.fields[i].setFunc = newSetFunc(f.Type, l, occursSize) if sr.fields[i].end > sr.len { sr.len = sr.fields[i].end }