diff --git a/BUILD.bazel b/BUILD.bazel index 2575a00e..691476fa 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "error_formatter.go", "imports.go", "interpreter.go", + "jsonml.go", "runtime_error.go", "thunks.go", "util.go", diff --git a/builtins.go b/builtins.go index e426b069..bce6a170 100644 --- a/builtins.go +++ b/builtins.go @@ -1512,6 +1512,64 @@ func builtinParseYAML(i *interpreter, str value) (value, error) { return jsonToValue(i, elems[0]) } +func builtinParseXmlJsonml(i *interpreter, arguments []value) (value, error) { + strv := arguments[0] + pwsv := arguments[1] + + sval, err := i.getString(strv) + if err != nil { + return nil, err + } + s := sval.getGoString() + + pws := false + if pwsv.getType() != nullType { + pwsval, err := i.getBoolean(pwsv) + if err != nil { + return nil, err + } + pws = pwsval.value + } + + json, err := BuildJsonmlFromString(s, pws) + if err != nil { + return nil, i.Error(fmt.Sprintf("failed to parse XML: %v", err.Error())) + } + + arr, err := arrayToValue(i, json) + if err != nil { + return nil, err + } + return arr, nil +} + +func arrayToValue(i *interpreter, json []interface{}) (*valueArray, error) { + var elements []*cachedThunk + var err error + for _, e := range json { + var val value + switch e := e.(type) { + case string: + val = makeValueString(e) + case map[string]interface{}: + val, err = jsonToValue(i, e) + if err != nil { + return nil, err + } + case []interface{}: + val, err = arrayToValue(i, e) + if err != nil { + return nil, err + } + default: + return nil, i.Error(fmt.Sprintf("invalid type for section: %v", reflect.TypeOf(e))) + } + elements = append(elements, readyThunk(val)) + } + + return makeValueArray(elements), nil +} + func jsonEncode(v interface{}) (string, error) { buf := new(bytes.Buffer) enc := json.NewEncoder(buf) @@ -2097,12 +2155,12 @@ func builtinAvg(i *interpreter, arrv value) (value, error) { if err != nil { return nil, err } - + len := float64(arr.length()) if len == 0 { return nil, i.Error("Cannot calculate average of an empty array.") } - + sumValue, err := builtinSum(i, arrv) if err != nil { return nil, err @@ -2112,7 +2170,7 @@ func builtinAvg(i *interpreter, arrv value) (value, error) { return nil, err } - avg := sum.value/len + avg := sum.value / len return makeValueNumber(avg), nil } @@ -2520,6 +2578,7 @@ var funcBuiltins = buildBuiltinMap([]builtin{ &unaryBuiltin{name: "parseInt", function: builtinParseInt, params: ast.Identifiers{"str"}}, &unaryBuiltin{name: "parseJson", function: builtinParseJSON, params: ast.Identifiers{"str"}}, &unaryBuiltin{name: "parseYaml", function: builtinParseYAML, params: ast.Identifiers{"str"}}, + &generalBuiltin{name: "parseXmlJsonml", function: builtinParseXmlJsonml, params: []generalBuiltinParameter{{name: "str"}, {name: "preserveWhitespace", defaultValue: &nullValue}}}, &generalBuiltin{name: "manifestJsonEx", function: builtinManifestJSONEx, params: []generalBuiltinParameter{{name: "value"}, {name: "indent"}, {name: "newline", defaultValue: &valueFlatString{value: []rune("\n")}}, {name: "key_val_sep", defaultValue: &valueFlatString{value: []rune(": ")}}}}, diff --git a/jsonml.go b/jsonml.go new file mode 100644 index 00000000..0ae8a2ce --- /dev/null +++ b/jsonml.go @@ -0,0 +1,149 @@ +/* +Copyright 2019 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jsonnet + +import ( + "encoding/xml" + "errors" + "fmt" + "io" + "strings" +) + +type stack struct { + elements []interface{} +} + +func (s *stack) Push(e interface{}) { + s.elements = append(s.elements, e) +} + +func (s *stack) Pop() (interface{}, error) { + if len(s.elements) == 0 { + return nil, errors.New("cannot pop from empty stack") + } + l := len(s.elements) + e := s.elements[l-1] + s.elements = s.elements[:l-1] + return e, nil +} + +func (s *stack) Size() int { + return len(s.elements) +} + +type jsonMLBuilder struct { + stack *stack + preserveWhitespace bool + currDepth int +} + +// BuildJsonmlFromString returns a jsomML form of given xml string. +func BuildJsonmlFromString(s string, preserveWhitespace bool) ([]interface{}, error) { + b := newBuilder(preserveWhitespace) + d := xml.NewDecoder(strings.NewReader(s)) + + for { + token, err := d.Token() + if err == io.EOF { + break + } + + if err != nil { + return nil, err + } + + if err := b.addToken(token); err != nil { + return nil, err + } + } + + if b.stack.Size() == 0 { + // No nodes has been identified + return nil, fmt.Errorf("%s is not a valid XML", s) + } + + return b.build(), nil +} + +func newBuilder(preserveWhitespace bool) *jsonMLBuilder { + return &jsonMLBuilder{ + stack: &stack{}, + preserveWhitespace: preserveWhitespace, + } +} + +func (b *jsonMLBuilder) addToken(token xml.Token) error { + switch token.(type) { + case xml.StartElement: + // check for multiple roots + if b.currDepth == 0 && b.stack.Size() > 0 { + // There are multiple root elements + return errors.New("XML cannot have multiple root elements") + } + + t := token.(xml.StartElement) + node := []interface{}{t.Name.Local} + // Add Attributes + if len(t.Attr) > 0 { + attr := make(map[string]interface{}) + for _, a := range t.Attr { + attr[a.Name.Local] = a.Value + } + node = append(node, attr) + } + b.stack.Push(node) + b.currDepth++ + case xml.CharData: + t := token.(xml.CharData) + s := string(t) + if !b.preserveWhitespace { + s = strings.TrimSpace(s) + } + if len(s) > 0 { // Skip empty strings + b.appendToLastNode(string(t)) + } + case xml.EndElement: + b.squashLastNode() + b.currDepth-- + } + + return nil +} + +func (b *jsonMLBuilder) build() []interface{} { + root, _ := b.stack.Pop() + return root.([]interface{}) +} + +func (b *jsonMLBuilder) appendToLastNode(e interface{}) { + if b.stack.Size() == 0 { + return + } + node, _ := b.stack.Pop() + n := node.([]interface{}) + n = append(n, e) + b.stack.Push(n) +} + +func (b *jsonMLBuilder) squashLastNode() { + if b.stack.Size() < 2 { + return + } + n, _ := b.stack.Pop() + b.appendToLastNode(n) +} diff --git a/linter/internal/types/stdlib.go b/linter/internal/types/stdlib.go index 63c0eed3..2646c64e 100644 --- a/linter/internal/types/stdlib.go +++ b/linter/internal/types/stdlib.go @@ -106,13 +106,14 @@ func prepareStdlib(g *typeGraph) { // Parsing - "parseInt": g.newSimpleFuncType(numberType, "str"), - "parseOctal": g.newSimpleFuncType(numberType, "str"), - "parseHex": g.newSimpleFuncType(numberType, "str"), - "parseJson": g.newSimpleFuncType(jsonType, "str"), - "parseYaml": g.newSimpleFuncType(jsonType, "str"), - "encodeUTF8": g.newSimpleFuncType(numberArrayType, "str"), - "decodeUTF8": g.newSimpleFuncType(stringType, "arr"), + "parseInt": g.newSimpleFuncType(numberType, "str"), + "parseOctal": g.newSimpleFuncType(numberType, "str"), + "parseHex": g.newSimpleFuncType(numberType, "str"), + "parseJson": g.newSimpleFuncType(jsonType, "str"), + "parseYaml": g.newSimpleFuncType(jsonType, "str"), + "parseXmlJsonml": g.newFuncType(jsonType, []ast.Parameter{required("str"), optional("preserveWhitespace")}), + "encodeUTF8": g.newSimpleFuncType(numberArrayType, "str"), + "decodeUTF8": g.newSimpleFuncType(stringType, "arr"), // Manifestation @@ -152,7 +153,7 @@ func prepareStdlib(g *typeGraph) { "minArray": g.newFuncType(anyArrayType, []ast.Parameter{required("arr"), optional("keyF")}), "maxArray": g.newFuncType(anyArrayType, []ast.Parameter{required("arr"), optional("keyF")}), "contains": g.newSimpleFuncType(boolType, "arr", "elem"), - "avg": g.newSimpleFuncType(numberType, "arr"), + "avg": g.newSimpleFuncType(numberType, "arr"), "all": g.newSimpleFuncType(boolArrayType, "arr"), "any": g.newSimpleFuncType(boolArrayType, "arr"), "remove": g.newSimpleFuncType(anyArrayType, "arr", "elem"), diff --git a/testdata/builtinParseXmlJsonml.golden b/testdata/builtinParseXmlJsonml.golden new file mode 100644 index 00000000..607ef562 --- /dev/null +++ b/testdata/builtinParseXmlJsonml.golden @@ -0,0 +1,18 @@ +[ + "svg", + { + "height": "100", + "width": "100" + }, + [ + "circle", + { + "cx": "50", + "cy": "50", + "fill": "red", + "r": "40", + "stroke": "black", + "stroke-width": "3" + } + ] +] diff --git a/testdata/builtinParseXmlJsonml.jsonnet b/testdata/builtinParseXmlJsonml.jsonnet new file mode 100644 index 00000000..c5b8f526 --- /dev/null +++ b/testdata/builtinParseXmlJsonml.jsonnet @@ -0,0 +1 @@ +std.parseXmlJsonml('') \ No newline at end of file diff --git a/testdata/builtinParseXmlJsonml.linter.golden b/testdata/builtinParseXmlJsonml.linter.golden new file mode 100644 index 00000000..e69de29b diff --git a/testdata/builtinParseXmlJsonml2.golden b/testdata/builtinParseXmlJsonml2.golden new file mode 100644 index 00000000..84826ae2 --- /dev/null +++ b/testdata/builtinParseXmlJsonml2.golden @@ -0,0 +1,7 @@ +[ + "svg", + [ + "circle", + " Foobar" + ] +] diff --git a/testdata/builtinParseXmlJsonml2.jsonnet b/testdata/builtinParseXmlJsonml2.jsonnet new file mode 100644 index 00000000..99d80a23 --- /dev/null +++ b/testdata/builtinParseXmlJsonml2.jsonnet @@ -0,0 +1 @@ +std.parseXmlJsonml(' Foobar', preserveWhitespace=true) \ No newline at end of file diff --git a/testdata/builtinParseXmlJsonml2.linter.golden b/testdata/builtinParseXmlJsonml2.linter.golden new file mode 100644 index 00000000..e69de29b